<commit_before># -*- coding: utf-8 -*-
"""Configuration Loader loads configuration from external YAML file.
"""
import yaml
from pluginDBAPI import DBAPIconfiguration
from acceptor import Acceptor

class ConfigurationError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

class ConfigLoader(object):
    """ ConfigLoader loads configuration from file.
        Configuration is loaded through load()
        Later, DB configuration, URI and Content-Type acceptors can be
        retrieved through getters.
    """
    def __init__(self):
        self.version = 1.01
        self.dbconf = DBAPIconfiguration()
        self.typeAcceptor = None
        self.uriAcceptor = None
        self.entryPoints = []
        self.maxDepth = 0
        self.loaded = False

    def load(self, fname):
        """Loads configuration from YAML file.
        """
        cfile = open(fname)
        root = yaml.safe_load(cfile)

        db_check = self.db_check(root)
        version_check = self.version_check(root)
        if db_check and version_check:
            self.loaded = self.set_up(root)

        cfile.close()

    def db_check(self, root):
        if 'database' not in root:
            print("Database not specified")
            return False
        else:
            return True

    def version_check(self, root):
        version_check = False
        if 'version' not in root:
            print("Version not specified")
        elif root['version'] == self.version:
            version_check = True
        else:
            print("Configuration version doesn't match")
        return version_check

    def set_max_depth(self, root):
        if 'maxDepth' in root:
            if root['maxDepth'] >= 0:
                self.maxDepth = root['maxDepth']
            else:
                print("Max depth must be zero or positive! Setting to 0.")

    def set_entry_points(self, root):
        if 'entryPoints' not in root:
            print("Entry points should be specified")
        elif not root['entryPoints']:
            print("At least one entry point should be specified")
        else:
            epSet = root['entryPoints']
            for ep in epSet:
                self.entryPoints.append(ep)

    def set_up(self, root):
        self.dbconf.setDbname(root['database'])

        self.set_max_depth(root)
        self.set_entry_points(root)

        cts = 'content-types'
        ct = 'content-type'
        ct_dsc = 'Content type'

        us = 'urls'
        u = 'url'
        u_dsc = 'URL'

        try:
            self.typeAcceptor = ConfigLoader.get_acceptor(cts, ct, ct_dsc, root)
            self.uriAcceptor = ConfigLoader.get_acceptor(us, u, u_dsc, root)
        except ConfigurationError as e:
            print(e.msg)
            return False
        return True

    @staticmethod
    def get_acceptor(tags_string, tag_string, description, root):
        acceptor = Acceptor(False)
        if tags_string in root:
            tags = root[tags_string]
            if tags:
                ConfigLoader.run_tags(tags, description, acceptor, tag_string)
        return acceptor

    @staticmethod
    def run_tags(tags, description, acceptor, tag_string):
        for tag in tags:
            if tag_string not in tag:
                raise ConfigurationError(description+" not specified")
            if 'plugins' in tag:
                ConfigLoader.set_plugin_accept_tag_value(tag, tag_string, acceptor)
            else:
                print("Forbid "+tag[tag_string])
                acceptor.setDefaultAcceptValue(tag[tag_string], False)

    @staticmethod
    def set_plugin_accept_tag_value(tag, tag_string, acceptor):
        if tag['plugins']:
            for plugin in tag['plugins']:
                acceptor.setPluginAcceptValue(plugin, tag[tag_string], True)

    def getDbconf(self):
        """ Retrieve DB configuration.
        """
        if self.loaded:
            return self.dbconf
        else:
            return None

    def getTypeAcceptor(self):
        """ Retrieve Acceptor instance for Content-Type.
        """
        if self.loaded:
            return self.typeAcceptor
        else:
            return None

    def getUriAcceptor(self):
        """ Retrieve Acceptor instance for URI.
        """
        if self.loaded:
            return self.uriAcceptor
        else:
            return None

    def getEntryPoints(self):
        """ Retrieve list of URIs for initial requests.
        """
        if self.loaded:
            return self.entryPoints
        else:
            return None

    def getMaxDepth(self):
        """ Get maximum depth for crawling (0 for no limit)
        """
        if self.loaded:
            return self.maxDepth
        else:
            return None

    def getDefaults(self, root, setKeyword, keyword):
        defaultSet = root.find(setKeyword)
        if defaultSet is not None:
            acceptor = Acceptor(defaultSet.attrib['default'] == 'True')

            defaultElements = defaultSet.findall(keyword)
            if defaultElements is not None:
                for element in defaultElements:
                    acceptor.setDefaultAcceptValue(element.attrib['key'],
                                                   element.attrib['accept'] ==
                                                   'True')

                return acceptor
            else:
                return None

    def getResolutions(self, root):
        defaults = root.find('resolutions')
        if defaults is not None:
            self.uriAcceptor = self.getDefaults(defaults, 'uris', 'uri')
            self.typeAcceptor = self.getDefaults(defaults, 'contentTypes',
                                                 'contentType')

    def getPluginDefaults(self, pluginId, root, setKeyword, keyword, acceptor):
        defaultSet = root.find(setKeyword)
        if defaultSet is not None:
            value = defaultSet.attrib['default'] == 'True'
            acceptor.setPluginAcceptValueDefault(pluginId, value)

            defaultElements = defaultSet.findall(keyword)
            if defaultElements is not None:
                for element in defaultElements:
                    acceptor.setPluginAcceptValue(pluginId,
                                                  element.attrib['key'],
                                                  element.attrib['accept'] ==
                                                  'True')

    def getPluginResolutions(self, pluginId, root, uriAcceptor, typeAcceptor):
        defaults = root.find('resolutions')
        if defaults is not None:
            u = 'uri'
            us = 'uris'
            self.getPluginDefaults(pluginId, defaults, us, u, uriAcceptor)
            self.getPluginDefaults(pluginId, defaults, 'contentTypes',
                                   'contentType', typeAcceptor)
<commit_msg>ConfigLoader - refactoring: remove dead code<commit_after># -*- coding: utf-8 -*-
"""Configuration Loader loads configuration from external YAML file.
"""
import yaml
from pluginDBAPI import DBAPIconfiguration
from acceptor import Acceptor

class ConfigurationError(Exception):
    def __init__(self, msg):
        self.msg = msg

class ConfigLoader(object):
    """ ConfigLoader loads configuration from file.
        Configuration is loaded through load()
        Later, DB configuration, URI and Content-Type acceptors can be
        retrieved through getters.
    """
    def __init__(self):
        self.version = 1.01
        self.dbconf = DBAPIconfiguration()
        self.typeAcceptor = None
        self.uriAcceptor = None
        self.entryPoints = []
        self.maxDepth = 0
        self.loaded = False

    def load(self, fname):
        """Loads configuration from YAML file.
        """
        cfile = open(fname)
        root = yaml.safe_load(cfile)

        db_check = self.db_check(root)
        version_check = self.version_check(root)
        if db_check and version_check:
            self.loaded = self.set_up(root)

        cfile.close()

    def db_check(self, root):
        if 'database' not in root:
            print("Database not specified")
            return False
        else:
            return True

    def version_check(self, root):
        version_check = False
        if 'version' not in root:
            print("Version not specified")
        elif root['version'] == self.version:
            version_check = True
        else:
            print("Configuration version doesn't match")
        return version_check

    def set_max_depth(self, root):
        if 'maxDepth' in root:
            if root['maxDepth'] >= 0:
                self.maxDepth = root['maxDepth']
            else:
                print("Max depth must be zero or positive! Setting to 0.")

    def set_entry_points(self, root):
        if 'entryPoints' not in root:
            print("Entry points should be specified")
        elif not root['entryPoints']:
            print("At least one entry point should be specified")
        else:
            epSet = root['entryPoints']
            for ep in epSet:
                self.entryPoints.append(ep)

    def set_up(self, root):
        self.dbconf.setDbname(root['database'])

        self.set_max_depth(root)
        self.set_entry_points(root)

        cts = 'content-types'
        ct = 'content-type'
        ct_dsc = 'Content type'

        us = 'urls'
        u = 'url'
        u_dsc = 'URL'

        try:
            self.typeAcceptor = ConfigLoader.get_acceptor(cts, ct, ct_dsc, root)
            self.uriAcceptor = ConfigLoader.get_acceptor(us, u, u_dsc, root)
        except ConfigurationError as e:
            print(e.msg)
            return False
        return True

    @staticmethod
    def get_acceptor(tags_string, tag_string, description, root):
        acceptor = Acceptor(False)
        if tags_string in root:
            tags = root[tags_string]
            if tags:
                ConfigLoader.run_tags(tags, description, acceptor, tag_string)
        return acceptor

    @staticmethod
    def run_tags(tags, description, acceptor, tag_string):
        for tag in tags:
            if tag_string not in tag:
                raise ConfigurationError(description+" not specified")
            if 'plugins' in tag:
                ConfigLoader.set_plugin_accept_tag_value(tag, tag_string, acceptor)
            else:
                print("Forbid "+tag[tag_string])
                acceptor.setDefaultAcceptValue(tag[tag_string], False)

    @staticmethod
    def set_plugin_accept_tag_value(tag, tag_string, acceptor):
        if tag['plugins']:
            for plugin in tag['plugins']:
                acceptor.setPluginAcceptValue(plugin, tag[tag_string], True)

    def getDbconf(self):
        """ Retrieve DB configuration.
        """
        if self.loaded:
            return self.dbconf
        else:
            return None

    def getTypeAcceptor(self):
        """ Retrieve Acceptor instance for Content-Type.
        """
        if self.loaded:
            return self.typeAcceptor
        else:
            return None

    def getUriAcceptor(self):
        """ Retrieve Acceptor instance for URI.
        """
        if self.loaded:
            return self.uriAcceptor
        else:
            return None

    def getEntryPoints(self):
        """ Retrieve list of URIs for initial requests.
        """
        if self.loaded:
            return self.entryPoints
        else:
            return None

    def getMaxDepth(self):
        """ Get maximum depth for crawling (0 for no limit)
        """
        if self.loaded:
            return self.maxDepth
        else:
            return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

import datetime
import email
import email.header
import fileinput
import logging
from logging.handlers import SysLogHandler
import smtplib
import sys

import six

from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.utils import timezone
from django.utils import translation
from django.utils.encoding import smart_text
from django.utils.formats import localize

from modoboa.admin.models import Mailbox
from modoboa.lib.email_utils import split_mailbox
from modoboa.parameters import tools as param_tools

from ...models import ARmessage, ARhistoric
from ...modo_extension import PostfixAutoreply

logger = logging.getLogger()

def safe_subject(msg):
    """Clean message subject and return it."""
    decoded = email.header.decode_header(msg.get("Subject"))
    subject = ""
    for sub, charset in decoded:
        if isinstance(sub, six.text_type):
            subject += sub
            continue
        # charset can be None
        charset = charset if charset else "utf-8"
        try:
            subject += sub.decode(charset)
        except UnicodeDecodeError:
            pass
    return " ".join(subject.split())

def send_autoreply(sender, mailbox, armessage, original_msg):
    """Send an autoreply message."""
    if armessage.fromdate > timezone.now():
        # Too soon, come back later
        return

    condition = (
        armessage.untildate is not None and
        armessage.untildate < timezone.now())
    if condition:
        # ARmessage has expired, disable it
        armessage.enabled = False
        armessage.save(update_fields=["enabled"])
        return

    try:
        lastar = ARhistoric.objects.get(armessage=armessage.id, sender=sender)
        timeout = param_tools.get_global_parameter(
            "autoreplies_timeout", app="modoboa_postfix_autoreply")
        delta = datetime.timedelta(seconds=int(timeout))
        now = timezone.make_aware(datetime.datetime.now(),
                                  timezone.get_default_timezone())
        if lastar.last_sent + delta > now:
            logger.debug(
                "no autoreply message sent because delta (%s) < timetout (%s)",
                delta, timeout
            )
            return

    except ARhistoric.DoesNotExist:
        lastar = ARhistoric()
        lastar.armessage = armessage
        lastar.sender = sender

    headers = {
        "Auto-Submitted": "auto-replied",
        "Precedence": "bulk"
    }
    message_id = original_msg.get("Message-ID")
    if message_id:
        headers.update({"In-Reply-To": message_id, "References": message_id})

    subject = safe_subject(original_msg)
    with translation.override(mailbox.user.language):
        context = {
            "name": mailbox.user.fullname,
            "fromdate": localize(armessage.fromdate),
            "untildate": ""
        }
        if armessage.untildate:
            context.update({"untildate": localize(armessage.untildate)})
    content = armessage.content % context
    msg = EmailMessage(
        "Auto: {} Re: {}".format(armessage.subject, subject),
        smart_text(content),
        mailbox.user.encoded_address,
        [sender],
        headers=headers
    )
    try:
        msg.send()
    except smtplib.SMTPException as exp:
        logger.error("Failed to send autoreply message: %s", exp)
        sys.exit(1)

    logger.debug(
        "autoreply message sent to %s", mailbox.user.encoded_address)

    lastar.last_sent = datetime.datetime.now()
    lastar.save()

class Command(BaseCommand):
    """Command definition."""

    help = "Send autoreply emails"

    def add_arguments(self, parser):
        """Add extra arguments to command line."""
        parser.add_argument(
            "--debug", action="store_true", dest="debug", default=False
        )
        parser.add_argument(
            "--syslog-socket-path", default="/dev/log",
            help="Path to syslog socket"
        )
        parser.add_argument("sender")
        parser.add_argument("recipient", nargs="+")

    def handle(self, *args, **options):
        logger.addHandler(
            SysLogHandler(address=options["syslog_socket_path"]))
        logger.setLevel(logging.ERROR)
        if options["debug"]:
            logger.setLevel(logging.DEBUG)

        logger.debug(
            "autoreply sender=%s recipient=%s",
            options["sender"], ",".join(options["recipient"])
        )

        sender = options["sender"]

        sender_localpart = split_mailbox(sender.lower())[0]
        if (
            (sender_localpart in ('mailer-daemon', 'listserv', 'majordomo')) or
            (sender_localpart.startswith('owner-')) or
            (sender_localpart.endswith('-request'))
        ):
            logger.debug(
                "Skip auto reply, this mail comes from a mailing list")
            return

        content = six.StringIO()
        for line in fileinput.input([]):
            content.write(line)
        content.seek(0)

        original_msg = email.message_from_file(content)

        # Mailing list filter based on
        # https://tools.ietf.org/html/rfc5230#page-7
        ml_known_headers = [
            "List-Id", "List-Help", "List-Subscribe", "List-Unsubscribe",
            "List-Post", "List-Owner", "List-Archive"
        ]
        from_ml = False
        for header in ml_known_headers:
            if header in original_msg:
                from_ml = True
                break
        condition = (
            original_msg.get("Precedence") == "bulk" or
            original_msg.get("X-Mailer") == "PHPMailer" or
            from_ml
        )
        if condition:
            logger.debug(
                "Skip auto reply, this mail comes from a mailing list")
            return

        PostfixAutoreply().load()
        for fulladdress in options["recipient"]:
            address, domain = split_mailbox(fulladdress)
            try:
                mbox = Mailbox.objects.get(
                    address=address, domain__name=domain)
            except Mailbox.DoesNotExist:
                msg = "Unknown recipient %s" % (fulladdress)
                logger.debug("autoreply %s", msg)
                continue
            try:
                armessage = ARmessage.objects.get(mbox=mbox.id, enabled=True)
                logger.debug("autoreply message found")
            except ARmessage.DoesNotExist:
                logger.debug("autoreply message not found")
                continue

            send_autoreply(sender, mbox, armessage, original_msg)
<commit_msg>Py >= 3.5 SysLogHandler silently fails if /dev/log isn't available<commit_after>#!/usr/bin/env python
# coding: utf-8

from __future__ import unicode_literals

import datetime
import email
import email.header
import fileinput
import logging
from logging.handlers import SysLogHandler
import smtplib
import socket
import sys

import six

from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.utils import timezone
from django.utils import translation
from django.utils.encoding import smart_text
from django.utils.formats import localize

from modoboa.admin.models import Mailbox
from modoboa.lib.email_utils import split_mailbox
from modoboa.parameters import tools as param_tools

from ...models import ARmessage, ARhistoric
from ...modo_extension import PostfixAutoreply

logger = logging.getLogger()

def safe_subject(msg):
    """Clean message subject and return it."""
    decoded = email.header.decode_header(msg.get("Subject"))
    subject = ""
    for sub, charset in decoded:
        if isinstance(sub, six.text_type):
            subject += sub
            continue
        # charset can be None
        charset = charset if charset else "utf-8"
        try:
            subject += sub.decode(charset)
        except UnicodeDecodeError:
            pass
    return " ".join(subject.split())

def send_autoreply(sender, mailbox, armessage, original_msg):
    """Send an autoreply message."""
    if armessage.fromdate > timezone.now():
        # Too soon, come back later
        return

    condition = (
        armessage.untildate is not None and
        armessage.untildate < timezone.now())
    if condition:
        # ARmessage has expired, disable it
        armessage.enabled = False
        armessage.save(update_fields=["enabled"])
        return

    try:
        lastar = ARhistoric.objects.get(armessage=armessage.id, sender=sender)
        timeout = param_tools.get_global_parameter(
            "autoreplies_timeout", app="modoboa_postfix_autoreply")
        delta = datetime.timedelta(seconds=int(timeout))
        now = timezone.make_aware(datetime.datetime.now(),
                                  timezone.get_default_timezone())
        if lastar.last_sent + delta > now:
            logger.debug(
                "no autoreply message sent because delta (%s) < timetout (%s)",
                delta, timeout
            )
            return

    except ARhistoric.DoesNotExist:
        lastar = ARhistoric()
        lastar.armessage = armessage
        lastar.sender = sender

    headers = {
        "Auto-Submitted": "auto-replied",
        "Precedence": "bulk"
    }
    message_id = original_msg.get("Message-ID")
    if message_id:
        headers.update({"In-Reply-To": message_id, "References": message_id})

    subject = safe_subject(original_msg)
    with translation.override(mailbox.user.language):
        context = {
            "name": mailbox.user.fullname,
            "fromdate": localize(armessage.fromdate),
            "untildate": ""
        }
        if armessage.untildate:
            context.update({"untildate": localize(armessage.untildate)})
    content = armessage.content % context
    msg = EmailMessage(
        "Auto: {} Re: {}".format(armessage.subject, subject),
        smart_text(content),
        mailbox.user.encoded_address,
        [sender],
        headers=headers
    )
    try:
        msg.send()
    except smtplib.SMTPException as exp:
        logger.error("Failed to send autoreply message: %s", exp)
        sys.exit(1)

    logger.debug(
        "autoreply message sent to %s", mailbox.user.encoded_address)

    lastar.last_sent = datetime.datetime.now()
    lastar.save()

class Command(BaseCommand):
    """Command definition."""

    help = "Send autoreply emails"

    def add_arguments(self, parser):
        """Add extra arguments to command line."""
        parser.add_argument(
            "--debug", action="store_true", dest="debug", default=False
        )
        parser.add_argument(
            "--syslog-socket-path", default="/dev/log",
            help="Path to syslog socket"
        )
        parser.add_argument("sender")
        parser.add_argument("recipient", nargs="+")

    def handle(self, *args, **options):
        try:
            handler = SysLogHandler(address=options["syslog_socket_path"])
        except socket.error as ex:
            if ex.errno == 2:
                # try the default, localhost:514
                handler = SysLogHandler()
            else:
                raise

        logger.addHandler(handler)
        logger.setLevel(logging.ERROR)
        if options["debug"]:
            logger.setLevel(logging.DEBUG)

        logger.debug(
            "autoreply sender=%s recipient=%s",
            options["sender"], ",".join(options["recipient"])
        )

        sender = options["sender"]

        sender_localpart = split_mailbox(sender.lower())[0]
        if (
            (sender_localpart in ('mailer-daemon', 'listserv', 'majordomo')) or
            (sender_localpart.startswith('owner-')) or
            (sender_localpart.endswith('-request'))
        ):
            logger.debug(
                "Skip auto reply, this mail comes from a mailing list")
            return

        content = six.StringIO()
        for line in fileinput.input([]):
            content.write(line)
        content.seek(0)

        original_msg = email.message_from_file(content)

        # Mailing list filter based on
        # https://tools.ietf.org/html/rfc5230#page-7
        ml_known_headers = [
            "List-Id", "List-Help", "List-Subscribe", "List-Unsubscribe",
            "List-Post", "List-Owner", "List-Archive"
        ]
        from_ml = False
        for header in ml_known_headers:
            if header in original_msg:
                from_ml = True
                break
        condition = (
            original_msg.get("Precedence") == "bulk" or
            original_msg.get("X-Mailer") == "PHPMailer" or
            from_ml
        )
        if condition:
            logger.debug(
                "Skip auto reply, this mail comes from a mailing list")
            return

        PostfixAutoreply().load()
        for fulladdress in options["recipient"]:
            address, domain = split_mailbox(fulladdress)
            try:
                mbox = Mailbox.objects.get(
                    address=address, domain__name=domain)
            except Mailbox.DoesNotExist:
                msg = "Unknown recipient %s" % (fulladdress)
                logger.debug("autoreply %s", msg)
                continue
            try:
                armessage = ARmessage.objects.get(mbox=mbox.id, enabled=True)
                logger.debug("autoreply message found")
            except ARmessage.DoesNotExist:
                logger.debug("autoreply message not found")
                continue

            send_autoreply(sender, mbox, armessage, original_msg)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding:utf-8 -*-

import logging
from datetime import datetime, timedelta
from random import getrandbits
from os import urandom

from google.appengine.dist import use_library
use_library('django', '1.2')

from google.appengine.ext import db, webapp
from google.appengine.ext.webapp import template

class Account(db.Model):
    session_id = db.StringProperty()
    session_token = db.StringProperty()
    last_login = db.DateTimeProperty()
    imakoko_user = db.StringProperty()
    imakoko_secret = db.StringProperty()
    twitter_user = db.StringProperty()
    twitter_token = db.StringProperty()
    twitter_secret = db.StringProperty()

class BasePage(webapp.RequestHandler):
    def get_sid_str(self):
        sid_str = self.request.cookies.get(u'IMAKOKO_SID')
        return str(sid_str) if sid_str else ''

    def get_account(self):
        sid_str = self.get_sid_str()
        if not sid_str:
            return None
        id_pair = sid_str.split('_', 1)
        if len(id_pair) != 2 or not id_pair[0].isdigit():
            return None
        account = Account.get_by_id(long(id_pair[0]))
        if account and account.session_id == id_pair[1]:
            return account
        return None

    def put_with_new_sid(self, account):
        account.session_id = urandom(16).encode('hex')
        account.session_token = '%016x' % getrandbits(64)
        account.last_login = datetime.utcnow()
        account.put()
        self.set_sid_cookie(account)

    def set_sid_cookie(self, account):
        sid_str = '%d_%s' % (account.key().id(), account.session_id)
        expires = account.last_login + timedelta(90)
        self.response.headers.add_header(
            'Set-Cookie',
            'IMAKOKO_SID=' + sid_str + '; expires=' + expires.strftime('%a, %d-%b-%Y %H:%M:%S GMT') + '; path=/; httponly')

    def create_temporary_sid(self):
        sid_str = '%032x' % getrandbits(128)
        expires = datetime.utcnow() + timedelta(hours=1)
        self.response.headers.add_header(
            'Set-Cookie',
            'IMAKOKO_SID=' + sid_str + '; expires=' + expires.strftime('%a, %d-%b-%Y %H:%M:%S GMT') + '; path=/; httponly')
        return sid_str

    def show_html(self, html_file, template_values={}):
        self.response.headers['Content-Type'] = 'text/html; charset=UTF-8'
        self.response.out.write(template.render(html_file, template_values))

    def show_error_page(self):
        self.error(500)
        self.show_html('error.html')

    def handle_exception(self, exception, debug_mode):
        logging.exception(exception)
        self.show_error_page()
<commit_msg>Disable indexing for properties other than last_login.<commit_after>#!/usr/bin/env python
# -*- coding:utf-8 -*-

import logging
from datetime import datetime, timedelta
from random import getrandbits
from os import urandom

from google.appengine.dist import use_library
use_library('django', '1.2')

from google.appengine.ext import db, webapp
from google.appengine.ext.webapp import template

class Account(db.Model):
    session_id = db.StringProperty(indexed=False)
    session_token = db.StringProperty(indexed=False)
    last_login = db.DateTimeProperty()
    imakoko_user = db.StringProperty(indexed=False)
    imakoko_secret = db.StringProperty(indexed=False)
    twitter_user = db.StringProperty(indexed=False)
    twitter_token = db.StringProperty(indexed=False)
    twitter_secret = db.StringProperty(indexed=False)

class BasePage(webapp.RequestHandler):
    def get_sid_str(self):
        sid_str = self.request.cookies.get(u'IMAKOKO_SID')
        return str(sid_str) if sid_str else ''

    def get_account(self):
        sid_str = self.get_sid_str()
        if not sid_str:
            return None
        id_pair = sid_str.split('_', 1)
        if len(id_pair) != 2 or not id_pair[0].isdigit():
            return None
        account = Account.get_by_id(long(id_pair[0]))
        if account and account.session_id == id_pair[1]:
            return account
        return None

    def put_with_new_sid(self, account):
        account.session_id = urandom(16).encode('hex')
        account.session_token = '%016x' % getrandbits(64)
        account.last_login = datetime.utcnow()
        account.put()
        self.set_sid_cookie(account)

    def set_sid_cookie(self, account):
        sid_str = '%d_%s' % (account.key().id(), account.session_id)
        expires = account.last_login + timedelta(90)
        self.response.headers.add_header(
            'Set-Cookie',
            'IMAKOKO_SID=' + sid_str + '; expires=' + expires.strftime('%a, %d-%b-%Y %H:%M:%S GMT') + '; path=/; httponly')

    def create_temporary_sid(self):
        sid_str = '%032x' % getrandbits(128)
        expires = datetime.utcnow() + timedelta(hours=1)
        self.response.headers.add_header(
            'Set-Cookie',
            'IMAKOKO_SID=' + sid_str + '; expires=' + expires.strftime('%a, %d-%b-%Y %H:%M:%S GMT') + '; path=/; httponly')
        return sid_str

    def show_html(self, html_file, template_values={}):
        self.response.headers['Content-Type'] = 'text/html; charset=UTF-8'
        self.response.out.write(template.render(html_file, template_values))

    def show_error_page(self):
        self.error(500)
        self.show_html('error.html')

    def handle_exception(self, exception, debug_mode):
        logging.exception(exception)
        self.show_error_page()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys, os, json, re, time
import getpass
import subprocess
import readline

dopip = False
try:
    from elasticsearch import Elasticsearch
    from formatflowed import convertToWrapped
except:
    dopip = True

if dopip and (getpass.getuser() != "root"):
    print("It looks like you need to install some python modules first")
    print("Either run this as root to do so, or run: ")
    print("pip install elasticsearch formatflowed")
    sys.exit(-1)

elif dopip:
    print("Before we get started, we need to install some modules")
    print("Hang on!")
    try:
        subprocess.check_call(('pip3','install','elasticsearch','formatflowed'))
    except:
        print("Oh dear, looks like this failed :(")
        print("Please install elasticsearch and formatflowed before you try again:")
        print("pip install elasticsearch formatflowed")
        sys.exit(-1)

print("Welcome to the Pony Mail setup script!")
print("Let's start by determining some settings...")
print("")

hostname = ""
port = 0
dbname = ""
mlserver = ""
mldom = ""
wc = ""
wce = False

# If called with --defaults (like from Docker), use default values
if ('--defaults' in sys.argv):
    hostname = "localhost"
    port = 9200
    dbname = "ponymail"
    mlserver = "localhost"
    mldom = "example.org"
    wce = True

while hostname == "":
    hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ")

while port < 1:
    port = int(input("What port is ElasticSearch listening on? (normally 9200): "))

while dbname == "":
    dbname = input("What would you like to call the mail index (e.g. ponymail): ")

while mlserver == "":
    mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ")

while mldom == "":
    mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ")

while wc == "":
    wc = input("Would you like to enable the word cloud feature? (Y/N): ")
    if wc.lower() == "y":
        wce = True

print("Okay, I got all I need, setting up Pony Mail...")

print("Creating index " + dbname)

es = Elasticsearch([
    {
        'host': hostname,
        'port': port,
        'use_ssl': False,
        'url_prefix': ''
    }],
    max_retries=5,
    retry_on_timeout=True
    )
mappings = {
    "mbox" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "@import_timestamp" : {
          "type" : "date",
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
        },
        "@version" : {
          "type" : "long"
        },
        "body" : {
          "type" : "string"
        },
        "date" : {
          "type" : "date",
          "store" : True,
          "format" : "yyyy/MM/dd HH:mm:ss",
          "index" : "not_analyzed"
        },
        "epoch" : {
          "type" : "double",
          "index" : "not_analyzed"
        },
        "from" : {
          "type" : "string"
        },
        "from_raw" : {
          "type" : "string",
          "index" : "not_analyzed"
        },
        "in-reply-to" : {
          "type" : "string"
        },
        "list" : {
          "type" : "string"
        },
        "list_raw" : {
          "type" : "string",
          "index" : "not_analyzed"
        },
        "message-id" : {
          "type" : "string"
        },
        "mid" : {
          "type" : "string"
        },
        "private" : {
          "type" : "boolean"
        },
        "references" : {
          "type" : "string"
        },
        "subject" : {
          "type" : "string"
        },
        "to" : {
          "type" : "string"
        }
      }
    },
    "mbox_source" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "string",
          "index" : "not_analyzed"
        }
      }
    },
    "attachment" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "binary"
        }
      }
    },
    "mbox_source" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "binary"
        }
      }
    }
  }

res = es.indices.create(index = dbname, body = {
            "mappings" : mappings
        }
    )

print("Index created!")
print("Writing importer config (ponymail.cfg)")

with open("ponymail.cfg", "w") as f:
    f.write("""
###############################################################
# Pony Mail Configuration file

# Main ES configuration
[elasticsearch]
hostname:               %s
dbname:                 %s
port:                   %u
ssl:                    false

###############################################################
            """ % (hostname, dbname, port))
    f.close()

print("mod_lua configuration (config.lua)")
with open("../site/api/lib/config.lua", "w") as f:
    f.write("""
local config = {
    es_url = "http://%s:%u/%s/",
    mailserver = "%s",
    accepted_domains = "%s",
    wordcloud = %s,
    slow_count = false,
    email_footer = nil, -- see the docs for how to set this up.
    full_headers = false,
    maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists
    admin_oauth = {}, -- list of domains that may do administrative oauth (private list access)
                     -- add 'www.googleapis.com' to the list for google oauth to decide, for instance.
    antispam = true  -- Whether or not to add anti-spam measures aimed at anonymous users.
}
return config
            """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
    f.close()

print("All done, Pony Mail should...work now :)")
<commit_msg>satisfy wordcloud question<commit_after>#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys, os, json, re, time
import getpass
import subprocess
import readline

dopip = False
try:
    from elasticsearch import Elasticsearch
    from formatflowed import convertToWrapped
except:
    dopip = True

if dopip and (getpass.getuser() != "root"):
    print("It looks like you need to install some python modules first")
    print("Either run this as root to do so, or run: ")
    print("pip install elasticsearch formatflowed")
    sys.exit(-1)

elif dopip:
    print("Before we get started, we need to install some modules")
    print("Hang on!")
    try:
        subprocess.check_call(('pip3','install','elasticsearch','formatflowed'))
    except:
        print("Oh dear, looks like this failed :(")
        print("Please install elasticsearch and formatflowed before you try again:")
        print("pip install elasticsearch formatflowed")
        sys.exit(-1)

print("Welcome to the Pony Mail setup script!")
print("Let's start by determining some settings...")
print("")

hostname = ""
port = 0
dbname = ""
mlserver = ""
mldom = ""
wc = ""
wce = False

# If called with --defaults (like from Docker), use default values
if ('--defaults' in sys.argv):
    hostname = "localhost"
    port = 9200
    dbname = "ponymail"
    mlserver = "localhost"
    mldom = "example.org"
    wc = "Y"
    wce = True

while hostname == "":
    hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost): ")

while port < 1:
    port = int(input("What port is ElasticSearch listening on? (normally 9200): "))

while dbname == "":
    dbname = input("What would you like to call the mail index (e.g. ponymail): ")

while mlserver == "":
    mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org): ")

while mldom == "":
    mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org or *): ")

while wc == "":
    wc = input("Would you like to enable the word cloud feature? (Y/N): ")
    if wc.lower() == "y":
        wce = True

print("Okay, I got all I need, setting up Pony Mail...")

print("Creating index " + dbname)

es = Elasticsearch([
    {
        'host': hostname,
        'port': port,
        'use_ssl': False,
        'url_prefix': ''
    }],
    max_retries=5,
    retry_on_timeout=True
    )
mappings = {
    "mbox" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "@import_timestamp" : {
          "type" : "date",
          "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
        },
        "@version" : {
          "type" : "long"
        },
        "body" : {
          "type" : "string"
        },
        "date" : {
          "type" : "date",
          "store" : True,
          "format" : "yyyy/MM/dd HH:mm:ss",
          "index" : "not_analyzed"
        },
        "epoch" : {
          "type" : "double",
          "index" : "not_analyzed"
        },
        "from" : {
          "type" : "string"
        },
        "from_raw" : {
          "type" : "string",
          "index" : "not_analyzed"
        },
        "in-reply-to" : {
          "type" : "string"
        },
        "list" : {
          "type" : "string"
        },
        "list_raw" : {
          "type" : "string",
          "index" : "not_analyzed"
        },
        "message-id" : {
          "type" : "string"
        },
        "mid" : {
          "type" : "string"
        },
        "private" : {
          "type" : "boolean"
        },
        "references" : {
          "type" : "string"
        },
        "subject" : {
          "type" : "string"
        },
        "to" : {
          "type" : "string"
        }
      }
    },
    "mbox_source" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "string",
          "index" : "not_analyzed"
        }
      }
    },
    "attachment" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "binary"
        }
      }
    },
    "mbox_source" : {
      "_size" : {
        "enabled" : True,
        "store" : True
      },
      "properties" : {
        "source" : {
          "type" : "binary"
        }
      }
    }
  }

res = es.indices.create(index = dbname, body = {
            "mappings" : mappings
        }
    )

print("Index created!")
print("Writing importer config (ponymail.cfg)")

with open("ponymail.cfg", "w") as f:
    f.write("""
###############################################################
# Pony Mail Configuration file

# Main ES configuration
[elasticsearch]
hostname:               %s
dbname:                 %s
port:                   %u
ssl:                    false

###############################################################
            """ % (hostname, dbname, port))
    f.close()

print("mod_lua configuration (config.lua)")
with open("../site/api/lib/config.lua", "w") as f:
    f.write("""
local config = {
    es_url = "http://%s:%u/%s/",
    mailserver = "%s",
    accepted_domains = "%s",
    wordcloud = %s,
    slow_count = false,
    email_footer = nil, -- see the docs for how to set this up.
    full_headers = false,
    maxResults = 5000, -- max emails to return in one go. Might need to be bumped for large lists
    admin_oauth = {}, -- list of domains that may do administrative oauth (private list access)
                     -- add 'www.googleapis.com' to the list for google oauth to decide, for instance.
    antispam = true  -- Whether or not to add anti-spam measures aimed at anonymous users.
}
return config
            """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
    f.close()

print("All done, Pony Mail should...work now :)")
<|endoftext|>


<commit_before>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
import csv
import pandas as pd

class DownloadUnzipFR(DownloadUnzipTask):

    resolution = Parameter()
    table_theme = Parameter()

    URL_base = 'http://www.insee.fr/fr/ppp/bases-de-donnees/donnees-detaillees/rp2012/infracommunal/'

    def download(self):

        themes = {
            'population':'infra-population-12/',
            'housing':'infra-logement-12/',
            'education':'infra-formation-12/',
            'household':'infra-famille-12/',
            'employment':'infra-activite-resident-12/'
                }

        iris = {
            'population':'infra-population-2012.zip',
            'housing':'infra-logement-2012.zip',
            'education':'infra-formation-2012.zip',
            'household':'infra-famille-2012.zip',
            'employment':'infra-activite-resident-2012.zip'
        }

        URL = self.URL_base + themes.get(self.table_theme) + iris.get(self.table_theme)

        shell('wget -O {output}.zip {url}'.format(
           output=self.output().path,
           url=URL
        ))

class DownloadFR(Task):

    table_theme = Parameter()

    URL_base = 'http://www.insee.fr/fr/ppp/bases-de-donnees/donnees-detaillees/rp2012/infracommunal/'

    def download(self):

        themes = {
            'population':'infra-population-12/',
            'housing':'infra-logement-12/',
            'education':'infra-formation-12/',
            'household':'infra-famille-12/',
            'employment':'infra-activite-resident-12/'
                }

        iris_overseas = {
            'population':'base-ic-evol-struct-pop-2012-com.xls',
            'housing':'base-ic-logement-2012-com.xls',
            'education':'base-ic-diplomes-formation-2012-com.xls',
            'household':'base-ic-couples-familles-menages-2012-com.xls',
            'employment':'base-ic-activite-residents-2012-com.xls'
                }

        URL = self.URL_base + themes.get(self.table_theme) + iris_overseas.get(self.table_theme)

        shell('wget -P {output} {url}'.format(
           output=self.output().path,
           url=URL
        ))

    def run(self):
        self.output().makedirs()
        self.download()

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class RawFRData(CSV2TempTableTask):

    resolution = Parameter()
    table_theme = Parameter()

    def requires(self):
        if self.resolution == 'iris':
            return DownloadUnzipFR(table_theme=self.table_theme, resolution=self.resolution)
        elif self.resolution == 'iris_overseas':
            return DownloadFR(table_theme=self.table_theme)
        else:
            raise Exception('resolution {} is not permitted'.format(self.resolution))
    def input_csv(self):
        #Read in excel file, it should be the only file in self.input().path
        xls = pd.ExcelFile(os.path.join(self.input().path,os.listdir(self.input().path)[0]))

        #Remove header
        df = xls.parse(skiprows=5,header=0)
        df.to_csv(os.path.join(self.input().path,'{resolution}_{table_theme}.csv'.format(resolution=self.resolution,table_theme=self.table_theme)),index=False,encoding='utf8')
        return os.path.join(self.input().path,'{resolution}_{table_theme}.csv'.format(resolution=self.resolution,table_theme=self.table_theme))

class SourceTags(TagsTask):
    def version(self):
        return 1

    def tags(self):
        return [
            OBSTag(id='insee',
                   name='INSEE',
                   type='source',
                   description='http://www.insee.fr/fr/bases-de-donnees/default.asp?page=recensement/resultats/2012/donnees-detaillees-recensement-2012.htm')
        ]

class FrenchColumns(ColumnsTask):

    table_theme = Parameter()

    def requires(self):
        requirements = {
            'sections': SectionTags(),
            'subsections': SubsectionTags(),
            'unittags': UnitTags(),
            'sourcetag': SourceTags()
        }
        if self.table_theme != 'population':
            requirements['population_vars'] = FrenchColumns(table_theme='population')
        return requirements

    def version(self):
        return 7

    def columns(self):
        cols = OrderedDict()
        input_ = self.input()

        subsectiontags = input_['subsections']
        unittags = input_['unittags']
        france = input_['sections']['fr']
        pop_columns = input_.get('population_vars', {})

        filepath = "frenchmetadata/French Variables - {}.tsv".format(self.table_theme.title())
        session = current_session()
        with open(os.path.join(os.path.dirname(__file__),filepath)) as tsvfile:
            tsvreader = csv.reader(tsvfile, delimiter="\t")
            # Skip first row (header)
            next(tsvreader, None)
            for line in tsvreader:
                # Ignoring "Universe" and "Description" columns for now...
                var_code,short_name,long_name,var_unit,denominators,subsections = line[0], line[1], line[2], line[3], line[4], line[5]
                denominators = denominators.split(',')
                # slugified_lib = underscore_slugify('{}'.format(short_name))
                targets_dict = {}
                for x in denominators:
                    x = x.strip()
                    targets_dict[cols.get(x, pop_columns[x].get(session) if x in pop_columns else None)] = 'denominator'
                targets_dict.pop(None, None)
                cols[var_code] = OBSColumn(
                    id=var_code,
                    type='Numeric',
                    name=short_name,
                    description =long_name,
                    # Ranking of importance, sometimes used to favor certain measures in auto-selection
                    # Weight of 0 will hide this column from the user.  We generally use between 0 and 10
                    weight=5,
                    aggregate='sum',
                    # Tags are our way of noting aspects of this measure like its unit, the country
                    # it's relevant to, and which section(s) of the catalog it should appear in
                    # Need to fix Subsection and UnitTags! Problem with unittag "families"
                    tags=[france, unittags[var_unit]],
                    targets= targets_dict
                )
                subsections = subsections.split(',')
                for s in subsections:
                    s = s.strip()
                    subsection_tag = subsectiontags[s]
                    cols[var_code].tags.append(subsection_tag)

        insee_source = input_['sourcetag']['insee']

        for _,col in cols.iteritems():
            col.tags.append(insee_source)
        return cols

class DownloadOutputAreas(DownloadUnzipTask):
    # Note that this set of IRIS contours is from 2013, may need to find 2014 contours to match the data

    URL = 'https://www.data.gouv.fr/s/resources/contour-des-iris-insee-tout-en-un/20150428-161348/iris-2013-01-01.zip'

    def download(self):
        shell('wget -O {output}.zip {url}'.format(
        output=self.output().path,
        url=self.URL))

class ImportOutputAreas(Shp2TempTableTask):

    def requires(self):
        return DownloadOutputAreas()

    def input_shp(self):
        # may need to point to directory iris-2013-01-01?
        return os.path.join(self.input().path, 'iris-2013-01-01.shp')

class OutputAreaColumns(ColumnsTask):

    def version(self):
        return 1

    def requires(self):
        return {
            'subsections': SubsectionTags(),
            'sections': SectionTags(),
        }

    def columns(self):
        input_ = self.input()
        geom = OBSColumn(
            type='Geometry',
            name='IRIS areas',
            description='IRIS regions are defined by INSEE census for purposes of all municipalities '
                        'of over 10000 inhabitants and most towns from 5000 to 10000.',
            weight=5,
            tags=[input_['subsections']['boundary'], input_['sections']['fr']]
        )
        geomref = OBSColumn(
            type='Text',
            name='DCOMIRIS',
            description='Full Code IRIS. Result of the concatenation of DEPCOM and IRIS attributes. ',
            weight=0,
            targets={geom: GEOM_REF}
        )

        return OrderedDict([
            ('the_geom', geom),
            ('dcomiris', geomref)
        ])

class OutputAreas(TableTask):

    def requires(self):
        return {
            'geom_columns': OutputAreaColumns(),
            'data': ImportOutputAreas(),
        }

    def version(self):
        return 2

    def timespan(self):
        return 2013

    def columns(self):
        input_ = self.input()
        cols = OrderedDict()
        cols.update(input_['geom_columns'])
        return cols

    def populate(self):
        session = current_session()
        session.execute('INSERT INTO {output} '
                        'SELECT ST_MakeValid(wkb_geometry), DCOMIRIS '
                        'FROM {input}'.format(
                            output=self.output().table,
                            input=self.input()['data'].table,
                        ))

class FranceCensus(TableTask):

    table_theme = Parameter()
    resolution = Parameter()

    def version(self):
        return 7

    def timespan(self):
        return '2012'

    def requires(self):
        requirements = {
            'data': RawFRData(table_theme=self.table_theme, resolution=self.resolution),
            'meta': FrenchColumns(table_theme=self.table_theme),
            'geometa': OutputAreaColumns(),
        }
        return requirements

    def columns(self):
        cols = OrderedDict()
        cols['IRIS'] = self.input()['geometa']['dcomiris']
        cols.update(self.input()['meta'])
        return cols

    def populate(self):
        session = current_session()

        column_targets = self.columns()
        colnames = ', '.join(column_targets.keys())
        colnames_typed = ','.join(['{}::{}'.format(colname, ct.get(session).type)
                              for colname, ct in column_targets.iteritems()])
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['data'].table
                        ))

class AllGeomsThemesTables(WrapperTask):
    def requires(self):
        topics = ['population', 'housing', 'education', 'household', 'employment']
        for resolution in ('iris', 'iris_overseas'):
            for table_theme in topics:
                yield FranceCensus(table_theme=table_theme, resolution=resolution)

class AllGeometries(WrapperTask):

    def requires(self):
        geom_types = ('iris', 'overseas_iris')
        for geom in geom_types:
            yield OutputAreas(geom=geom)
<commit_msg>changing wrapper task for all geoms<commit_after>from tasks.util import (Shp2TempTableTask, TempTableTask, TableTask, TagsTask, ColumnsTask,
                        DownloadUnzipTask, CSV2TempTableTask,
                        underscore_slugify, shell, classpath)
from tasks.meta import current_session, DENOMINATOR, GEOM_REF
from collections import OrderedDict
from luigi import IntParameter, Parameter, WrapperTask, Task, LocalTarget
import os
from tasks.meta import OBSTable, OBSColumn, OBSTag
from tasks.tags import SectionTags, SubsectionTags, UnitTags
import csv
import pandas as pd

class DownloadUnzipFR(DownloadUnzipTask):

    table_theme = Parameter()

    URL_base = 'http://www.insee.fr/fr/ppp/bases-de-donnees/donnees-detaillees/rp2012/infracommunal/'

    def download(self):

        themes = {
            'population':'infra-population-12/',
            'housing':'infra-logement-12/',
            'education':'infra-formation-12/',
            'household':'infra-famille-12/',
            'employment':'infra-activite-resident-12/'
                }

        iris = {
            'population':'infra-population-2012.zip',
            'housing':'infra-logement-2012.zip',
            'education':'infra-formation-2012.zip',
            'household':'infra-famille-2012.zip',
            'employment':'infra-activite-resident-2012.zip'
        }

        URL = self.URL_base + themes.get(self.table_theme) + iris.get(self.table_theme)

        shell('wget -O {output}.zip {url}'.format(
           output=self.output().path,
           url=URL
        ))

class DownloadFR(Task):

    table_theme = Parameter()

    URL_base = 'http://www.insee.fr/fr/ppp/bases-de-donnees/donnees-detaillees/rp2012/infracommunal/'

    def download(self):

        themes = {
            'population':'infra-population-12/',
            'housing':'infra-logement-12/',
            'education':'infra-formation-12/',
            'household':'infra-famille-12/',
            'employment':'infra-activite-resident-12/'
                }

        iris_overseas = {
            'population':'base-ic-evol-struct-pop-2012-com.xls',
            'housing':'base-ic-logement-2012-com.xls',
            'education':'base-ic-diplomes-formation-2012-com.xls',
            'household':'base-ic-couples-familles-menages-2012-com.xls',
            'employment':'base-ic-activite-residents-2012-com.xls'
                }

        URL = self.URL_base + themes.get(self.table_theme) + iris_overseas.get(self.table_theme)

        shell('wget -P {output} {url}'.format(
           output=self.output().path,
           url=URL
        ))

    def run(self):
        self.output().makedirs()
        self.download()

    def output(self):
        return LocalTarget(os.path.join('tmp', classpath(self), self.task_id))

class RawFRData(CSV2TempTableTask):

    resolution = Parameter()
    table_theme = Parameter()

    def requires(self):
        if self.resolution == 'iris':
            return DownloadUnzipFR(table_theme=self.table_theme)
        elif self.resolution == 'iris_overseas':
            return DownloadFR(table_theme=self.table_theme)
        else:
            raise Exception('resolution {} is not permitted'.format(self.resolution))
    def input_csv(self):
        #Read in excel file, it should be the only file in self.input().path
        xls = pd.ExcelFile(os.path.join(self.input().path,os.listdir(self.input().path)[0]))

        #Remove header
        df = xls.parse(skiprows=5,header=0)
        df.to_csv(os.path.join(self.input().path,'{resolution}_{table_theme}.csv'.format(resolution=self.resolution,table_theme=self.table_theme)),index=False,encoding='utf8')
        return os.path.join(self.input().path,'{resolution}_{table_theme}.csv'.format(resolution=self.resolution,table_theme=self.table_theme))

class SourceTags(TagsTask):
    def version(self):
        return 1

    def tags(self):
        return [
            OBSTag(id='insee',
                   name='INSEE',
                   type='source',
                   description='http://www.insee.fr/fr/bases-de-donnees/default.asp?page=recensement/resultats/2012/donnees-detaillees-recensement-2012.htm')
        ]

class FrenchColumns(ColumnsTask):

    table_theme = Parameter()

    def requires(self):
        requirements = {
            'sections': SectionTags(),
            'subsections': SubsectionTags(),
            'unittags': UnitTags(),
            'sourcetag': SourceTags()
        }
        if self.table_theme != 'population':
            requirements['population_vars'] = FrenchColumns(table_theme='population')
        return requirements

    def version(self):
        return 7

    def columns(self):
        cols = OrderedDict()
        input_ = self.input()

        subsectiontags = input_['subsections']
        unittags = input_['unittags']
        france = input_['sections']['fr']
        pop_columns = input_.get('population_vars', {})

        filepath = "frenchmetadata/French Variables - {}.tsv".format(self.table_theme.title())
        session = current_session()
        with open(os.path.join(os.path.dirname(__file__),filepath)) as tsvfile:
            tsvreader = csv.reader(tsvfile, delimiter="\t")
            # Skip first row (header)
            next(tsvreader, None)
            for line in tsvreader:
                # Ignoring "Universe" and "Description" columns for now...
                var_code,short_name,long_name,var_unit,denominators,subsections = line[0], line[1], line[2], line[3], line[4], line[5]
                denominators = denominators.split(',')
                # slugified_lib = underscore_slugify('{}'.format(short_name))
                targets_dict = {}
                for x in denominators:
                    x = x.strip()
                    targets_dict[cols.get(x, pop_columns[x].get(session) if x in pop_columns else None)] = 'denominator'
                targets_dict.pop(None, None)
                cols[var_code] = OBSColumn(
                    id=var_code,
                    type='Numeric',
                    name=short_name,
                    description =long_name,
                    # Ranking of importance, sometimes used to favor certain measures in auto-selection
                    # Weight of 0 will hide this column from the user.  We generally use between 0 and 10
                    weight=5,
                    aggregate='sum',
                    # Tags are our way of noting aspects of this measure like its unit, the country
                    # it's relevant to, and which section(s) of the catalog it should appear in
                    # Need to fix Subsection and UnitTags! Problem with unittag "families"
                    tags=[france, unittags[var_unit]],
                    targets= targets_dict
                )
                subsections = subsections.split(',')
                for s in subsections:
                    s = s.strip()
                    subsection_tag = subsectiontags[s]
                    cols[var_code].tags.append(subsection_tag)

        insee_source = input_['sourcetag']['insee']

        for _,col in cols.iteritems():
            col.tags.append(insee_source)
        return cols

class DownloadOutputAreas(DownloadUnzipTask):
    # Note that this set of IRIS contours is from 2013, may need to find 2014 contours to match the data

    URL = 'https://www.data.gouv.fr/s/resources/contour-des-iris-insee-tout-en-un/20150428-161348/iris-2013-01-01.zip'

    def download(self):
        shell('wget -O {output}.zip {url}'.format(
        output=self.output().path,
        url=self.URL))

class ImportOutputAreas(Shp2TempTableTask):

    def requires(self):
        return DownloadOutputAreas()

    def input_shp(self):
        # may need to point to directory iris-2013-01-01?
        return os.path.join(self.input().path, 'iris-2013-01-01.shp')

class OutputAreaColumns(ColumnsTask):

    def version(self):
        return 2

    def requires(self):
        return {
            'subsections': SubsectionTags(),
            'sections': SectionTags(),
        }

    def columns(self):
        input_ = self.input()
        geom = OBSColumn(
            type='Geometry',
            name='IRIS and Commune areas',
            description='IRIS regions are defined by INSEE census for purposes of all municipalities '
                        'of over 10000 inhabitants and most towns from 5000 to 10000. For areas in which '
                        'IRIS is not defined, the commune area is given instead. ',
            weight=5,
            tags=[input_['subsections']['boundary'], input_['sections']['fr']]
        )
        geomref = OBSColumn(
            type='Text',
            name='DCOMIRIS',
            description='Full Code IRIS. Result of the concatenation of DEPCOM and IRIS attributes. ',
            weight=0,
            targets={geom: GEOM_REF}
        )

        return OrderedDict([
            ('the_geom', geom),
            ('dcomiris', geomref)
        ])

class OutputAreas(TableTask):

    def requires(self):
        return {
            'geom_columns': OutputAreaColumns(),
            'data': ImportOutputAreas(),
        }

    def version(self):
        return 2

    def timespan(self):
        return 2013

    def columns(self):
        input_ = self.input()
        cols = OrderedDict()
        cols.update(input_['geom_columns'])
        return cols

    def populate(self):
        session = current_session()
        session.execute('INSERT INTO {output} '
                        'SELECT ST_MakeValid(wkb_geometry), DCOMIRIS '
                        'FROM {input}'.format(
                            output=self.output().table,
                            input=self.input()['data'].table,
                        ))

class FranceCensus(TableTask):

    table_theme = Parameter()

    def version(self):
        return 8

    def timespan(self):
        return '2012'

    def requires(self):
        requirements = {
            'iris_data': RawFRData(table_theme=self.table_theme, resolution='iris'),
            'overseas_data': RawFRData(table_theme=self.table_theme, resolution='iris_overseas'),
            'meta': FrenchColumns(table_theme=self.table_theme),
            'geometa': OutputAreaColumns(),
        }
        return requirements

    def columns(self):
        cols = OrderedDict()
        cols['IRIS'] = self.input()['geometa']['dcomiris']
        cols.update(self.input()['meta'])
        return cols

    def populate(self):
        session = current_session()

        column_targets = self.columns()
        colnames = ', '.join(column_targets.keys())
        colnames_typed = ','.join(['{}::{}'.format(colname, ct.get(session).type)
                              for colname, ct in column_targets.iteritems()])
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['iris_data'].table
                        ))
        session.execute('INSERT INTO {output} ({ids}) '
                        'SELECT {ids_typed} '
                        'FROM {input} '.format(
                            ids=colnames,
                            ids_typed=colnames_typed,
                            output=self.output().table,
                            input=self.input()['overseas_data'].table
                        ))

class AllGeomsThemesTables(WrapperTask):
    def requires(self):
        topics = ['population', 'housing', 'education', 'household', 'employment']
        for table_theme in topics:
            yield FranceCensus(table_theme=table_theme)
<|endoftext|>


<commit_before>import asyncio
from random import random, uniform
import logging
import logging.config
from pathlib import Path
from collections import namedtuple
from functools import wraps, partial

import yaml
from bs4 import UnicodeDammit
from lxml import html
import aiohttp
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from async_timeout import timeout

from proxypool.config import PHANTOMJS_PATH, HEADERS, DELAY, VERBOSE

PROJECT_ROOT = Path(__file__).parent

def _log_async(func):
    """Send func to be executed by ThreadPoolExecutor of event loop."""

    @wraps(func)
    def wrapper(*args, **kwargs):
        loop = asyncio.get_event_loop()
        return loop.run_in_executor(None, partial(func, *args, **kwargs))

    return wrapper

class _LoggerAsync:
    """Logger's async proxy.

    Logging were executed in a thread pool executor to avoid blocking the event loop.
    """

    def __init__(self, *, is_server=False):
        logging.config.dictConfig(
            yaml.load(open(str(PROJECT_ROOT / 'logging.yaml'), 'r')))  # load config from YAML file

        if is_server:
            self._logger = logging.getLogger('server_logger')
        elif VERBOSE:
            self._logger = logging.getLogger('console_logger')  # output to both stdout and file
        else:
            self._logger = logging.getLogger('file_logger')

    def __getattr__(self, name):
        if hasattr(self._logger, name):
            return getattr(self._logger, name)
        else:
            msg = 'logger object has no attribute {!r}'
            raise AttributeError(msg.format(name))

    @_log_async
    def debug(self, msg, *args, **kwargs):
        self._logger.debug(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def info(self, msg, *args, **kwargs):
        self._logger.info(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def warning(self, msg, *args, **kwargs):
        self._logger.warning(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def error(self, msg, *args, **kwargs):
        self._logger.error(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def exception(self, msg, *args, exc_info=True, **kwargs):
        self._logger.exception(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def critical(self, msg, *args, **kwargs):
        self._logger.critical(msg, *args, exc_info=False, stack_info=False, **kwargs)

logger = _LoggerAsync()

Result = namedtuple('Result', 'content rule')

def decode_html(html_string):
    """Use bs4 to decode html file.

    Source: http://lxml.de/elementsoup.html#Using only the encoding detection
    """

    converted = UnicodeDammit(html_string)
    if not converted.unicode_markup:
        raise UnicodeDecodeError(
            "Failed to detect encoding, tried [%s]",
            ', '.join(converted.tried_encodings))
    return converted.unicode_markup

async def _fetch(url, session):
    await asyncio.sleep(uniform(DELAY - 0.5, DELAY + 1))
    logger.debug('crawling proxy web page {0}'.format(url.content))
    try:
        async with session.get(url.content, headers=HEADERS, timeout=10) as response:
            if response.status != 200:
                raise aiohttp.errors.ClientConnectionError(
                    'get {} return "{} {}"'.format(url.content, response.status, response.reason))
            page = await response.text()
            parsed = html.fromstring(decode_html(page))
            return parsed
    except asyncio.TimeoutError:
        pass

async def page_download(url_gen, pages, flag):
    """Download web page with aiohttp.

    Args:
        url_gen: url generator for next web page.
        pages: asyncio.Queue object, save downloaded web pages.
        flag: asyncio.Event object, stop flag.
    """

    async with aiohttp.ClientSession() as session:
        for url in url_gen:
            if flag.is_set():
                break

            parsed = None
            try:
                parsed = await _fetch(url, session)
                await pages.put(Result(parsed, url.rule))
            except Exception as e:
                logger.error(e)
            finally:
                try:
                    url_gen.send(parsed)
                except StopIteration:
                    break

async def page_download_phantomjs(url_gen, pages, element, flag):
    """Download web page with PhantomJS.

    Args:
        url_gen: url generator for next web page.
        pages: asyncio.Queue object, save downloaded web pages.
        element: element for PhantomJS to check if page was loaded.
        flag: asyncio.Event object, stop flag.
    """

    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = HEADERS
    browser = webdriver.PhantomJS(executable_path=PHANTOMJS_PATH, desired_capabilities=dcap)
    for url in url_gen:
        if flag.is_set():
            break

        await asyncio.sleep(uniform(DELAY - 0.5, DELAY + 1))
        logger.debug('phantomjs was crawling proxy web page {0}'.format(url.content))
        try:
            with timeout(10):
                browser.get(url.content)
                while browser.page_source.find(element) == -1:
                    await asyncio.sleep(random())

            parsed = html.fromstring(decode_html(browser.page_source))
            await pages.put(Result(parsed, url.rule))
            url_gen.send(parsed)
        except StopIteration:
            break
        except asyncio.TimeoutError:
            continue  # TODO: use a proxy
        except Exception as e:
            logger.error(e)
<commit_msg>fix parsed was None but still put in pages<commit_after>import asyncio
from random import random, uniform
import logging
import logging.config
from pathlib import Path
from collections import namedtuple
from functools import wraps, partial

import yaml
from bs4 import UnicodeDammit
from lxml import html
import aiohttp
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from async_timeout import timeout

from proxypool.config import PHANTOMJS_PATH, HEADERS, DELAY, VERBOSE

PROJECT_ROOT = Path(__file__).parent

def _log_async(func):
    """Send func to be executed by ThreadPoolExecutor of event loop."""

    @wraps(func)
    def wrapper(*args, **kwargs):
        loop = asyncio.get_event_loop()
        return loop.run_in_executor(None, partial(func, *args, **kwargs))

    return wrapper

class _LoggerAsync:
    """Logger's async proxy.

    Logging were executed in a thread pool executor to avoid blocking the event loop.
    """

    def __init__(self, *, is_server=False):
        logging.config.dictConfig(
            yaml.load(open(str(PROJECT_ROOT / 'logging.yaml'), 'r')))  # load config from YAML file

        if is_server:
            self._logger = logging.getLogger('server_logger')
        elif VERBOSE:
            self._logger = logging.getLogger('console_logger')  # output to both stdout and file
        else:
            self._logger = logging.getLogger('file_logger')

    def __getattr__(self, name):
        if hasattr(self._logger, name):
            return getattr(self._logger, name)
        else:
            msg = 'logger object has no attribute {!r}'
            raise AttributeError(msg.format(name))

    @_log_async
    def debug(self, msg, *args, **kwargs):
        self._logger.debug(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def info(self, msg, *args, **kwargs):
        self._logger.info(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def warning(self, msg, *args, **kwargs):
        self._logger.warning(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def error(self, msg, *args, **kwargs):
        self._logger.error(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def exception(self, msg, *args, exc_info=True, **kwargs):
        self._logger.exception(msg, *args, exc_info=False, stack_info=False, **kwargs)

    @_log_async
    def critical(self, msg, *args, **kwargs):
        self._logger.critical(msg, *args, exc_info=False, stack_info=False, **kwargs)

logger = _LoggerAsync()

Result = namedtuple('Result', 'content rule')

def decode_html(html_string):
    """Use bs4 to decode html file.

    Source: http://lxml.de/elementsoup.html#Using only the encoding detection
    """

    converted = UnicodeDammit(html_string)
    if not converted.unicode_markup:
        raise UnicodeDecodeError(
            "Failed to detect encoding, tried [%s]",
            ', '.join(converted.tried_encodings))
    return converted.unicode_markup

async def _fetch(url, session):
    await asyncio.sleep(uniform(DELAY - 0.5, DELAY + 1))
    logger.debug('crawling proxy web page {0}'.format(url.content))

    async with session.get(url.content, headers=HEADERS, timeout=10) as response:
        if response.status != 200:
            raise aiohttp.errors.ClientConnectionError(
                'get {} return "{} {}"'.format(url.content, response.status, response.reason))
        page = await response.text()
        parsed = html.fromstring(decode_html(page))
        return parsed

async def page_download(url_gen, pages, flag):
    """Download web page with aiohttp.

    Args:
        url_gen: url generator for next web page.
        pages: asyncio.Queue object, save downloaded web pages.
        flag: asyncio.Event object, stop flag.
    """

    async with aiohttp.ClientSession() as session:
        for url in url_gen:
            if flag.is_set():
                break

            parsed = None
            try:
                parsed = await _fetch(url, session)
            except asyncio.TimeoutError:
                pass
            except Exception as e:
                logger.error(e)
            else:
                await pages.put(Result(parsed, url.rule))
            finally:
                try:
                    url_gen.send(parsed)
                except StopIteration:
                    break

async def page_download_phantomjs(url_gen, pages, element, flag):
    """Download web page with PhantomJS.

    Args:
        url_gen: url generator for next web page.
        pages: asyncio.Queue object, save downloaded web pages.
        element: element for PhantomJS to check if page was loaded.
        flag: asyncio.Event object, stop flag.
    """

    dcap = dict(DesiredCapabilities.PHANTOMJS)
    dcap["phantomjs.page.settings.userAgent"] = HEADERS
    browser = webdriver.PhantomJS(executable_path=PHANTOMJS_PATH, desired_capabilities=dcap)
    for url in url_gen:
        if flag.is_set():
            break

        await asyncio.sleep(uniform(DELAY - 0.5, DELAY + 1))
        logger.debug('phantomjs was crawling proxy web page {0}'.format(url.content))
        try:
            with timeout(10):
                browser.get(url.content)
                while browser.page_source.find(element) == -1:
                    await asyncio.sleep(random())

            parsed = html.fromstring(decode_html(browser.page_source))
            await pages.put(Result(parsed, url.rule))
            url_gen.send(parsed)
        except StopIteration:
            break
        except asyncio.TimeoutError:
            continue  # TODO: use a proxy
        except Exception as e:
            logger.error(e)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import api

g = api.Gotham2()

unitSquare = ([-0.5, 0,  0.5,
                0.5, 0,  0.5,
                0.5, 0, -0.5,
               -0.5, 0, -0.5],
              [   0, 1,  3,
                  1, 2,  3])

# back wall
g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0, 0, -1)
g.rotate(90, 1, 0, 0)
g.scale(2.0,2.0,2.0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

## floor
#g.material('matte', 'Kd', (0.8, 0.8, 0.8))
#g.pushMatrix()
#g.translate(0,-1,0)
#g.scale(2.0, 2.0, 2.0)
#g.mesh(unitSquare[0], unitSquare[1])
#g.popMatrix()

# ceiling
g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0,1,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

# left wall
g.material('matte', 'Kd', (0.8, 0.2, 0.2))
g.pushMatrix()
g.translate(-1,0,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(-90, 0, 0, 1)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

# right wall
g.material('matte', 'Kd', (0.2, 0.8, 0.2))
g.pushMatrix()
g.translate(1,0,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(90, 0, 0, 1)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.material('phongreflection', 'Kr', (1.0, 1.0, 1.0), 'eta', 2.485, 'exponent', 100000)
g.material('mirror', 'Kr', (1.0, 1.0, 1.0))
#g.sphere(-0.4, -0.66, -0.15, 0.33)

# floor
g.material('phongreflection', 'Kr', (0.8, 0.8, 0.8), 'eta', 2.485, 'exponent', 25)
#g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0,-1,0)
g.scale(2.0, 2.0, 2.0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.material('perfectglass', 'Kr', (1.0, 1.0, 1.0), 'Kt', (1.0, 1.0, 1.0), 'eta', 1.5)
#g.material('thinglass', 'Kr', (1.0, 1.0, 1.0), 'Kt', (1.0, 1.0, 1.0))
g.sphere(0.4,-0.66,0.25,0.33)

# lights
g.material('light', 'power', (10,10,10))

g.pushMatrix()
g.translate(-0.5, 0.95, 0)
g.scale(0.5, 0.5, 0.5)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.pushMatrix()
g.translate(0.5, 0.95, 0)
g.scale(0.5, 0.5, 0.5)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.attribute("path::sampler", "kelemen")
#g.attribute("path::sampler", "whitted")
#g.attribute("path::maxlength", "20")
g.attribute("path::maxlength", "10")
#g.attribute("path::russianroulette::function", "always")
g.attribute("renderer::algorithm", "metropolis")
g.attribute("importance::function", "constant")

(w,h) = (512,512)
g.pushMatrix()
g.lookAt( (0,0,3.0), (0,0,-1), (0,1,0) )
g.camera(float(w) / h, 60.0, 0.01)
g.popMatrix()

g.render((w,h), 4)

<commit_msg>Put the other sphere back in the box<commit_after>#!/usr/bin/env python
import api

g = api.Gotham2()

unitSquare = ([-0.5, 0,  0.5,
                0.5, 0,  0.5,
                0.5, 0, -0.5,
               -0.5, 0, -0.5],
              [   0, 1,  3,
                  1, 2,  3])

# back wall
g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0, 0, -1)
g.rotate(90, 1, 0, 0)
g.scale(2.0,2.0,2.0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

## floor
#g.material('matte', 'Kd', (0.8, 0.8, 0.8))
#g.pushMatrix()
#g.translate(0,-1,0)
#g.scale(2.0, 2.0, 2.0)
#g.mesh(unitSquare[0], unitSquare[1])
#g.popMatrix()

# ceiling
g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0,1,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

# left wall
g.material('matte', 'Kd', (0.8, 0.2, 0.2))
g.pushMatrix()
g.translate(-1,0,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(-90, 0, 0, 1)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

# right wall
g.material('matte', 'Kd', (0.2, 0.8, 0.2))
g.pushMatrix()
g.translate(1,0,0)
g.scale(2.0, 2.0, 2.0)
g.rotate(90, 0, 0, 1)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.material('phongreflection', 'Kr', (1.0, 1.0, 1.0), 'eta', 2.485, 'exponent', 100000)
g.material('mirror', 'Kr', (1.0, 1.0, 1.0))
g.sphere(-0.4, -0.66, -0.15, 0.33)

# floor
g.material('phongreflection', 'Kr', (0.8, 0.8, 0.8), 'eta', 2.485, 'exponent', 25)
#g.material('matte', 'Kd', (0.8, 0.8, 0.8))
g.pushMatrix()
g.translate(0,-1,0)
g.scale(2.0, 2.0, 2.0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.material('perfectglass', 'Kr', (1.0, 1.0, 1.0), 'Kt', (1.0, 1.0, 1.0), 'eta', 1.5)
#g.material('thinglass', 'Kr', (1.0, 1.0, 1.0), 'Kt', (1.0, 1.0, 1.0))
g.sphere(0.4,-0.66,0.25,0.33)

# lights
g.material('light', 'power', (10,10,10))

g.pushMatrix()
g.translate(-0.5, 0.95, 0)
g.scale(0.5, 0.5, 0.5)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.pushMatrix()
g.translate(0.5, 0.95, 0)
g.scale(0.5, 0.5, 0.5)
g.rotate(180.0, 1.0, 0, 0)
g.mesh(unitSquare[0], unitSquare[1])
g.popMatrix()

g.attribute("path::sampler", "kelemen")
#g.attribute("path::sampler", "whitted")
#g.attribute("path::maxlength", "20")
g.attribute("path::maxlength", "10")
#g.attribute("path::russianroulette::function", "always")
g.attribute("renderer::algorithm", "metropolis")
g.attribute("importance::function", "constant")

(w,h) = (512,512)
g.pushMatrix()
g.lookAt( (0,0,3.0), (0,0,-1), (0,1,0) )
g.camera(float(w) / h, 60.0, 0.01)
g.popMatrix()

g.render((w,h), 4)

<|endoftext|>


<commit_before>import os
import re

from django.http import HttpResponse
from django.conf import settings
from django.core.urlresolvers import reverse

def _sub_meta(html, meta_name, replacement):
    return _subvar(html, r'<meta name="' + meta_name + r'" content=".*">',
                   replacement)

def _subvar(html, template, replacement):
    regexp = "(%s)(.*)(%s)" % tuple(template.split(".*"))
    return re.sub(regexp, r'\1' + replacement + r'\3', html)

def _sub_base_href(html, base_url):
    return _subvar(html, r'<base href=".*">', base_url)

def _sub_publish_url(html, publish_url):
    return _sub_meta(html, 'publish-url', publish_url)

def _sub_remix_url(html, remix_url):
    return _sub_meta(html, 'remix-url', remix_url)

def _sub_deployment_type(html):
    if settings.DEV:
        deployment_type = 'development'
    else:
        deployment_type = 'production'
    return _sub_meta(html, 'deployment-type', deployment_type)

def _frontend_html(base_url, publish_url, remix_url):
    mydir = os.path.dirname(__file__)
    html = open(os.path.join(mydir, 'friendlycode', 'index.html')).read()
    html = _sub_remix_url(html, remix_url)
    html = _sub_base_href(_sub_publish_url(html, publish_url), base_url)
    return _sub_deployment_type(html)

def _editor(request, remix_url):
    rel_base_url = '%sfriendlycode/' % settings.MEDIA_URL
    return HttpResponse(_frontend_html(
        base_url=request.build_absolute_uri(rel_base_url),
        publish_url=request.build_absolute_uri("/")[:-1],
        remix_url=remix_url
        ))

def default_editor(request):
    return _editor(request,
                   '%sfriendlycode/default-content.html' % settings.MEDIA_URL)

def editor(request, **kwargs):
    viewname = kwargs['remix']
    del kwargs['remix']
    to_remix = reverse(viewname, kwargs=kwargs)
    return _editor(request, to_remix)
<commit_msg>Cache the `open` call to `friendlycode/index.html`.<commit_after>import os
import re

from django.http import HttpResponse
from django.conf import settings
from django.core.urlresolvers import reverse

class memoize(object):
    """
    Cache the results of a function only if settings.DEV is False
    """
    contents = None
    def __init__(self, func):
        self.func = func

    def __call__(self, *args):
        def inner(*args):
            if not self.contents or settings.DEV:
                self.contents = self.func(*args)
            return self.contents
        return inner(*args)

@memoize
def _friendly_code_html():
    mydir = os.path.dirname(__file__)
    index_file = os.path.join(mydir, 'friendlycode', 'index.html')
    with open(index_file) as f:
        return f.read()

def _sub_meta(html, meta_name, replacement):
    return _subvar(html, r'<meta name="' + meta_name + r'" content=".*">',
                   replacement)

def _subvar(html, template, replacement):
    regexp = "(%s)(.*)(%s)" % tuple(template.split(".*"))
    return re.sub(regexp, r'\1' + replacement + r'\3', html)

def _sub_base_href(html, base_url):
    return _subvar(html, r'<base href=".*">', base_url)

def _sub_publish_url(html, publish_url):
    return _sub_meta(html, 'publish-url', publish_url)

def _sub_remix_url(html, remix_url):
    return _sub_meta(html, 'remix-url', remix_url)

def _sub_deployment_type(html):
    if settings.DEV:
        deployment_type = 'development'
    else:
        deployment_type = 'production'
    return _sub_meta(html, 'deployment-type', deployment_type)

def _frontend_html(base_url, publish_url, remix_url):
    html = _friendly_code_html()
    html = _sub_remix_url(html, remix_url)
    html = _sub_base_href(_sub_publish_url(html, publish_url), base_url)
    return _sub_deployment_type(html)

def _editor(request, remix_url):
    rel_base_url = '%sfriendlycode/' % settings.MEDIA_URL
    return HttpResponse(_frontend_html(
        base_url=request.build_absolute_uri(rel_base_url),
        publish_url=request.build_absolute_uri("/")[:-1],
        remix_url=remix_url
        ))

def default_editor(request):
    return _editor(request,
                   '%sfriendlycode/default-content.html' % settings.MEDIA_URL)

def editor(request, **kwargs):
    viewname = kwargs['remix']
    del kwargs['remix']
    to_remix = reverse(viewname, kwargs=kwargs)
    return _editor(request, to_remix)
<|endoftext|>