<commit_before># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Handler for the /lab URLs."""

from urlparse import urlunparse

import datetime
import bson

import handlers.base
import handlers.common
import handlers.response as hresponse
import models
import models.lab as mlab
import models.token as mtoken
import utils.validator as validator
import utils.db

# pylint: disable=too-many-public-methods
class LabHandler(handlers.base.BaseHandler):
    """Handle all traffic through the /lab URLs."""

    def __init__(self, application, request, **kwargs):
        super(LabHandler, self).__init__(application, request, **kwargs)

    @property
    def collection(self):
        return self.db[models.LAB_COLLECTION]

    @staticmethod
    def _valid_keys(method):
        return handlers.common.LAB_VALID_KEYS.get(method, None)

    @staticmethod
    def _token_validation_func():
        return handlers.common.valid_token_th

    def _post(self, *args, **kwargs):
        response = hresponse.HandlerResponse(201)

        json_obj = kwargs["json_obj"]

        valid_contact, reason = validator.is_valid_lab_contact_data(json_obj)
        if valid_contact:
            lab_id = kwargs.get("id", None)
            status_code, reason, result, headers = self._create_or_update(
                json_obj, lab_id)

            response.status_code = status_code
            response.result = result
            if reason:
                if kwargs["reason"]:
                    reason += "\n" + kwargs["reason"]
                response.reason = reason
            if headers:
                response.headers = headers
        else:
            response.status_code = 400
            if reason:
                if kwargs["reason"]:
                    reason += "\n" + kwargs["reason"]
                response.reason = reason

        return response

    def _create_or_update(self, json_obj, lab_id):
        """Create or update a new lab object.

        If the request comes in with a specified lab name, it will be treated
        as an update request.

        :param json_obj: The JSON data as sent in the request.
        :type json_obj: dict
        :param lab_id: The ID part of the request.
        :type lab_id: str
        :return A tuple with: status code, reason, result and headers.
        """
        status_code = None
        reason = None
        result = None
        headers = None
        old_lab = None
        name = json_obj.get(models.NAME_KEY)

        if lab_id:
            try:
                old_lab = utils.db.find_one(
                    self.collection,
                    [bson.objectid.ObjectId(lab_id)]
                )
            except bson.errors.InvalidId, ex:
                self.log.exception(ex)
                self.log.error("Wrong ID value '%s' passed as doc ID", lab_id)
                reason = "Wrong ID value provided"
        else:
            old_lab = utils.db.find_one(
                self.collection, [name], field=models.NAME_KEY
            )

        if all([lab_id, old_lab]):
            self.log.info(
                "Updating lab with ID '%s' from IP address %s",
                old_lab.get(models.ID_KEY), self.request.remote_ip
            )
            status_code, reason, result, headers = self._update_lab(
                json_obj, old_lab
            )
        elif all([lab_id, not old_lab]):
            status_code = 404
            reason = "Lab with name '%s' not found" % lab_id
        elif all([not old_lab, not lab_id]):
            self.log.info("Creating new lab object")
            status_code, reason, result, headers = self._create_new_lab(
                json_obj)
        else:
            status_code = 400
            if not reason:
                reason = (
                    "Lab with name '%s' already exists: did you mean to "
                    "update it?" % name
                )

        return status_code, reason, result, headers

    def _update_lab(self, json_obj, old_lab):
        """Update an existing lab object.

        :param json_obj: The JSON object with the lab data.
        :type json_obj: dict
        :param old_lab: The JSON object of the lab from the db.
        :type old_lab: dict
        :return A tuple with: status code, reason, result and headers.
        """
        status_code = None
        reason = None
        result = None
        headers = None

        # Locally used to store the contact information from the new lab object.
        new_contact = None

        old_lab = mlab.LabDocument.from_json(old_lab)
        new_lab = mlab.LabDocument.from_json(json_obj)

        if new_lab.name:
            if old_lab.name != new_lab.name:
                # The is no setter for the name field in the Lab model.
                old_lab._name = new_lab.name

        if new_lab.contact:
            if old_lab.contact != new_lab.contact:
                old_lab.contact = new_lab.contact
                new_contact = new_lab.contact

        if new_lab.token:
            self._update_lab_token(old_lab, new_lab, new_contact)

        if new_lab.address:
            if old_lab.address != new_lab.address:
                old_lab.address = new_lab.address

        if old_lab.private != new_lab.private:
            old_lab.private = new_lab.private

        old_lab.updated_on = datetime.datetime.now(tz=bson.tz_util.utc)

        status_code, _ = utils.db.save(self.db, old_lab)
        if status_code != 201:
            reason = "Error updating lab '%s'" % old_lab.name
        else:
            reason = "Lab '%s' updated" % old_lab.name
            status_code = 200

        return status_code, reason, result, headers

    def _update_lab_token(self, old_lab, new_lab, new_contact):
        """Update references of lab token.

        :param old_lab: The lab object as found in the database.
        :type old_lab: LabDocument
        :param new_lab: The new lab object as passed by the user.
        :type new_lab: LabDocument
        :param new_contact: The contact information as found in the new lab
        document.
        :type new_contact: dict
        """
        # If the user specifies a new token, it will be doing so using the
        # actual token value, not its ID. We need to make sure we still
        # have the old token as defined in the old lab document, find the
        # new token and update accordingly using the token ID.
        old_token = utils.db.find_one(
            self.db[models.TOKEN_COLLECTION],
            [old_lab.token], field=models.TOKEN_KEY
        )
        new_token = utils.db.find_one(
            self.db[models.TOKEN_COLLECTION],
            [new_lab.token], field=models.TOKEN_KEY
        )

        if old_token:
            old_token = mtoken.Token.from_json(old_token)
        if new_token:
            new_token = mtoken.Token.from_json(new_token)

        if all([old_token, new_token]):
            # Both old and new tokens?
            # Expire the old one and save it.
            if old_token.token != new_token.token:
                old_lab.token = new_token.id

            old_token.expired = True
            ret_code, _ = utils.db.save(self.db, old_token)
            if ret_code != 201:
                self.log.warn("Error expiring old token '%s'", old_token.id)

        if all([old_token, not new_token, new_contact]):
            # Just the old token?
            # Make sure its contact information are correct and save it.
            old_token.username = (
                new_contact[models.NAME_KEY] +
                " " +
                new_contact[models.SURNAME_KEY]
            )
            old_token.email = new_contact[models.EMAIL_KEY]
            ret_code, _ = utils.db.save(self.db, old_token)
            if ret_code != 201:
                self.log.warn("Error updating old token '%s'", old_token.id)

    def _create_new_lab(self, json_obj):
        """Create a new lab in the database.

        :param json_obj: The JSON object with the lab data.
        :type json_obj: dict
        :return A tuple with: status code, reason, result and headers.
        """
        token_id = None
        ret_val = None
        reason = "New lab created"
        result = None
        headers = None

        lab_doc = mlab.LabDocument.from_json(json_obj)
        lab_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)

        if lab_doc.token:
            token_json = utils.db.find_one(
                self.db[models.TOKEN_COLLECTION],
                [lab_doc.token],
                field=models.TOKEN_KEY
            )
            if token_json:
                token = mtoken.Token.from_json(token_json)
                token_id = token.id
                ret_val = 200
            else:
                ret_val = 500
        else:
            token = mtoken.Token()
            token.email = lab_doc.contact[models.EMAIL_KEY]
            token.username = (
                lab_doc.contact[models.NAME_KEY] +
                " " +
                lab_doc.contact[models.SURNAME_KEY]
            )
            token.is_post_token = True
            ret_val, token_id = utils.db.save(self.db, token, manipulate=True)

        if ret_val == 201 or ret_val == 200:
            lab_doc.token = token_id
            ret_val, lab_id = utils.db.save(self.db, lab_doc, manipulate=True)
            if ret_val == 201:
                result = {
                    models.ID_KEY: lab_id,
                    models.NAME_KEY: lab_doc.name,
                    models.TOKEN_KEY: token.token
                }
                location = urlunparse(
                    (
                        'http',
                        self.request.headers.get('Host'),
                        self.request.uri + '/' + lab_doc.name,
                        '', '', ''
                    )
                )
                headers = {'Location': location}
            else:
                reason = "Error saving new lab '%s'" % lab_doc.name
        else:
            reason = (
                "Error saving or finding the token for lab '%s'" % lab_doc.name
            )

        return (ret_val, reason, result, headers)

    def execute_delete(self, *args, **kwargs):
        # TODO: need to expire or delete token as well.
        response = None

        if self.validate_req_token("DELETE"):
            if kwargs and kwargs.get('id', None):
                lab_id = kwargs['id']
                try:
                    lab_id = bson.objectid.ObjectId(lab_id)
                    if utils.db.find_one(self.collection, [lab_id]):
                        response = self._delete(lab_id)
                        if response.status_code == 200:
                            response.reason = "Resource '%s' deleted" % lab_id
                    else:
                        response = hresponse.HandlerResponse(404)
                        response.reason = "Resource '%s' not found" % lab_id
                except bson.errors.InvalidId, ex:
                    self.log.exception(ex)
                    self.log.error(
                        "Wrong ID value '%s' passed as doc ID", lab_id
                    )
                    response = hresponse.HandlerResponse(400)
                    response.reason = "Wrong ID value provided"
            else:
                spec = handlers.common.get_query_spec(
                    self.get_query_arguments, self._valid_keys("DELETE")
                )
                if spec:
                    response = self._delete(spec)
                    if response.status_code == 200:
                        response.reason = (
                            "Resources identified with '%s' deleted" % spec
                        )
                else:
                    response = hresponse.HandlerResponse(400)
                    response.result = None
                    response.reason = (
                        "No valid data provided to execute a DELETE"
                    )
        else:
            response = hresponse.HandlerResponse(403)
            response.reason = handlers.common.NOT_VALID_TOKEN

        return response

    def _delete(self, spec_or_id):
        response = hresponse.HandlerResponse(200)
        response.result = None

        response.status_code = utils.db.delete(self.collection, spec_or_id)
        response.reason = self._get_status_message(response.status_code)

        return response
<commit_msg>Make lab token delete and a real lab one.<commit_after># This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Handler for the /lab URLs."""

from urlparse import urlunparse

import datetime
import bson

import handlers.base
import handlers.common
import handlers.response as hresponse
import models
import models.lab as mlab
import models.token as mtoken
import utils.validator as validator
import utils.db

# pylint: disable=too-many-public-methods
class LabHandler(handlers.base.BaseHandler):
    """Handle all traffic through the /lab URLs."""

    def __init__(self, application, request, **kwargs):
        super(LabHandler, self).__init__(application, request, **kwargs)

    @property
    def collection(self):
        return self.db[models.LAB_COLLECTION]

    @staticmethod
    def _valid_keys(method):
        return handlers.common.LAB_VALID_KEYS.get(method, None)

    @staticmethod
    def _token_validation_func():
        return handlers.common.valid_token_th

    def _post(self, *args, **kwargs):
        response = hresponse.HandlerResponse(201)

        json_obj = kwargs["json_obj"]

        valid_contact, reason = validator.is_valid_lab_contact_data(json_obj)
        if valid_contact:
            lab_id = kwargs.get("id", None)
            status_code, reason, result, headers = self._create_or_update(
                json_obj, lab_id)

            response.status_code = status_code
            response.result = result
            if reason:
                if kwargs["reason"]:
                    reason += "\n" + kwargs["reason"]
                response.reason = reason
            if headers:
                response.headers = headers
        else:
            response.status_code = 400
            if reason:
                if kwargs["reason"]:
                    reason += "\n" + kwargs["reason"]
                response.reason = reason

        return response

    def _create_or_update(self, json_obj, lab_id):
        """Create or update a new lab object.

        If the request comes in with a specified lab name, it will be treated
        as an update request.

        :param json_obj: The JSON data as sent in the request.
        :type json_obj: dict
        :param lab_id: The ID part of the request.
        :type lab_id: str
        :return A tuple with: status code, reason, result and headers.
        """
        status_code = None
        reason = None
        result = None
        headers = None
        old_lab = None
        name = json_obj.get(models.NAME_KEY)

        if lab_id:
            try:
                old_lab = utils.db.find_one(
                    self.collection,
                    [bson.objectid.ObjectId(lab_id)]
                )
            except bson.errors.InvalidId, ex:
                self.log.exception(ex)
                self.log.error("Wrong ID value '%s' passed as doc ID", lab_id)
                reason = "Wrong ID value provided"
        else:
            old_lab = utils.db.find_one(
                self.collection, [name], field=models.NAME_KEY
            )

        if all([lab_id, old_lab]):
            self.log.info(
                "Updating lab with ID '%s' from IP address %s",
                old_lab.get(models.ID_KEY), self.request.remote_ip
            )
            status_code, reason, result, headers = self._update_lab(
                json_obj, old_lab
            )
        elif all([lab_id, not old_lab]):
            status_code = 404
            reason = "Lab with name '%s' not found" % lab_id
        elif all([not old_lab, not lab_id]):
            self.log.info("Creating new lab object")
            status_code, reason, result, headers = self._create_new_lab(
                json_obj)
        else:
            status_code = 400
            if not reason:
                reason = (
                    "Lab with name '%s' already exists: did you mean to "
                    "update it?" % name
                )

        return status_code, reason, result, headers

    def _update_lab(self, json_obj, old_lab):
        """Update an existing lab object.

        :param json_obj: The JSON object with the lab data.
        :type json_obj: dict
        :param old_lab: The JSON object of the lab from the db.
        :type old_lab: dict
        :return A tuple with: status code, reason, result and headers.
        """
        status_code = None
        reason = None
        result = None
        headers = None

        # Locally used to store the contact information from the new lab object.
        new_contact = None

        old_lab = mlab.LabDocument.from_json(old_lab)
        new_lab = mlab.LabDocument.from_json(json_obj)

        if new_lab.name:
            if old_lab.name != new_lab.name:
                # The is no setter for the name field in the Lab model.
                old_lab._name = new_lab.name

        if new_lab.contact:
            if old_lab.contact != new_lab.contact:
                old_lab.contact = new_lab.contact
                new_contact = new_lab.contact

        if new_lab.token:
            self._update_lab_token(old_lab, new_lab, new_contact)

        if new_lab.address:
            if old_lab.address != new_lab.address:
                old_lab.address = new_lab.address

        if old_lab.private != new_lab.private:
            old_lab.private = new_lab.private

        old_lab.updated_on = datetime.datetime.now(tz=bson.tz_util.utc)

        status_code, _ = utils.db.save(self.db, old_lab)
        if status_code != 201:
            reason = "Error updating lab '%s'" % old_lab.name
        else:
            reason = "Lab '%s' updated" % old_lab.name
            status_code = 200

        return status_code, reason, result, headers

    def _update_lab_token(self, old_lab, new_lab, new_contact):
        """Update references of lab token.

        :param old_lab: The lab object as found in the database.
        :type old_lab: LabDocument
        :param new_lab: The new lab object as passed by the user.
        :type new_lab: LabDocument
        :param new_contact: The contact information as found in the new lab
        document.
        :type new_contact: dict
        """
        # If the user specifies a new token, it will be doing so using the
        # actual token value, not its ID. We need to make sure we still
        # have the old token as defined in the old lab document, find the
        # new token and update accordingly using the token ID.
        old_token = utils.db.find_one(
            self.db[models.TOKEN_COLLECTION],
            [old_lab.token], field=models.TOKEN_KEY
        )
        new_token = utils.db.find_one(
            self.db[models.TOKEN_COLLECTION],
            [new_lab.token], field=models.TOKEN_KEY
        )

        if old_token:
            old_token = mtoken.Token.from_json(old_token)
        if new_token:
            new_token = mtoken.Token.from_json(new_token)

        if all([old_token, new_token]):
            # Both old and new tokens?
            # Expire the old one and save it.
            if old_token.token != new_token.token:
                old_lab.token = new_token.id

            old_token.expired = True
            ret_code, _ = utils.db.save(self.db, old_token)
            if ret_code != 201:
                self.log.warn("Error expiring old token '%s'", old_token.id)

        if all([old_token, not new_token, new_contact]):
            # Just the old token?
            # Make sure its contact information are correct and save it.
            old_token.username = (
                new_contact[models.NAME_KEY] +
                " " +
                new_contact[models.SURNAME_KEY]
            )
            old_token.email = new_contact[models.EMAIL_KEY]
            ret_code, _ = utils.db.save(self.db, old_token)
            if ret_code != 201:
                self.log.warn("Error updating old token '%s'", old_token.id)

    def _create_new_lab(self, json_obj):
        """Create a new lab in the database.

        :param json_obj: The JSON object with the lab data.
        :type json_obj: dict
        :return A tuple with: status code, reason, result and headers.
        """
        token_id = None
        ret_val = None
        reason = "New lab created"
        result = None
        headers = None

        lab_doc = mlab.LabDocument.from_json(json_obj)
        lab_doc.created_on = datetime.datetime.now(tz=bson.tz_util.utc)

        if lab_doc.token:
            token_json = utils.db.find_one(
                self.db[models.TOKEN_COLLECTION],
                [lab_doc.token],
                field=models.TOKEN_KEY
            )
            if token_json:
                token = mtoken.Token.from_json(token_json)
                token_id = token.id
                ret_val = 200
            else:
                ret_val = 500
        else:
            token = mtoken.Token()
            token.email = lab_doc.contact[models.EMAIL_KEY]
            token.username = (
                lab_doc.contact[models.NAME_KEY] +
                " " +
                lab_doc.contact[models.SURNAME_KEY]
            )
            token.is_post_token = True
            token.is_delete_token = True
            token.is_lab_token = True
            ret_val, token_id = utils.db.save(self.db, token, manipulate=True)

        if ret_val == 201 or ret_val == 200:
            lab_doc.token = token_id
            ret_val, lab_id = utils.db.save(self.db, lab_doc, manipulate=True)
            if ret_val == 201:
                result = {
                    models.ID_KEY: lab_id,
                    models.NAME_KEY: lab_doc.name,
                    models.TOKEN_KEY: token.token
                }
                location = urlunparse(
                    (
                        'http',
                        self.request.headers.get('Host'),
                        self.request.uri + '/' + lab_doc.name,
                        '', '', ''
                    )
                )
                headers = {'Location': location}
            else:
                reason = "Error saving new lab '%s'" % lab_doc.name
        else:
            reason = (
                "Error saving or finding the token for lab '%s'" % lab_doc.name
            )

        return (ret_val, reason, result, headers)

    def execute_delete(self, *args, **kwargs):
        # TODO: need to expire or delete token as well.
        response = None

        if self.validate_req_token("DELETE"):
            if kwargs and kwargs.get('id', None):
                lab_id = kwargs['id']
                try:
                    lab_id = bson.objectid.ObjectId(lab_id)
                    if utils.db.find_one(self.collection, [lab_id]):
                        response = self._delete(lab_id)
                        if response.status_code == 200:
                            response.reason = "Resource '%s' deleted" % lab_id
                    else:
                        response = hresponse.HandlerResponse(404)
                        response.reason = "Resource '%s' not found" % lab_id
                except bson.errors.InvalidId, ex:
                    self.log.exception(ex)
                    self.log.error(
                        "Wrong ID value '%s' passed as doc ID", lab_id
                    )
                    response = hresponse.HandlerResponse(400)
                    response.reason = "Wrong ID value provided"
            else:
                spec = handlers.common.get_query_spec(
                    self.get_query_arguments, self._valid_keys("DELETE")
                )
                if spec:
                    response = self._delete(spec)
                    if response.status_code == 200:
                        response.reason = (
                            "Resources identified with '%s' deleted" % spec
                        )
                else:
                    response = hresponse.HandlerResponse(400)
                    response.result = None
                    response.reason = (
                        "No valid data provided to execute a DELETE"
                    )
        else:
            response = hresponse.HandlerResponse(403)
            response.reason = handlers.common.NOT_VALID_TOKEN

        return response

    def _delete(self, spec_or_id):
        response = hresponse.HandlerResponse(200)
        response.result = None

        response.status_code = utils.db.delete(self.collection, spec_or_id)
        response.reason = self._get_status_message(response.status_code)

        return response
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2012 Michael Babich
# See LICENSE.txt or http://opensource.org/licenses/MIT

'''Acts as an interface between what Flask serves and what goes on in
the rest of the application.
'''
from avenue import app, api
from flask import render_template, make_response
from copy import copy
import yaml
from os import path

heading = 'Zombie Raptor'
navbar = []
navbar.append({'title'   : 'Zombie Raptor Blog',
               'content' : 'Read new updates from the Zombie Raptor team!',
               'link'    : '/'})

navbar.append({'title'   : 'Main Forum',
               'content' : 'Visit the main forum!',
               'link'    : '/f/main'})

browser_upgrade = '

'

tags = { 'post'  : ['post', '#aabbcc', '/'],
         'test'  : ['test', '#ffbb99', '/'],
         'micro' : ['micro', '#aabbcc', '/'],
         'link'  : ['link', '#aabbcc', '/'],
         'news'  : ['news', '#bbeebb', '/']}

def button(text):
    return '
%s
' % text

buttons = '
%s%s%s%s&nbsp
' % (button(u'↳'), button('+'), button('-'), button('#'))

def render_forum(thread_title='', main_title='', html_title='', posts=[], threaded=False, content=''):
    return render_template('forum.html',
                           style='night',
                           sidebar=navbar,
                           thread_title=thread_title,
                           main_title=main_title,
                           html_title=html_title,
                           posts=posts,
                           threaded=threaded,
                           content=content)

@app.route('/')
def index():
    '''The main page.
    '''
    words = '
Expect stuff from Zombie Raptor in the near future.
'

    page_title = heading

    css = 'night'

    post = {'level' : 0, 'content' : words, 'author' : 'admin', 'date' : 'now'}

    return render_forum(main_title=heading,
                        thread_title="Zombie Raptor Launches... at some point.",
                        html_title=page_title,
                        posts=[post],
                        content='blog')

@app.route('/f/')
def f():
    return ''

@app.route('/f/main/')
def main_forum():
    main_forum_ = open(path.join(path.dirname(__file__), 'data', 'main_forum.yml'))
    test = yaml.load(main_forum_)
    main_forum_.close()

    page_title = '%s :: %s' % ('Main Forum', heading)

    for post in test['posts']:
        for i in range(len(post['tags'])):
            post['tags'][i] = tags[post['tags'][i]]

    return render_forum(main_title=heading + ' -- Main Forum',
                        thread_title='Active Threads',
                        html_title=page_title,
                        posts=test['posts'],
                        content='index')

@app.route('/f/main/post/')
def post():
    return ''

@app.route('/f/main/post/1')
def sample_post():
    sample = open(path.join(path.dirname(__file__), 'data', 'sample.yml'))
    thread = yaml.load(sample)
    sample.close()

    page_title = '%s :: %s :: %s' % (thread['title'], 'Main Forum', heading)

    return render_forum(main_title=heading + ' -- Main Forum',
                        thread_title=thread['title'],
                        html_title=page_title,
                        posts=thread['posts'],
                        threaded=True,
                        content='post')

@app.route('/night.css')
def night():
    conf = open(path.join(path.dirname(__file__), 'data', 'style.yml'))
    style = yaml.load(conf)
    conf.close()

    response = make_response(render_template('main.css',
                                             text=style['text'],
                                             background=style['background'],
                                             post=style['post']))
    response.mimetype = 'text/css'
    return response
<commit_msg>Remove unused code; fix variable name.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2012 Michael Babich
# See LICENSE.txt or http://opensource.org/licenses/MIT

'''Acts as an interface between what Flask serves and what goes on in
the rest of the application.
'''
from avenue import app, api
from flask import render_template, make_response
from copy import copy
import yaml
from os import path

site_name = 'Zombie Raptor'
navbar = []
navbar.append({'title'   : 'Zombie Raptor Blog',
               'content' : 'Read new updates from the Zombie Raptor team!',
               'link'    : '/'})

navbar.append({'title'   : 'Main Forum',
               'content' : 'Visit the main forum!',
               'link'    : '/f/main'})

browser_upgrade = '

'

tags = { 'post'  : ['post', '#aabbcc', '/'],
         'test'  : ['test', '#ffbb99', '/'],
         'micro' : ['micro', '#aabbcc', '/'],
         'link'  : ['link', '#aabbcc', '/'],
         'news'  : ['news', '#bbeebb', '/']}

def render_forum(thread_title='', main_title='', html_title='', posts=[], threaded=False, content=''):
    return render_template('forum.html',
                           style='night',
                           sidebar=navbar,
                           thread_title=thread_title,
                           main_title=main_title,
                           html_title=html_title,
                           posts=posts,
                           threaded=threaded,
                           content=content)

@app.route('/')
def index():
    '''The main page.
    '''
    words = '
Expect stuff from Zombie Raptor in the near future.
'

    page_title = site_name

    css = 'night'

    post = {'level' : 0, 'content' : words, 'author' : 'admin', 'date' : 'now'}

    return render_forum(main_title=site_name,
                        thread_title="Zombie Raptor Launches... at some point.",
                        html_title=page_title,
                        posts=[post],
                        content='blog')

@app.route('/f/')
def f():
    return ''

@app.route('/f/main/')
def main_forum():
    main_forum_ = open(path.join(path.dirname(__file__), 'data', 'main_forum.yml'))
    test = yaml.load(main_forum_)
    main_forum_.close()

    page_title = '%s :: %s' % ('Main Forum', site_name)

    for post in test['posts']:
        for i in range(len(post['tags'])):
            post['tags'][i] = tags[post['tags'][i]]

    return render_forum(main_title=site_name + ' -- Main Forum',
                        thread_title='Active Threads',
                        html_title=page_title,
                        posts=test['posts'],
                        content='index')

@app.route('/f/main/post/')
def post():
    return ''

@app.route('/f/main/post/1')
def sample_post():
    sample = open(path.join(path.dirname(__file__), 'data', 'sample.yml'))
    thread = yaml.load(sample)
    sample.close()

    page_title = '%s :: %s :: %s' % (thread['title'], 'Main Forum', site_name)

    return render_forum(main_title=site_name + ' -- Main Forum',
                        thread_title=thread['title'],
                        html_title=page_title,
                        posts=thread['posts'],
                        threaded=True,
                        content='post')

@app.route('/night.css')
def night():
    conf = open(path.join(path.dirname(__file__), 'data', 'style.yml'))
    style = yaml.load(conf)
    conf.close()

    response = make_response(render_template('main.css',
                                             text=style['text'],
                                             background=style['background'],
                                             post=style['post']))
    response.mimetype = 'text/css'
    return response
<|endoftext|>


<commit_before>#!/usr/bin/env python
import optparse
import sys
import numpy
import math

def main(argv):
	#Shameless copy
	optparser = optparse.OptionParser()
	optparser.add_option("-d", "--data", dest="train", default="data/hansards", help="Data filename prefix (default=data)")
	optparser.add_option("-e", "--english", dest="english", default="e", help="Suffix of English filename (default=e)")
	optparser.add_option("-f", "--french", dest="french", default="f", help="Suffix of French filename (default=f)")
	optparser.add_option("-n", "--num_sentences", dest="num_sents", default=sys.maxint, type="int", help="Number of sentences to use for training and alignment")
	(opts, _) = optparser.parse_args()
	f_data = "%s.%s" % (opts.train, opts.french)
	e_data = "%s.%s" % (opts.train, opts.english)

	sys.stderr.write("Start training...\n")
	bitext = [[sentence.strip().split() for sentence in pair] for pair in zip(open(f_data), open(e_data))[:opts.num_sents]]
	revtext = [[sentence.strip().split() for sentence in pair] for pair in zip(open(e_data), open(f_data))[:opts.num_sents]]

	e2f = trainFastAlign(bitext, max_iter=5)
	f2e = trainFastAlign(revtext, max_iter=5)

	global A
	A = []

	grow_diagonal(e2f, f2e)
	# finalAnd(e2f)
	# finalAnd(f2e, reverse = True)
	output()

def trainModel1(bitext, f_index, e_index, f_count, e_count, max_iter = 20, convergeThreshold=1e-2):
	"""

	Train model 1
	"""
	#Construct array as e_count x f_count
	#Probability P(e|f)
	tef = numpy.zeros((e_count,f_count))
	#record new pef to compare with old one
	ntef = numpy.zeros((e_count,f_count))
	#Initialize parameters with uniform distribution
	ntef.fill(float(1)/float(e_count))

	it = 0
	while it < max_iter and sum(sum(numpy.absolute(tef-ntef))) > convergeThreshold:
		it += 1
		tef = ntef
		# Initialize Count for C(e|f)
		cef = numpy.zeros((e_count,f_count))
		totalf = numpy.zeros(f_count)
		for e,f in bitext:
			en = [None] + e
			#Compute normalization
			for e_word in en:
				totale = float(0)
				for f_word in f:
					totale += tef[e_index[e_word]][f_index[f_word]]
				for f_word in f:
					cef[e_index[e_word]][f_index[f_word]] += float(tef[e_index[e_word]][f_index[f_word]]) / float(totale)
					totalf[f_index[f_word]] += float(tef[e_index[e_word]][f_index[f_word]]) / float(totale)
		#Estimate probabilities
		ntef = (cef.T / totalf[:,None]).T

	return ntef

def trainFastAlign(bitext, max_iter = 50, convergeThreshold = 1e-2):
	"""

	Main function of training fast align

	bitext: list
	contains language pair as (target, source)

	max_iter: int
	set max iteration that the model will train

	convergeThreshold: float
	set threshold of convergence, currently un-used.
	"""

	f_set = set()
	e_set = set()
	#Get word counts
	for e,f in bitext:
		e_set.update(e)
		f_set.update(f)

	#Add None word for index
	e_set.add(None)

	f_count = len(f_set)
	e_count = len(e_set)

	#Mapping location in dictionary
	e_index = {}
	f_index = {}
	for (n,f) in enumerate(f_set):
		f_index[f] = n

	for (n,e) in enumerate(e_set):
		e_index[e] = n

	global p0, lamb

	#Initialize
	p0 = 0.08
	lamb = 4.0

	# tef using model1 go get better initilization
	tef = trainModel1(bitext, f_index, e_index, f_count, e_count, max_iter = 5)

	for it in range(max_iter):
		cef = numpy.zeros((e_count,f_count))
		likelihood = 0.0

		for (n, (e, f)) in enumerate(bitext):
			en = [None] + e
			le = len(en)
			lf = len(f)

			prob_e = numpy.zeros(le)
			for (j, f_word) in enumerate(f):
				sum_prob = 0.0
				#Uniform
				prob_a_i = 1.0 / le
				for (i, e_word) in enumerate(en):
					if i == 0:
						prob_a_i = p0
					else:
						az = computeZ(j+1, lf, le - 1, lamb) / (1.0-p0)
						prob_a_i = computeProb(j+1, i, lf, le - 1, lamb) / az

					prob_e[i] = tef[e_index[e_word]][f_index[f_word]] * prob_a_i
					sum_prob += prob_e[i]

				# print sum_prob
				for (i, e_word) in enumerate(en):
					c = prob_e[i] / sum_prob
					cef[e_index[e_word]][f_index[f_word]] += c
				likelihood += math.log(sum_prob);

		base2_likelihood = likelihood / math.log(2);
		# print base2_likelihood
		#Estimate probabilities
		tef += cef
		totale = numpy.sum(tef, axis = 1)

		for row in range(tef.shape[0]):
			if totale[row] == 0.0:
				totale[row] = 1.0
			tef[row,:] = tef[row,:] / totale[row]
		# print tef

	#Align using tef
	return align(bitext, tef, f_index, e_index)

def computeZ(i, lf, le, lamb):
	"""

	Compute normalization constant Z

	i: int
	location of current word

	lf, le: int
	length of target sentence(lf) and source sentence(le)

	lamb: float
	Hyper parameters, currently set to be 4.0

	"""

	split = float(i) * float(le) / float(lf)
	floor = int(split)
	ceil = floor + 1

	ratio = math.exp(-lamb / le)
	num_top = le - floor

	high = 0.0
	low = 0.0
	if num_top != 0:
		high = computeProb(i, ceil, lf, le, lamb) * (1.0 - math.pow(ratio, num_top)) / (1.0 - ratio)
	if floor != 0:
		low = computeProb(i, floor, lf, le, lamb) * (1.0 - math.pow(ratio, floor)) / (1.0 - ratio)

	return high + low

def computeProb(i, j, lf, le, lamb):
	"""

	Compute numerator

	i,j : int
	alignment of word i to j

	lf, le: int
	length of target sentence(lf) and source sentence(le)

	lamb: float
	Hyper parameters, currently set to be 4.0

	"""
	h = -1.0 * abs( float(j) / float(le) - float(i) / float(lf))
	return math.exp(h * lamb)

def align(bitext, tef, f_index, e_index):
	"""

	Compute alignment
	"""
	res = []
	for (n,(e, f)) in enumerate(bitext):
		align = []
		en = [None] + e
		le = len(en)
		lf = len(f)
		#Create array that records the mapping, and initialize all to be False, means unaligned
		res.append(numpy.zeros((le-1,lf), dtype=bool))

		for (j, f_word) in enumerate(f):
			max_i = 0
			max_prob = 0
			prob_a_i = 0.0
			for (i, e_word) in enumerate(en):
				if i == 0:
					prob_a_i = p0
				else:
					az = computeZ(j+1, lf, le-1, lamb) / (1.0-p0)
					prob_a_i = computeProb(j+1, i , lf, le-1, lamb) / az

				tProb = tef[e_index[e_word]][f_index[f_word]] * prob_a_i
				if tProb > max_prob:
					max_i = i
					max_prob = tProb
			#if null word do not write
			if max_i > 0:
				# sys.stdout.write("%i-%i " % (max_i-1,j))
				res[n][max_i-1][j] = True
		# sys.stdout.write("\n")
	return res

def grow_diagonal(e2f, f2e):
	"""

	Use grow diagonal to improve, input show be a n length numpy array with shpae (length of e, length of f)

	"""
	neighboring = ((-1,0),(0,-1),(1,0),(0,1),(-1,-1),(-1,1),(1,-1),(1,1))

	for n in range(len(e2f)):
		#Find intersection array and union array
		A.append(e2f[n] & f2e[n].T)
		# Below buggy
		U = e2f[n] | f2e[n].T

		oldA = numpy.zeros(A[n].shape, dtype=bool)

		row = A[n].shape[0]
		col = A[n].shape[1]

		while numpy.array_equal(oldA,A[n]) == False:
			#Add new aligns
			oldA = A[n]
			for e in range(row):
				for f in range(col):
					if A[n][e][f] == True:
						for (i,j) in neighboring:
							e_new = e + i
							f_new = f + j
							if 0 <= e_new < row and 0 <= f_new < col:
								if (numpy.sum(A[n][e_new,:]) == 0 or numpy.sum(A[n][:,f_new]) == 0) and U[e_new][f_new] == True:
									A[n][e_new][f_new] = True

def finalAnd(e2f_in, reverse = False):
	for n in range(len(e2f_in)):
		e2f = e2f_in[n]
		if reverse == True:
			e2f = e2f_in[n].T

		row = A[n].shape[0]
		col = A[n].shape[1]

		for e_new in range(row):
			for f_new in range(col):
				if (numpy.sum(A[n][e_new,:]) == 0 or numpy.sum(A[n][:,f_new]) == 0) and e2f[e_new][f_new] == True:
					A[n][e_new][f_new] = True

def output():
	"""

	Output result in source-target pairs
	"""
	for n in range(len(A)):
		row = A[n].shape[0]
		col = A[n].shape[1]

		for i in range(row):
			for j in range(col):
				if A[n][i][j] == True:
					sys.stdout.write("%i-%i " % (i, j))
					break
		sys.stdout.write("\n")

#Main entry
if __name__ == '__main__':
	main(sys.argv)
<commit_msg>Comment grow-diag, just use intersection<commit_after>#!/usr/bin/env python
import optparse
import sys
import numpy
import math

def main(argv):
	#Shameless copy
	optparser = optparse.OptionParser()
	optparser.add_option("-d", "--data", dest="train", default="data/hansards", help="Data filename prefix (default=data)")
	optparser.add_option("-e", "--english", dest="english", default="e", help="Suffix of English filename (default=e)")
	optparser.add_option("-f", "--french", dest="french", default="f", help="Suffix of French filename (default=f)")
	optparser.add_option("-n", "--num_sentences", dest="num_sents", default=sys.maxint, type="int", help="Number of sentences to use for training and alignment")
	(opts, _) = optparser.parse_args()
	f_data = "%s.%s" % (opts.train, opts.french)
	e_data = "%s.%s" % (opts.train, opts.english)

	sys.stderr.write("Start training...\n")
	bitext = [[sentence.strip().split() for sentence in pair] for pair in zip(open(f_data), open(e_data))[:opts.num_sents]]
	revtext = [[sentence.strip().split() for sentence in pair] for pair in zip(open(e_data), open(f_data))[:opts.num_sents]]

	e2f = trainFastAlign(bitext, max_iter=5)
	f2e = trainFastAlign(revtext, max_iter=5)

	global A
	A = []

	grow_diagonal(e2f, f2e)
	# finalAnd(e2f)
	# finalAnd(f2e, reverse = True)
	output()

def trainModel1(bitext, f_index, e_index, f_count, e_count, max_iter = 20, convergeThreshold=1e-2):
	"""

	Train model 1
	"""
	#Construct array as e_count x f_count
	#Probability P(e|f)
	tef = numpy.zeros((e_count,f_count))
	#record new pef to compare with old one
	ntef = numpy.zeros((e_count,f_count))
	#Initialize parameters with uniform distribution
	ntef.fill(float(1)/float(e_count))

	it = 0
	while it < max_iter and sum(sum(numpy.absolute(tef-ntef))) > convergeThreshold:
		it += 1
		tef = ntef
		# Initialize Count for C(e|f)
		cef = numpy.zeros((e_count,f_count))
		totalf = numpy.zeros(f_count)
		for e,f in bitext:
			en = [None] + e
			#Compute normalization
			for e_word in en:
				totale = float(0)
				for f_word in f:
					totale += tef[e_index[e_word]][f_index[f_word]]
				for f_word in f:
					cef[e_index[e_word]][f_index[f_word]] += float(tef[e_index[e_word]][f_index[f_word]]) / float(totale)
					totalf[f_index[f_word]] += float(tef[e_index[e_word]][f_index[f_word]]) / float(totale)
		#Estimate probabilities
		ntef = (cef.T / totalf[:,None]).T

	return ntef

def trainFastAlign(bitext, max_iter = 50, convergeThreshold = 1e-2):
	"""

	Main function of training fast align

	bitext: list
	contains language pair as (target, source)

	max_iter: int
	set max iteration that the model will train

	convergeThreshold: float
	set threshold of convergence, currently un-used.
	"""

	f_set = set()
	e_set = set()
	#Get word counts
	for e,f in bitext:
		e_set.update(e)
		f_set.update(f)

	#Add None word for index
	e_set.add(None)

	f_count = len(f_set)
	e_count = len(e_set)

	#Mapping location in dictionary
	e_index = {}
	f_index = {}
	for (n,f) in enumerate(f_set):
		f_index[f] = n

	for (n,e) in enumerate(e_set):
		e_index[e] = n

	global p0, lamb

	#Initialize
	p0 = 0.08
	lamb = 4.0

	# tef using model1 go get better initilization
	tef = trainModel1(bitext, f_index, e_index, f_count, e_count, max_iter = 5)

	for it in range(max_iter):
		cef = numpy.zeros((e_count,f_count))
		likelihood = 0.0

		for (n, (e, f)) in enumerate(bitext):
			en = [None] + e
			le = len(en)
			lf = len(f)

			prob_e = numpy.zeros(le)
			for (j, f_word) in enumerate(f):
				sum_prob = 0.0
				#Uniform
				prob_a_i = 1.0 / le
				for (i, e_word) in enumerate(en):
					if i == 0:
						prob_a_i = p0
					else:
						az = computeZ(j+1, lf, le - 1, lamb) / (1.0-p0)
						prob_a_i = computeProb(j+1, i, lf, le - 1, lamb) / az

					prob_e[i] = tef[e_index[e_word]][f_index[f_word]] * prob_a_i
					sum_prob += prob_e[i]

				# print sum_prob
				for (i, e_word) in enumerate(en):
					c = prob_e[i] / sum_prob
					cef[e_index[e_word]][f_index[f_word]] += c
				likelihood += math.log(sum_prob);

		base2_likelihood = likelihood / math.log(2);
		# print base2_likelihood
		#Estimate probabilities
		tef += cef
		totale = numpy.sum(tef, axis = 1)

		for row in range(tef.shape[0]):
			if totale[row] == 0.0:
				totale[row] = 1.0
			tef[row,:] = tef[row,:] / totale[row]
		# print tef

	#Align using tef
	return align(bitext, tef, f_index, e_index)

def computeZ(i, lf, le, lamb):
	"""

	Compute normalization constant Z

	i: int
	location of current word

	lf, le: int
	length of target sentence(lf) and source sentence(le)

	lamb: float
	Hyper parameters, currently set to be 4.0

	"""

	split = float(i) * float(le) / float(lf)
	floor = int(split)
	ceil = floor + 1

	ratio = math.exp(-lamb / le)
	num_top = le - floor

	high = 0.0
	low = 0.0
	if num_top != 0:
		high = computeProb(i, ceil, lf, le, lamb) * (1.0 - math.pow(ratio, num_top)) / (1.0 - ratio)
	if floor != 0:
		low = computeProb(i, floor, lf, le, lamb) * (1.0 - math.pow(ratio, floor)) / (1.0 - ratio)

	return high + low

def computeProb(i, j, lf, le, lamb):
	"""

	Compute numerator

	i,j : int
	alignment of word i to j

	lf, le: int
	length of target sentence(lf) and source sentence(le)

	lamb: float
	Hyper parameters, currently set to be 4.0

	"""
	h = -1.0 * abs( float(j) / float(le) - float(i) / float(lf))
	return math.exp(h * lamb)

def align(bitext, tef, f_index, e_index):
	"""

	Compute alignment
	"""
	res = []
	for (n,(e, f)) in enumerate(bitext):
		align = []
		en = [None] + e
		le = len(en)
		lf = len(f)
		#Create array that records the mapping, and initialize all to be False, means unaligned
		res.append(numpy.zeros((le-1,lf), dtype=bool))

		for (j, f_word) in enumerate(f):
			max_i = 0
			max_prob = 0
			prob_a_i = 0.0
			for (i, e_word) in enumerate(en):
				if i == 0:
					prob_a_i = p0
				else:
					az = computeZ(j+1, lf, le-1, lamb) / (1.0-p0)
					prob_a_i = computeProb(j+1, i , lf, le-1, lamb) / az

				tProb = tef[e_index[e_word]][f_index[f_word]] * prob_a_i
				if tProb > max_prob:
					max_i = i
					max_prob = tProb
			#if null word do not write
			if max_i > 0:
				# sys.stdout.write("%i-%i " % (max_i-1,j))
				res[n][max_i-1][j] = True
		# sys.stdout.write("\n")
	return res

def grow_diagonal(e2f, f2e):
	"""

	Use grow diagonal to improve, input show be a n length numpy array with shpae (length of e, length of f)

	"""
	neighboring = ((-1,0),(0,-1),(1,0),(0,1),(-1,-1),(-1,1),(1,-1),(1,1))

	for n in range(len(e2f)):
		#Find intersection array and union array
		A.append(e2f[n] & f2e[n].T)
		# Not as good as direct intersection
		# U = e2f[n] | f2e[n].T

		# oldA = numpy.zeros(A[n].shape, dtype=bool)

		# row = A[n].shape[0]
		# col = A[n].shape[1]

		# while numpy.array_equal(oldA,A[n]) == False:
		# 	#Add new aligns
		# 	oldA = A[n]
		# 	for e in range(row):
		# 		for f in range(col):
		# 			if A[n][e][f] == True:
		# 				for (i,j) in neighboring:
		# 					e_new = e + i
		# 					f_new = f + j
		# 					if 0 <= e_new < row and 0 <= f_new < col:
		# 						if (numpy.sum(A[n][e_new,:]) == 0 or numpy.sum(A[n][:,f_new]) == 0) and U[e_new][f_new] == True:
		# 							A[n][e_new][f_new] = True

def finalAnd(e2f_in, reverse = False):
	for n in range(len(e2f_in)):
		e2f = e2f_in[n]
		if reverse == True:
			e2f = e2f_in[n].T

		row = A[n].shape[0]
		col = A[n].shape[1]

		for e_new in range(row):
			for f_new in range(col):
				if (numpy.sum(A[n][e_new,:]) == 0 or numpy.sum(A[n][:,f_new]) == 0) and e2f[e_new][f_new] == True:
					A[n][e_new][f_new] = True

def output():
	"""

	Output result in source-target pairs
	"""
	for n in range(len(A)):
		row = A[n].shape[0]
		col = A[n].shape[1]

		for i in range(row):
			for j in range(col):
				if A[n][i][j] == True:
					sys.stdout.write("%i-%i " % (i, j))
					break
		sys.stdout.write("\n")

#Main entry
if __name__ == '__main__':
	main(sys.argv)
<|endoftext|>


<commit_before># from __future__ import print_function
# from pprint import pprint
from app import db
from flask import Blueprint, render_template, flash, redirect, url_for
from sqlalchemy.orm import aliased
from sqlalchemy.exc import IntegrityError
from .forms import EventForm, EventTypeForm, PriceForm, CommodityForm
from .models import Event, EventType, Price, Commodity, CommodityType

hermes = Blueprint('hermes', __name__)

def _bookmark(table):
	global __TABLE__
	__TABLE__ = table

def _get_table_info(table):
	def get_event():
		form_fields = ['symbol', 'type_id', 'value', 'date']
		table_headers = ['Symbol', 'Name', 'Unit', 'Value', 'Date']
		query = db.session.query(Event, EventType, Commodity).join(EventType) \
			.join(Commodity, EventType.unit).order_by(Event.date)
		data_fields = [(0, 'symbol'), (1, 'name'), (2, 'symbol'), (0, 'value'),
			(0, 'date')]
		return form_fields, table_headers, query, data_fields

	def get_event_type():
		form_fields = ['name', 'commodity_id']
		table_headers = ['Type Name', 'Unit']
		query = db.session.query(EventType, Commodity).join(Commodity) \
			.order_by(EventType.name)
		data_fields = [(0, 'name'), (1, 'symbol')]
		return form_fields, table_headers, query, data_fields

	def get_price():
		form_fields = ['commodity_id', 'currency_id', 'date', 'close']
		table_headers = ['Stock', 'Currency', 'Date', 'Price']
		Currency = aliased(Commodity)
		query = db.session.query(Price, Commodity, Currency) \
			.join(Price.commodity).join(Currency, Price.currency) \
			.order_by(Price.date)
		data_fields = [(1, 'symbol'), (2, 'symbol'), (0, 'date'), (0, 'close')]
		return form_fields, table_headers, query, data_fields

	def get_commodity():
		form_fields = ['symbol', 'name', 'type_id', 'data_source_id',
			'exchange_id']
		table_headers = ['Symbol', 'Name', 'Type']
		query = db.session.query(Commodity, CommodityType).join(CommodityType) \
			.order_by(Commodity.name)
		data_fields = [(0, 'symbol'), (0, 'name'), (1, 'name')]
		return form_fields, table_headers, query, data_fields

	switch = {'event': get_event(),
		'event_type': get_event_type(),
		'price': get_price(),
		'commodity': get_commodity()}

	return switch.get(table)

def _get_plural(word):
	if word[-1] == 'y':
		return word[:-1] + 'ies'
	else:
		return word + 's'

@hermes.route('/<table>/', methods=['GET', 'POST'])
def get(table):
	plural_table = _get_plural(table).replace('_', ' ')
	table_as_class = table.title().replace('_', '')
	table_title = table.title().replace('_', ' ')
	plural_table_title = plural_table.title()
	form_fields, table_headers, query, data_fields = _get_table_info(table)

	id = table
	post_location = 'hermes.add'
	post_table = table
	title = '%s' % plural_table_title
	table_caption = '%s List' % table_title
	form_caption = '%s Entry Form' % table_title
	heading = 'Add %s to the database' % plural_table
	text = 'On this page you can add %s to the database and see them ' \
		'instantly updated in the lists below.' % plural_table
	results = query.all()

	try:
		form = eval('%sForm.new()' % table_as_class)
	except AttributeError:
		form = eval('%sForm()' % table_as_class)

	kwargs = {'id': id, 'title': title, 'heading': heading, 'text': text,
		'rows': results, 'form': form, 'form_caption': form_caption,
		'table_caption': table_caption, 'table_headers': table_headers,
		'data_fields': data_fields, 'form_fields': form_fields,
		'post_location': post_location, 'post_table': post_table}

	return render_template('entry.html', **kwargs)

@hermes.route('/add/<table>/', methods=['GET', 'POST'])
def add(table):
	table_as_class = table.title().replace('_', '')
	try:
		form = eval('%sForm.new()' % table_as_class)
	except AttributeError:
		form = eval('%sForm()' % table_as_class)

	if form.validate_on_submit():
		entry = eval('%s()' % table_as_class)
		form.populate_obj(entry)
		db.session.add(entry)
		_bookmark(table)
		db.session.commit()
		flash('Success! A new %s was posted.' % table.replace('_', ' '),
			'alert alert-success')

	else:
		[flash('%s: %s' % (k.title(), v[0]), 'alert alert-error')
			for k, v in form.errors.iteritems()]

	return redirect(url_for('hermes.get', table=table))

@hermes.route('/worth/', methods=['GET', 'POST'])
def worth():
	pass

@hermes.route('/api/', methods=['GET', 'POST'])
def api():
	pass

@hermes.errorhandler(409)
@hermes.errorhandler(IntegrityError)
def duplicate_values(e):
	flash('Error: %s' % e.orig[0], 'alert alert-error')
	return redirect(url_for('hermes.get', table=__TABLE__))
<commit_msg>Remove api route and edit worth route<commit_after># from __future__ import print_function
# from pprint import pprint
from app import db
from flask import Blueprint, render_template, flash, redirect, url_for
from sqlalchemy.orm import aliased
from sqlalchemy.exc import IntegrityError
from .forms import EventForm, EventTypeForm, PriceForm, CommodityForm
from .models import Event, EventType, Price, Commodity, CommodityType

hermes = Blueprint('hermes', __name__)

def _bookmark(table):
	global __TABLE__
	__TABLE__ = table

def _get_table_info(table):
	def get_event():
		form_fields = ['symbol', 'type_id', 'value', 'date']
		table_headers = ['Symbol', 'Name', 'Unit', 'Value', 'Date']
		query = db.session.query(Event, EventType, Commodity).join(EventType) \
			.join(Commodity, EventType.unit).order_by(Event.date)
		data_fields = [(0, 'symbol'), (1, 'name'), (2, 'symbol'), (0, 'value'),
			(0, 'date')]
		return form_fields, table_headers, query, data_fields

	def get_event_type():
		form_fields = ['name', 'commodity_id']
		table_headers = ['Type Name', 'Unit']
		query = db.session.query(EventType, Commodity).join(Commodity) \
			.order_by(EventType.name)
		data_fields = [(0, 'name'), (1, 'symbol')]
		return form_fields, table_headers, query, data_fields

	def get_price():
		form_fields = ['commodity_id', 'currency_id', 'date', 'close']
		table_headers = ['Stock', 'Currency', 'Date', 'Price']
		Currency = aliased(Commodity)
		query = db.session.query(Price, Commodity, Currency) \
			.join(Price.commodity).join(Currency, Price.currency) \
			.order_by(Price.date)
		data_fields = [(1, 'symbol'), (2, 'symbol'), (0, 'date'), (0, 'close')]
		return form_fields, table_headers, query, data_fields

	def get_commodity():
		form_fields = ['symbol', 'name', 'type_id', 'data_source_id',
			'exchange_id']
		table_headers = ['Symbol', 'Name', 'Type']
		query = db.session.query(Commodity, CommodityType).join(CommodityType) \
			.order_by(Commodity.name)
		data_fields = [(0, 'symbol'), (0, 'name'), (1, 'name')]
		return form_fields, table_headers, query, data_fields

	switch = {'event': get_event(),
		'event_type': get_event_type(),
		'price': get_price(),
		'commodity': get_commodity()}

	return switch.get(table)

def _get_plural(word):
	if word[-1] == 'y':
		return word[:-1] + 'ies'
	else:
		return word + 's'

@hermes.route('/<table>/', methods=['GET', 'POST'])
def get(table):
	plural_table = _get_plural(table).replace('_', ' ')
	table_as_class = table.title().replace('_', '')
	table_title = table.title().replace('_', ' ')
	plural_table_title = plural_table.title()
	form_fields, table_headers, query, data_fields = _get_table_info(table)

	id = table
	post_location = 'hermes.add'
	post_table = table
	title = '%s' % plural_table_title
	table_caption = '%s List' % table_title
	form_caption = '%s Entry Form' % table_title
	heading = 'Add %s to the database' % plural_table
	text = 'On this page you can add %s to the database and see them ' \
		'instantly updated in the lists below.' % plural_table
	results = query.all()

	try:
		form = eval('%sForm.new()' % table_as_class)
	except AttributeError:
		form = eval('%sForm()' % table_as_class)

	kwargs = {'id': id, 'title': title, 'heading': heading, 'text': text,
		'rows': results, 'form': form, 'form_caption': form_caption,
		'table_caption': table_caption, 'table_headers': table_headers,
		'data_fields': data_fields, 'form_fields': form_fields,
		'post_location': post_location, 'post_table': post_table}

	return render_template('entry.html', **kwargs)

@hermes.route('/add/<table>/', methods=['GET', 'POST'])
def add(table):
	table_as_class = table.title().replace('_', '')
	try:
		form = eval('%sForm.new()' % table_as_class)
	except AttributeError:
		form = eval('%sForm()' % table_as_class)

	if form.validate_on_submit():
		entry = eval('%s()' % table_as_class)
		form.populate_obj(entry)
		db.session.add(entry)
		_bookmark(table)
		db.session.commit()
		flash('Success! A new %s was posted.' % table.replace('_', ' '),
			'alert alert-success')

	else:
		[flash('%s: %s' % (k.title(), v[0]), 'alert alert-error')
			for k, v in form.errors.iteritems()]

	return redirect(url_for('hermes.get', table=table))

@hermes.route('/worth/')
def worth():
	pass

@hermes.errorhandler(409)
@hermes.errorhandler(IntegrityError)
def duplicate_values(e):
	flash('Error: %s' % e.orig[0], 'alert alert-error')
	return redirect(url_for('hermes.get', table=__TABLE__))
<|endoftext|>


<commit_before>"""Makes the AndroidEnv interface flat.

Removes the structure from observation and action. Keeps only the pixel
observation. Exposes action as an int32 scalar. This makes it easier to plug in
to conventional agents. This wrapper expects a discretized action space.
This wrapper should be applied agent-side.
"""

from typing import Union, Dict, Any

from android_env.wrappers import base_wrapper
import dm_env
from dm_env import specs
import numpy as np

RGB_CHANNELS = (0, 1, 2)

def _extract_screen_pixels(obs: np.ndarray):
  """Get only screen pixels by removing previous action layer."""
  is_grayscale_image = obs.shape[-1] == 2
  if is_grayscale_image:
    return np.expand_dims(obs[..., 0], -1)
  return obs[..., RGB_CHANNELS]

def _get_no_action_observation_spec(obs_spec: specs.BoundedArray):
  """Create an observation spec without the action layer."""
  shape = np.array(obs_spec.shape)
  shape[2] -= 1
  minimum = obs_spec.minimum
  maximum = obs_spec.maximum
  is_scalar = lambda x: np.isscalar(x) or np.ndim(x) == 0
  if not is_scalar(minimum):
    minimum = _extract_screen_pixels(minimum)
  if not is_scalar(maximum):
    maximum = _extract_screen_pixels(maximum)
  return obs_spec.replace(shape=shape, minimum=minimum, maximum=maximum)

class FlatInterfaceWrapper(base_wrapper.BaseWrapper):
  """Simple interface for AndroidEnv."""

  def __init__(self,
               env: dm_env.Environment,
               flat_actions: bool = True,
               flat_observations: bool = True,
               keep_action_layer: bool = True):
    super().__init__(env)
    self._flat_actions = flat_actions
    self._flat_observations = flat_observations
    self._keep_action_layer = keep_action_layer
    self._action_name = list(self._env.action_spec())[0]
    self._assert_base_env()

  def _assert_base_env(self):
    base_action_spec = self._env.action_spec()
    assert len(base_action_spec) == 1, self._env.action_spec()
    assert isinstance(base_action_spec, dict)
    assert isinstance(base_action_spec[self._action_name], specs.BoundedArray)

  def _process_action(self, action: Union[int, np.ndarray, Dict[str, Any]]):
    if self._flat_actions:
      return {self._action_name: action}
    else:
      return action

  def _process_timestep(self, timestep: dm_env.TimeStep) -> dm_env.TimeStep:
    if self._flat_observations:
      step_type, reward, discount, observation = timestep
      # Keep only the pixels.
      pixels = observation['pixels']
      pixels = pixels if self._keep_action_layer else _extract_screen_pixels(
          pixels)
      return dm_env.TimeStep(
          step_type=step_type,
          reward=reward,
          discount=discount,
          observation=pixels)
    else:
      return timestep

  def reset(self) -> dm_env.TimeStep:
    timestep = self._env.reset()
    return self._process_timestep(timestep)

  def step(self, action: int) -> dm_env.TimeStep:
    timestep = self._env.step(self._process_action(action))
    return self._process_timestep(timestep)

  def observation_spec(self) -> specs.Array:
    if self._flat_observations:
      pixels_spec = self._env.observation_spec()['pixels']
      if not self._keep_action_layer:
        return _get_no_action_observation_spec(pixels_spec)
      return pixels_spec
    else:
      return self._env.observation_spec()

  def action_spec(self) -> specs.Array:
    if self._flat_actions:
      return self._env.action_spec()[self._action_name]
    else:
      return self._env.action_spec()
<commit_msg>Remove comment not relevant for open-source release.<commit_after>"""Makes the AndroidEnv interface flat.

Removes the structure from observation and action. Keeps only the pixel
observation. Exposes action as an int32 scalar. This makes it easier to plug in
to conventional agents. This wrapper expects a discretized action space.
"""

from typing import Union, Dict, Any

from android_env.wrappers import base_wrapper
import dm_env
from dm_env import specs
import numpy as np

RGB_CHANNELS = (0, 1, 2)

def _extract_screen_pixels(obs: np.ndarray):
  """Get only screen pixels by removing previous action layer."""
  is_grayscale_image = obs.shape[-1] == 2
  if is_grayscale_image:
    return np.expand_dims(obs[..., 0], -1)
  return obs[..., RGB_CHANNELS]

def _get_no_action_observation_spec(obs_spec: specs.BoundedArray):
  """Create an observation spec without the action layer."""
  shape = np.array(obs_spec.shape)
  shape[2] -= 1
  minimum = obs_spec.minimum
  maximum = obs_spec.maximum
  is_scalar = lambda x: np.isscalar(x) or np.ndim(x) == 0
  if not is_scalar(minimum):
    minimum = _extract_screen_pixels(minimum)
  if not is_scalar(maximum):
    maximum = _extract_screen_pixels(maximum)
  return obs_spec.replace(shape=shape, minimum=minimum, maximum=maximum)

class FlatInterfaceWrapper(base_wrapper.BaseWrapper):
  """Simple interface for AndroidEnv."""

  def __init__(self,
               env: dm_env.Environment,
               flat_actions: bool = True,
               flat_observations: bool = True,
               keep_action_layer: bool = True):
    super().__init__(env)
    self._flat_actions = flat_actions
    self._flat_observations = flat_observations
    self._keep_action_layer = keep_action_layer
    self._action_name = list(self._env.action_spec())[0]
    self._assert_base_env()

  def _assert_base_env(self):
    base_action_spec = self._env.action_spec()
    assert len(base_action_spec) == 1, self._env.action_spec()
    assert isinstance(base_action_spec, dict)
    assert isinstance(base_action_spec[self._action_name], specs.BoundedArray)

  def _process_action(self, action: Union[int, np.ndarray, Dict[str, Any]]):
    if self._flat_actions:
      return {self._action_name: action}
    else:
      return action

  def _process_timestep(self, timestep: dm_env.TimeStep) -> dm_env.TimeStep:
    if self._flat_observations:
      step_type, reward, discount, observation = timestep
      # Keep only the pixels.
      pixels = observation['pixels']
      pixels = pixels if self._keep_action_layer else _extract_screen_pixels(
          pixels)
      return dm_env.TimeStep(
          step_type=step_type,
          reward=reward,
          discount=discount,
          observation=pixels)
    else:
      return timestep

  def reset(self) -> dm_env.TimeStep:
    timestep = self._env.reset()
    return self._process_timestep(timestep)

  def step(self, action: int) -> dm_env.TimeStep:
    timestep = self._env.step(self._process_action(action))
    return self._process_timestep(timestep)

  def observation_spec(self) -> specs.Array:
    if self._flat_observations:
      pixels_spec = self._env.observation_spec()['pixels']
      if not self._keep_action_layer:
        return _get_no_action_observation_spec(pixels_spec)
      return pixels_spec
    else:
      return self._env.observation_spec()

  def action_spec(self) -> specs.Array:
    if self._flat_actions:
      return self._env.action_spec()[self._action_name]
    else:
      return self._env.action_spec()
<|endoftext|>


<commit_before>#!/usr/bin/python

import pywikibot
import re
from datetime import date
import datetime
import logging
import sys

########
# Changing this to 1 makes your changes live on the report page, do not set to
# live mode unless you have been approved for bot usage. Do not merge commits
# where this is not default to 0
########
live = 0
########
# Style: 0 = with date sectionsl 1 = without
########
style = 0
########
# Version Number
########
version = '0.8.2'
########

'''
Copyright (c) 2017 Wugpodes

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''

def monthConvert(name):
    '''
    Takes in either the name of the month or the number of the month and returns
    the opposite. An input of str(July) would return int(7) while an input of
    int(6) would return str(June).
    Takes:   int OR string
    Returns: string OR int
    '''
    if type(name) is str:
        if name == "January": return 1
        elif name == "February": return 2
        elif name == "March": return 3
        elif name == "April": return 4
        elif name == "May": return 5
        elif name == "June": return 6
        elif name == "July": return 7
        elif name == "August": return 8
        elif name == "September": return 9
        elif name == "October": return 10
        elif name == "November": return 11
        elif name == "December": return 12
        else: raise ValueError
    elif type(name) is int:
        if name == 1:return('January')
        elif name == 2:return('February')
        elif name == 3:return('March')
        elif name == 4:return('April')
        elif name == 5:return('May')
        elif name == 6:return('June')
        elif name == 7:return('July')
        elif name == 8:return('August')
        elif name == 9:return('September')
        elif name == 10:return('October')
        elif name == 11: return('November')
        elif name == 12: return('December')
        else: raise ValueError

def checkPage(title):
    '''
    Takes the title of a DYK nom page and loads the wikitext. Looks for a DYK
    tick that shows it was approved or approved (AGF). If so it adds it to
    various lists so that it can be added to the approved page.
    Returns None, implicitly
    '''
    global dates
    global nonDate
    global entries
    global problem
    global nclosed
    global aclosed
    global pageCnt
    title = title.lstrip('{').rstrip('}')
    approved = computeNomStatus(title)
    if approved == -1:
        entries.pop()
        if pageCnt == 0:
            nclosed+=1
        else:
            aclosed+=1
    elif approved == 1:
        dates[-1][1].append('{{'+title+'}}')
        nonDate.append('{{'+title+'}}')
        entries.pop()
    else:
        pass

def computeNomStatus(link,status=0):
    global nclosed
    global aclosed
    global pageCnt
    link = link.lstrip('{').rstrip('}')
    if 'Template:' not in link:
        link = 'Template:'+link
    page = pywikibot.Page(site,link)
    if 'Please do not modify this page.' in page.text:
        status=-1
    else:
        dykchecklist = []
        dykc = 0
        for line in page.text.split('\n'):
            if dykc == 1:
                dykchecklist.append(line)
                if '}}' in line:
                    dykc=0
                    status = computeDYKChecklistStatus('\n'.join(dykchecklist))
            elif '{{DYK checklist' in line:
                if '}}' in line:
                    status = computeDYKChecklistStatus(line)
                else:
                    dykchecklist.append(line)
                    dykc = 1
            if '[[File:Symbol confirmed.svg|16px]]' in line \
            or '[[File:Symbol voting keep.svg|16px]]' in line:
                status = 1
            elif '[[File:Symbol question.svg|16px]]' in line \
            or '[[File:Symbol possible vote.svg|16px]]' in line \
            or '[[File:Symbol delete vote.svg|16px]]' in line \
            or '[[File:Symbol redirect vote 4.svg|16px]]' in line:
                status = 0
    return(status)

def mergeNominations(item = ''):
    '''
    Takes an index for the list dates and appends the entries in that section
    to the proper section on the approved page. Returns None, implicitly
    '''
    global approvedPageDates
    global dates
    if item == '':
        pass
    else:
        for entry in approvedPageDates:
            if dates[item][0] in entry:
                entry[1]+=dates[item][1][1:]

def computeYear(oMonth):
    '''
    Gives correct year over one year change, ie, for 11 months. Sections older
    than 11 months will not yield the correct results
    '''
    now = datetime.datetime.now()
    cMonth = now.month
    cYear = now.year
    if oMonth <= cMonth:
        year = cYear
    else:
        year = cYear - 1
    return(year)

def startLogging(loglevel):
    numeric_level = getattr(logging, loglevel.upper(), None)
    if numeric_level != None:
        logging.basicConfig(
            level=numeric_level,
            filename='Testing.log',
            format='%(asctime)s - %(levelname)s - %(message)s',
            datefmt='%Y-%m-%d %I:%M:%S %p'
        )
    else:
        logging.basicConfig(
            level=logging.WARNING,
            filename='./log/DYKMoverBot.log',
            format='%(asctime)s - %(levelname)s - %(message)s',
            datefmt='%Y-%m-%d %I:%M:%S %p'
        )
        logging.warning('Invalid log level \'%s\'. ' \
                       +'Defaulting to WARNING' % loglevel)

def checkArgs(arg):
    arg = arg.split('=')
    if arg[0] == '--log' or arg[0] == '-l':
        startLogging(arg[1])
    else:
        raise ValueError('Unknown command line argument \'%s\'' % arg[0])

def computeDYKChecklistStatus(template):
    if type(template) is list:
        template='\n'.join(template)
    elif type(template) is not str:
        raise ValueError('template must be either list or string')
    dykcr = re.compile(r'status\s*=\s*(?:y|Y)')
    match = re.search(dykcr,template)
    if match != None:
        return(1)
    else:
        return(0)

def writeOut(p):
    global approvedPage
    global nomPage
    global entries
    global approvedText
    global nonDate
    global version
    global read
    abort = 0
    page = pywikibot.Page(site,read+'/Approved')
    ap = page.text == approvedPage.text
    nom = pywikibot.Page(site,read)
    np = nom.text  == nomPage.text
    if not ap and not np:
        abort = 1
        logging.error('Approved page and nom page has changed, '\
                        +'aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='I ran into an edit conflict on both [[WP:DYKN/A]] and [[WP:DYKN]]'\
            +' at ~~~~~ and did not write either page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
        return()
    if not ap:
        abort = 1
        logging.error('Approved page has changed, aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='During a test I ran into an edit conflict on [[WP:DYKN/A]]'\
            +' at ~~~~~ and did not write the page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
    else:
        aText=''.join(approvedText)
    if not np:
        abort = 1
        logging.error('Nom page has changed, aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='During a test I ran into an edit conflict on [[WP:DYKN]]'\
            +' at ~~~~~ and did not write the page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
    else:
        nText='\n'.join(entries)
    if abort == 0:
        page = pywikibot.Page(site,p)
        page.text=nText
        page.save(str(len(nonDate))+' approved nominations to [[/Approved|'\
              +'approved page]], removing '+str(nclosed)+' closed nominations,'\
              +' WugBot v'+version)
        page = pywikibot.Page(site,p+'/Approved')
        page.text=aText
        page.save('moving '+str(len(nonDate))+' tentatively approved nominations '\
              +'from [[WP:DYKN]], removing '+str(aclosed)+' closed nominations,'\
              +' WugBot v'+version)

# Start log
#for i in range(1,len(sys.argv)):
#    checkArgs(sys.argv[i])

logging.info("### Starting new run ###")
logging.info("live is set to %s" % live)
logging.info("style is set to %s" % style)
logging.info("DYKMoverBot version %s" % version)

if live == -1:
    read  = 'User:Wugapodes/DYKTest'
    write = 'User:Wugapodes/DYKTest'
elif live == 0:
    read  = 'Template talk:Did you know'
    write = 'User:Wugapodes/DYKTest'
elif live == 1:
    read  = 'Template talk:Did you know'
    write = 'Template talk:Did you know'

# Load the various pages
logging.info("Loading pages")
site = pywikibot.Site('en', 'wikipedia')
nomPage      = pywikibot.Page(site,read)
approvedPage = pywikibot.Page(site,read+'/Approved')

dateRegex = re.compile(r'on (.*?) (\d+)=')

DYKpage = nomPage.text.split('\n')
dates   = []
nonDate = []
entries = []
problem = []
nclosed = 0
aclosed = 0
pageCnt = 0

# Get approved nominations on the DYK page
logging.info("Reading Nomination page")
for line in DYKpage:
    entries.append(line)
    if '==Articles' in line:
        matches=dateRegex.search(line)
        month = monthConvert(str(matches.group(1)))
        day = int(matches.group(2))
        dt = datetime.date(month=month,day=day,year=computeYear(month))
        section = line
        dates.append([dt,[section]])
    elif 'Did you know nominations/' in line and '<!--' not in line:
        if '}}{{' in line:
            logging.debug("Multiple nominations on one line.\n"\
                         +"Section: %s" % line)
            splitLine = line.split('}}{{')
            for title in splitLine:
                try:
                    checkPage(title)
                except Exception as e:
                    logging.error(
                        "Function checkPage did not complete successfully.\n"\
                        +str(e)+"\n"+title
                    )
                    logging.debug(line)
                    print(e)
                    problem.append([line,e])
                    continue
        else:
            line = line.split('}')[0]
            try:
                checkPage(line)
            except Exception as e:
                logging.error(
                    "Function checkPage did not complete successfully.\n"\
                    +str(e)+"\n"+line
                )
                print(e)
                problem.append([line,e])
                continue

approveddates=[]
for item in dates:
    if len(item[-1]) > 1:
        approveddates.append(item)

pageCnt = 1
# Get the text on the approved page and then update the proper sections
# with entries newly approved
logging.info("Reading Approved page")
approvedPageText = approvedPage.text.split('\n')
sectionName = ''
approvedPageDates = []
oldLine = ''
datesToRemove = []
if style != 1:
    logging.info("Starting style 0 parsing")
    for line in approvedPageText:
        if '==Articles' in line:
            mergeNominations(oldLine)
            matches=dateRegex.search(line)
            month = monthConvert(str(matches.group(1)))
            day = int(matches.group(2))
            dt = datetime.date(month=month,day=day,year=computeYear(month))
            sectionName = line
            approvedPageDates.append([dt,[sectionName]])
            for item in approveddates:
                if dt in item:
                    approvedPageDates[-1][1]+=item[-1][1:]
                else:
                    continue
        elif 'Did you know nominations/' in line and '<!--' not in line:
            if '}}{{' in line:
                splitLine = line.split('}}{{')
                for title in splitLine:
                    if computeNomStatus(title) >= 0:
                        if '{{'+title+'}}' in approvedPageDates[-1][1]:
                            continue
                        try:
                            approvedPageDates[-1][1].append('{{'+title+'}}')
                        except Exception as e:
                            logging.warning("The approved page is empty?\n"+str(e))
                    else:
                        if pageCnt == 0:
                            nclosed+=1
                        else:
                            aclosed+=1
            else:
                line = line.split('}')[0]
                title = line.split('{')[-1]
                if computeNomStatus(title) >= 0:
                    if '{{'+title+'}}' in approvedPageDates[-1][1]:
                        continue
                    try:
                        approvedPageDates[-1][1].append('{{'+title+'}}')
                    except Exception as e:
                        logging.warning("The approved page is empty?\n"+str(e))
                else:
                    if pageCnt == 0:
                        nclosed+=1
                    else:
                        aclosed+=1
        if '==Special occasion holding area==' in line:
            break
    adts = [x[0] for x in approvedPageDates]
    newSections = [x for x in dates if x[0] not in adts]
    approvedPageDates+=newSections
    toPrint=[]
    approvedPageDates.sort(key=lambda x: x[0])
    for entry in approvedPageDates:
        if len(entry[1]) > 1:
            toPrint+=entry[1]

# Create the page text to be output
logging.info("Creating output text")
passed = 0
approvedText = [
        "{{/top}}\n",
        "=Nominations=\n",
        "==Approved nominations==\n",
        "<!-- This section will hold approved nominations, with the templates "\
        +"transcluded in the same manner as the regular nominations page. "\
        +"While the exact format of the section has not yet been decided-while"\
        +" it seems unlikely that it will be by date, it may be divided into "\
        +"other sections-it is likely that the oldest approvals will go at the"\
        +" top and the most recent ones at the bottom of each section. -->\n"
    ]

approvedText.append('\n'.join(toPrint))
for line in approvedPage.text.split('\n'):
    if '==Special occasion holding area==' in line:
        passed = 1
        approvedText.append('\n'+line+'\n')
    elif passed == 1:
        approvedText.append(line+'\n')

writeOut(write)

print('Done')
<commit_msg>fix type error, still v0.8.2<commit_after>#!/usr/bin/python

import pywikibot
import re
from datetime import date
import datetime
import logging
import sys

########
# Changing this to 1 makes your changes live on the report page, do not set to
# live mode unless you have been approved for bot usage. Do not merge commits
# where this is not default to 0
########
live = 0
########
# Style: 0 = with date sectionsl 1 = without
########
style = 0
########
# Version Number
########
version = '0.8.2'
########

'''
Copyright (c) 2017 Wugpodes

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''

def monthConvert(name):
    '''
    Takes in either the name of the month or the number of the month and returns
    the opposite. An input of str(July) would return int(7) while an input of
    int(6) would return str(June).
    Takes:   int OR string
    Returns: string OR int
    '''
    if type(name) is str:
        if name == "January": return 1
        elif name == "February": return 2
        elif name == "March": return 3
        elif name == "April": return 4
        elif name == "May": return 5
        elif name == "June": return 6
        elif name == "July": return 7
        elif name == "August": return 8
        elif name == "September": return 9
        elif name == "October": return 10
        elif name == "November": return 11
        elif name == "December": return 12
        else: raise ValueError
    elif type(name) is int:
        if name == 1:return('January')
        elif name == 2:return('February')
        elif name == 3:return('March')
        elif name == 4:return('April')
        elif name == 5:return('May')
        elif name == 6:return('June')
        elif name == 7:return('July')
        elif name == 8:return('August')
        elif name == 9:return('September')
        elif name == 10:return('October')
        elif name == 11: return('November')
        elif name == 12: return('December')
        else: raise ValueError

def checkPage(title):
    '''
    Takes the title of a DYK nom page and loads the wikitext. Looks for a DYK
    tick that shows it was approved or approved (AGF). If so it adds it to
    various lists so that it can be added to the approved page.
    Returns None, implicitly
    '''
    global dates
    global nonDate
    global entries
    global problem
    global nclosed
    global aclosed
    global pageCnt
    title = title.lstrip('{').rstrip('}')
    approved = computeNomStatus(title)
    if approved == -1:
        entries.pop()
        if pageCnt == 0:
            nclosed+=1
        else:
            aclosed+=1
    elif approved == 1:
        dates[-1][1].append('{{'+title+'}}')
        nonDate.append('{{'+title+'}}')
        entries.pop()
    else:
        pass

def computeNomStatus(link,status=0):
    global nclosed
    global aclosed
    global pageCnt
    link = link.lstrip('{').rstrip('}')
    if 'Template:' not in link:
        link = 'Template:'+link
    page = pywikibot.Page(site,link)
    if 'Please do not modify this page.' in page.text:
        status=-1
    else:
        dykchecklist = []
        dykc = 0
        for line in page.text.split('\n'):
            if dykc == 1:
                dykchecklist.append(line)
                if '}}' in line:
                    dykc=0
                    status = computeDYKChecklistStatus('\n'.join(dykchecklist))
            elif '{{DYK checklist' in line:
                if '}}' in line:
                    status = computeDYKChecklistStatus(line)
                else:
                    dykchecklist.append(line)
                    dykc = 1
            if '[[File:Symbol confirmed.svg|16px]]' in line \
            or '[[File:Symbol voting keep.svg|16px]]' in line:
                status = 1
            elif '[[File:Symbol question.svg|16px]]' in line \
            or '[[File:Symbol possible vote.svg|16px]]' in line \
            or '[[File:Symbol delete vote.svg|16px]]' in line \
            or '[[File:Symbol redirect vote 4.svg|16px]]' in line:
                status = 0
    return(status)

def mergeNominations(item = ''):
    '''
    Takes an index for the list dates and appends the entries in that section
    to the proper section on the approved page. Returns None, implicitly
    '''
    global approvedPageDates
    global dates
    if item == '':
        pass
    else:
        for entry in approvedPageDates:
            if dates[item][0] in entry:
                entry[1]+=dates[item][1][1:]

def computeYear(oMonth):
    '''
    Gives correct year over one year change, ie, for 11 months. Sections older
    than 11 months will not yield the correct results
    '''
    now = datetime.datetime.now()
    cMonth = now.month
    cYear = now.year
    if oMonth <= cMonth:
        year = cYear
    else:
        year = cYear - 1
    return(year)

def startLogging(loglevel):
    numeric_level = getattr(logging, loglevel.upper(), None)
    if numeric_level != None:
        logging.basicConfig(
            level=numeric_level,
            filename='Testing.log',
            format='%(asctime)s - %(levelname)s - %(message)s',
            datefmt='%Y-%m-%d %I:%M:%S %p'
        )
    else:
        logging.basicConfig(
            level=logging.WARNING,
            filename='./log/DYKMoverBot.log',
            format='%(asctime)s - %(levelname)s - %(message)s',
            datefmt='%Y-%m-%d %I:%M:%S %p'
        )
        logging.warning('Invalid log level \'%s\'. ' \
                       +'Defaulting to WARNING' % loglevel)

def checkArgs(arg):
    arg = arg.split('=')
    if arg[0] == '--log' or arg[0] == '-l':
        startLogging(arg[1])
    else:
        raise ValueError('Unknown command line argument \'%s\'' % arg[0])

def computeDYKChecklistStatus(template):
    if type(template) is list:
        template='\n'.join(template)
    elif type(template) is not str and type(template) is not unicode:
        raise ValueError('template must be either list or string')
    dykcr = re.compile(r'status\s*=\s*(?:y|Y)')
    match = re.search(dykcr,template)
    if match != None:
        return(1)
    else:
        return(0)

def writeOut(p):
    global approvedPage
    global nomPage
    global entries
    global approvedText
    global nonDate
    global version
    global read
    abort = 0
    page = pywikibot.Page(site,read+'/Approved')
    ap = page.text == approvedPage.text
    nom = pywikibot.Page(site,read)
    np = nom.text  == nomPage.text
    if not ap and not np:
        abort = 1
        logging.error('Approved page and nom page has changed, '\
                        +'aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='I ran into an edit conflict on both [[WP:DYKN/A]] and [[WP:DYKN]]'\
            +' at ~~~~~ and did not write either page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
        return()
    if not ap:
        abort = 1
        logging.error('Approved page has changed, aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='During a test I ran into an edit conflict on [[WP:DYKN/A]]'\
            +' at ~~~~~ and did not write the page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
    else:
        aText=''.join(approvedText)
    if not np:
        abort = 1
        logging.error('Nom page has changed, aborting to avoid edit conflict')
        page = pywikibot.Page(site,'User talk:WugBot')
        page.text+='During a test I ran into an edit conflict on [[WP:DYKN]]'\
            +' at ~~~~~ and did not write the page to avoid an edit conflict '\
            +'~~~~'
        page.save('Posting a note about failed test run due to edit conflict')
    else:
        nText='\n'.join(entries)
    if abort == 0:
        page = pywikibot.Page(site,p)
        page.text=nText
        page.save(str(len(nonDate))+' approved nominations to [[/Approved|'\
              +'approved page]], removing '+str(nclosed)+' closed nominations,'\
              +' WugBot v'+version)
        page = pywikibot.Page(site,p+'/Approved')
        page.text=aText
        page.save('moving '+str(len(nonDate))+' tentatively approved nominations '\
              +'from [[WP:DYKN]], removing '+str(aclosed)+' closed nominations,'\
              +' WugBot v'+version)

# Start log
#for i in range(1,len(sys.argv)):
#    checkArgs(sys.argv[i])

logging.info("### Starting new run ###")
logging.info("live is set to %s" % live)
logging.info("style is set to %s" % style)
logging.info("DYKMoverBot version %s" % version)

if live == -1:
    read  = 'User:Wugapodes/DYKTest'
    write = 'User:Wugapodes/DYKTest'
elif live == 0:
    read  = 'Template talk:Did you know'
    write = 'User:Wugapodes/DYKTest'
elif live == 1:
    read  = 'Template talk:Did you know'
    write = 'Template talk:Did you know'

# Load the various pages
logging.info("Loading pages")
site = pywikibot.Site('en', 'wikipedia')
nomPage      = pywikibot.Page(site,read)
approvedPage = pywikibot.Page(site,read+'/Approved')

dateRegex = re.compile(r'on (.*?) (\d+)=')

DYKpage = nomPage.text.split('\n')
dates   = []
nonDate = []
entries = []
problem = []
nclosed = 0
aclosed = 0
pageCnt = 0

# Get approved nominations on the DYK page
logging.info("Reading Nomination page")
for line in DYKpage:
    entries.append(line)
    if '==Articles' in line:
        matches=dateRegex.search(line)
        month = monthConvert(str(matches.group(1)))
        day = int(matches.group(2))
        dt = datetime.date(month=month,day=day,year=computeYear(month))
        section = line
        dates.append([dt,[section]])
    elif 'Did you know nominations/' in line and '<!--' not in line:
        if '}}{{' in line:
            logging.debug("Multiple nominations on one line.\n"\
                         +"Section: %s" % line)
            splitLine = line.split('}}{{')
            for title in splitLine:
                try:
                    checkPage(title)
                except Exception as e:
                    logging.error(
                        "Function checkPage did not complete successfully.\n"\
                        +str(e)+"\n"+title
                    )
                    logging.debug(line)
                    print(e)
                    problem.append([line,e])
                    continue
        else:
            line = line.split('}')[0]
            try:
                checkPage(line)
            except Exception as e:
                logging.error(
                    "Function checkPage did not complete successfully.\n"\
                    +str(e)+"\n"+line
                )
                print(e)
                problem.append([line,e])
                continue

approveddates=[]
for item in dates:
    if len(item[-1]) > 1:
        approveddates.append(item)

pageCnt = 1
# Get the text on the approved page and then update the proper sections
# with entries newly approved
logging.info("Reading Approved page")
approvedPageText = approvedPage.text.split('\n')
sectionName = ''
approvedPageDates = []
oldLine = ''
datesToRemove = []
if style != 1:
    logging.info("Starting style 0 parsing")
    for line in approvedPageText:
        if '==Articles' in line:
            mergeNominations(oldLine)
            matches=dateRegex.search(line)
            month = monthConvert(str(matches.group(1)))
            day = int(matches.group(2))
            dt = datetime.date(month=month,day=day,year=computeYear(month))
            sectionName = line
            approvedPageDates.append([dt,[sectionName]])
            for item in approveddates:
                if dt in item:
                    approvedPageDates[-1][1]+=item[-1][1:]
                else:
                    continue
        elif 'Did you know nominations/' in line and '<!--' not in line:
            if '}}{{' in line:
                splitLine = line.split('}}{{')
                for title in splitLine:
                    if computeNomStatus(title) >= 0:
                        if '{{'+title+'}}' in approvedPageDates[-1][1]:
                            continue
                        try:
                            approvedPageDates[-1][1].append('{{'+title+'}}')
                        except Exception as e:
                            logging.warning("The approved page is empty?\n"+str(e))
                    else:
                        if pageCnt == 0:
                            nclosed+=1
                        else:
                            aclosed+=1
            else:
                line = line.split('}')[0]
                title = line.split('{')[-1]
                if computeNomStatus(title) >= 0:
                    if '{{'+title+'}}' in approvedPageDates[-1][1]:
                        continue
                    try:
                        approvedPageDates[-1][1].append('{{'+title+'}}')
                    except Exception as e:
                        logging.warning("The approved page is empty?\n"+str(e))
                else:
                    if pageCnt == 0:
                        nclosed+=1
                    else:
                        aclosed+=1
        if '==Special occasion holding area==' in line:
            break
    adts = [x[0] for x in approvedPageDates]
    newSections = [x for x in dates if x[0] not in adts]
    approvedPageDates+=newSections
    toPrint=[]
    approvedPageDates.sort(key=lambda x: x[0])
    for entry in approvedPageDates:
        if len(entry[1]) > 1:
            toPrint+=entry[1]

# Create the page text to be output
logging.info("Creating output text")
passed = 0
approvedText = [
        "{{/top}}\n",
        "=Nominations=\n",
        "==Approved nominations==\n",
        "<!-- This section will hold approved nominations, with the templates "\
        +"transcluded in the same manner as the regular nominations page. "\
        +"While the exact format of the section has not yet been decided-while"\
        +" it seems unlikely that it will be by date, it may be divided into "\
        +"other sections-it is likely that the oldest approvals will go at the"\
        +" top and the most recent ones at the bottom of each section. -->\n"
    ]

approvedText.append('\n'.join(toPrint))
for line in approvedPage.text.split('\n'):
    if '==Special occasion holding area==' in line:
        passed = 1
        approvedText.append('\n'+line+'\n')
    elif passed == 1:
        approvedText.append(line+'\n')

writeOut(write)

print('Done')
<|endoftext|>