<commit_before>import logging
import os
import datetime
import json

from google.appengine.api import taskqueue
from google.appengine.ext import ndb
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template

from datafeeds.datafeed_fms import DatafeedFms
from datafeeds.datafeed_tba import DatafeedTba
from datafeeds.datafeed_usfirst import DatafeedUsfirst
from datafeeds.datafeed_offseason import DatafeedOffseason
from datafeeds.datafeed_twitter import DatafeedTwitter

from helpers.event_helper import EventHelper
from helpers.event_manipulator import EventManipulator
from helpers.event_team_manipulator import EventTeamManipulator
from helpers.match_manipulator import MatchManipulator
from helpers.match_helper import MatchHelper
from helpers.award_manipulator import AwardManipulator
from helpers.team_helper import TeamHelper, TeamTpidHelper
from helpers.team_manipulator import TeamManipulator

from models.event import Event
from models.event_team import EventTeam
from models.match import Match
from models.team import Team

from helpers.firebase.firebase_pusher import FirebasePusher

class FmsEventListGet(webapp.RequestHandler):
    """
    Fetch basic data about all current season events at once.
    """
    def get(self):
        df = DatafeedFms()
        events = df.getFmsEventList()

        # filter if first_eid is too high, meaning its a Championship Division
        # (we manually add these due to naming issues)
        events = filter(lambda e: int(e.first_eid) < 100000, events)

        events = EventManipulator.createOrUpdate(events)

        template_values = {
            "events": events
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_event_list_get.html')
        self.response.out.write(template.render(path, template_values))

class FmsTeamListGet(webapp.RequestHandler):
    """
    Fetch basic data about all current season teams at once.
    Doesn't get tpids or full data.
    """
    def get(self):
        df = DatafeedFms()
        teams = df.getFmsTeamList()
        TeamManipulator.createOrUpdate(teams)

        template_values = {
            "teams": teams
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_team_list_get.html')
        self.response.out.write(template.render(path, template_values))

class TbaVideosEnqueue(webapp.RequestHandler):
    """
    Handles enqueing grabing tba_videos for Matches at individual Events.
    """
    def get(self):
        events = Event.query()

        for event in events:
            taskqueue.add(
                url='/tasks/get/tba_videos/' + event.key_name,
                method='GET')

        template_values = {
            'event_count': Event.query().count(),
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/tba_videos_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class TbaVideosGet(webapp.RequestHandler):
    """
    Handles reading a TBA video listing page and updating the match objects in the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedTba()

        event = Event.get_by_id(event_key)
        match_filetypes = df.getVideos(event)
        if match_filetypes:
            matches_to_put = []
            for match in event.matches:
                if match.tba_videos != match_filetypes.get(match.key_name, []):
                    match.tba_videos = match_filetypes.get(match.key_name, [])
                    match.dirty = True
                    matches_to_put.append(match)

            MatchManipulator.createOrUpdate(matches_to_put)

            tbavideos = match_filetypes.items()
        else:
            logging.info("No tbavideos found for event " + event.key_name)
            tbavideos = []

        template_values = {
            'tbavideos': tbavideos,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/tba_videos_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventDetailsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing updates to individual USFIRST events.
    """
    def get(self, year):
        events = Event.query(Event.first_eid != None, Event.year == int(year))

        for event in events.fetch(200):
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_details/%s/%s' % (year, event.first_eid),
                method='GET')

        template_values = {
            'event_count': events.count(),
            'year': year,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_events_details_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventDetailsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST event page and creating or updating the model as needed.
    Includes registered Teams.
    """
    def get(self, year, first_eid):
        datafeed = DatafeedUsfirst()

        event = datafeed.getEventDetails(int(year), first_eid)
        event = EventManipulator.createOrUpdate(event)

        teams = datafeed.getEventTeams(int(year), first_eid)
        teams = TeamManipulator.createOrUpdate(teams)

        if teams:
            event_teams = [EventTeam(
                id = event.key.id() + "_" + team.key.id(),
                event = event.key,
                team = team.key,
                year = event.year)
                for team in teams]
            event_teams = EventTeamManipulator.createOrUpdate(event_teams)
        else:
            event_teams = []

        template_values = {
            'event': event,
            'event_teams': event_teams,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstAwardsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting awards for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_awards/%s' % (event.key_name),
                method='GET')
        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_awards_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstAwardsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        datafeed = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        new_awards = AwardManipulator.createOrUpdate(datafeed.getEventAwards(event))

        template_values = {
            'awards': new_awards,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_awards_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventListGet(webapp.RequestHandler):
    """
    Handles reading the USFIRST event list.
    Enqueues a bunch of detailed reads that actually establish Event objects.
    """
    def get(self, year):
        df = DatafeedUsfirst()
        events = df.getEventList(int(year))

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_details/%s/%s' % (year, event.first_eid),
                method='GET')

        template_values = {
            'events': events,
            'year': year
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_list_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstMatchesEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting match results for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_matches/' + event.key_name,
                method='GET')

        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_matches_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstMatchesGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        new_matches = MatchManipulator.createOrUpdate(df.getMatches(event))

        last_matches = MatchHelper.recentMatches(new_matches, 1)
        upcoming_matches = MatchHelper.upcomingMatches(new_matches, 8)
        try:
            FirebasePusher.updateEvent(event, last_matches, upcoming_matches)
        except:
            logging.warning("Enqueuing Firebase push failed!")

        template_values = {
            'matches': new_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_matches_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventRankingsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting rankings for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_rankings/' + event.key_name,
                method='GET')

        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_rankings_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventRankingsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST ranking page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        rankings = df.getEventRankings(event)
        if event.rankings_json != json.dumps(rankings):
            event.rankings_json = json.dumps(rankings)
            event.dirty = True

        EventManipulator.createOrUpdate(event)

        template_values = {'rankings': rankings,
                           'event_name': event.key_name}

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_rankings_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstTeamDetailsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing updates to individual USFIRST teams.
    """
    def get(self):
        offset = int(self.request.get("offset", 0))

        teams = Team.query().fetch(1000, offset=int(offset))
        for team in teams:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_team_details/' + team.key_name,
                method='GET')

        # FIXME omg we're just writing out? -gregmarra 2012 Aug 26
        self.response.out.write("%s team gets have been enqueued offset from %s.<br />" %(len(teams), offset))
        self.response.out.write("Reload with ?offset=%s to enqueue more." % (offset + len(teams)))

class UsfirstTeamDetailsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST team information page and updating the
    model accordingly.
    """
    def get(self, key_name):
        df = DatafeedUsfirst()
        team = df.getTeamDetails(Team.get_by_id(key_name))
        if team:
            team = TeamManipulator.createOrUpdate(team)
            success = True
        else:
            success = False

        template_values = {
            'team': team,
            'success': success,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_team_details_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstTeamsTpidsGet(webapp.RequestHandler):
    """
    A run-as-needed function that instantiates new Team objects based on
    FIRST's full team list.
    """
    def get(self, year):
        df = DatafeedUsfirst()
        skip = 0

        try:
            skip = self.request.get("skip")
            if skip == '':
                skip = 0
        except Exception, detail:
            logging.error('Failed to get skip value')

        logging.info("YEAR: %s", year)
        df.getTeamsTpids(int(year), skip)

        team_count = Team.query().count()

        template_values = {
            'team_count': team_count
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_teams_tpids.html')
        self.response.out.write(template.render(path, template_values))

class OffseasonMatchesGet(webapp.RequestHandler):
    """
    Handles reading an offseason match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedOffseason()

        event = Event.get_by_id(event_key)
        url = self.request.get('url')

        new_matches = MatchManipulator.createOrUpdate(df.getMatches(event, url))

        template_values = {
            'matches': new_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/offseason_matches_get.html')
        self.response.out.write(template.render(path, template_values))

class TwitterFrcfmsMatchesGet(webapp.RequestHandler):
    """
    Handles getting matches from @FRCFMS on Twitter, and returns a table of
    matches that can be manually manipulated and added
    """
    def get(self):
        df = DatafeedTwitter()

        event_matches = df.getMatches()

        template_values = {
            'event_matches': event_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/twitter_frcfms_matches_get.html')
        self.response.out.write(template.render(path, template_values))
<commit_msg>more try catch for firebase prep<commit_after>import logging
import os
import datetime
import json

from google.appengine.api import taskqueue
from google.appengine.ext import ndb
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template

from datafeeds.datafeed_fms import DatafeedFms
from datafeeds.datafeed_tba import DatafeedTba
from datafeeds.datafeed_usfirst import DatafeedUsfirst
from datafeeds.datafeed_offseason import DatafeedOffseason
from datafeeds.datafeed_twitter import DatafeedTwitter

from helpers.event_helper import EventHelper
from helpers.event_manipulator import EventManipulator
from helpers.event_team_manipulator import EventTeamManipulator
from helpers.match_manipulator import MatchManipulator
from helpers.match_helper import MatchHelper
from helpers.award_manipulator import AwardManipulator
from helpers.team_helper import TeamHelper, TeamTpidHelper
from helpers.team_manipulator import TeamManipulator

from models.event import Event
from models.event_team import EventTeam
from models.match import Match
from models.team import Team

from helpers.firebase.firebase_pusher import FirebasePusher

class FmsEventListGet(webapp.RequestHandler):
    """
    Fetch basic data about all current season events at once.
    """
    def get(self):
        df = DatafeedFms()
        events = df.getFmsEventList()

        # filter if first_eid is too high, meaning its a Championship Division
        # (we manually add these due to naming issues)
        events = filter(lambda e: int(e.first_eid) < 100000, events)

        events = EventManipulator.createOrUpdate(events)

        template_values = {
            "events": events
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_event_list_get.html')
        self.response.out.write(template.render(path, template_values))

class FmsTeamListGet(webapp.RequestHandler):
    """
    Fetch basic data about all current season teams at once.
    Doesn't get tpids or full data.
    """
    def get(self):
        df = DatafeedFms()
        teams = df.getFmsTeamList()
        TeamManipulator.createOrUpdate(teams)

        template_values = {
            "teams": teams
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_team_list_get.html')
        self.response.out.write(template.render(path, template_values))

class TbaVideosEnqueue(webapp.RequestHandler):
    """
    Handles enqueing grabing tba_videos for Matches at individual Events.
    """
    def get(self):
        events = Event.query()

        for event in events:
            taskqueue.add(
                url='/tasks/get/tba_videos/' + event.key_name,
                method='GET')

        template_values = {
            'event_count': Event.query().count(),
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/tba_videos_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class TbaVideosGet(webapp.RequestHandler):
    """
    Handles reading a TBA video listing page and updating the match objects in the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedTba()

        event = Event.get_by_id(event_key)
        match_filetypes = df.getVideos(event)
        if match_filetypes:
            matches_to_put = []
            for match in event.matches:
                if match.tba_videos != match_filetypes.get(match.key_name, []):
                    match.tba_videos = match_filetypes.get(match.key_name, [])
                    match.dirty = True
                    matches_to_put.append(match)

            MatchManipulator.createOrUpdate(matches_to_put)

            tbavideos = match_filetypes.items()
        else:
            logging.info("No tbavideos found for event " + event.key_name)
            tbavideos = []

        template_values = {
            'tbavideos': tbavideos,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/tba_videos_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventDetailsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing updates to individual USFIRST events.
    """
    def get(self, year):
        events = Event.query(Event.first_eid != None, Event.year == int(year))

        for event in events.fetch(200):
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_details/%s/%s' % (year, event.first_eid),
                method='GET')

        template_values = {
            'event_count': events.count(),
            'year': year,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_events_details_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventDetailsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST event page and creating or updating the model as needed.
    Includes registered Teams.
    """
    def get(self, year, first_eid):
        datafeed = DatafeedUsfirst()

        event = datafeed.getEventDetails(int(year), first_eid)
        event = EventManipulator.createOrUpdate(event)

        teams = datafeed.getEventTeams(int(year), first_eid)
        teams = TeamManipulator.createOrUpdate(teams)

        if teams:
            event_teams = [EventTeam(
                id = event.key.id() + "_" + team.key.id(),
                event = event.key,
                team = team.key,
                year = event.year)
                for team in teams]
            event_teams = EventTeamManipulator.createOrUpdate(event_teams)
        else:
            event_teams = []

        template_values = {
            'event': event,
            'event_teams': event_teams,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstAwardsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting awards for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_awards/%s' % (event.key_name),
                method='GET')
        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_awards_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstAwardsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        datafeed = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        new_awards = AwardManipulator.createOrUpdate(datafeed.getEventAwards(event))

        template_values = {
            'awards': new_awards,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_awards_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventListGet(webapp.RequestHandler):
    """
    Handles reading the USFIRST event list.
    Enqueues a bunch of detailed reads that actually establish Event objects.
    """
    def get(self, year):
        df = DatafeedUsfirst()
        events = df.getEventList(int(year))

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_details/%s/%s' % (year, event.first_eid),
                method='GET')

        template_values = {
            'events': events,
            'year': year
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_list_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstMatchesEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting match results for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_matches/' + event.key_name,
                method='GET')

        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_matches_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstMatchesGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        new_matches = MatchManipulator.createOrUpdate(df.getMatches(event))

        try:
            last_matches = MatchHelper.recentMatches(new_matches, 1)
            upcoming_matches = MatchHelper.upcomingMatches(new_matches, 8)
        except:
            logging.warning("Computing last/upcoming matches for Firebase failed!")
        try:
            FirebasePusher.updateEvent(event, last_matches, upcoming_matches)
        except:
            logging.warning("Enqueuing Firebase push failed!")

        template_values = {
            'matches': new_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_matches_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventRankingsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing getting rankings for USFIRST events.
    """
    def get(self, when):
        if when == "now":
            events = EventHelper.getEventsWithinADay()
        else:
            events = Event.query(Event.official == True)
            events = events.filter(Event.year == int(when))
            events = events.fetch(500)

        for event in events:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_event_rankings/' + event.key_name,
                method='GET')

        template_values = {
            'events': events,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_rankings_enqueue.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstEventRankingsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST ranking page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedUsfirst()

        event = Event.get_by_id(event_key)
        rankings = df.getEventRankings(event)
        if event.rankings_json != json.dumps(rankings):
            event.rankings_json = json.dumps(rankings)
            event.dirty = True

        EventManipulator.createOrUpdate(event)

        template_values = {'rankings': rankings,
                           'event_name': event.key_name}

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_rankings_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstTeamDetailsEnqueue(webapp.RequestHandler):
    """
    Handles enqueing updates to individual USFIRST teams.
    """
    def get(self):
        offset = int(self.request.get("offset", 0))

        teams = Team.query().fetch(1000, offset=int(offset))
        for team in teams:
            taskqueue.add(
                queue_name='usfirst',
                url='/tasks/get/usfirst_team_details/' + team.key_name,
                method='GET')

        # FIXME omg we're just writing out? -gregmarra 2012 Aug 26
        self.response.out.write("%s team gets have been enqueued offset from %s.<br />" %(len(teams), offset))
        self.response.out.write("Reload with ?offset=%s to enqueue more." % (offset + len(teams)))

class UsfirstTeamDetailsGet(webapp.RequestHandler):
    """
    Handles reading a USFIRST team information page and updating the
    model accordingly.
    """
    def get(self, key_name):
        df = DatafeedUsfirst()
        team = df.getTeamDetails(Team.get_by_id(key_name))
        if team:
            team = TeamManipulator.createOrUpdate(team)
            success = True
        else:
            success = False

        template_values = {
            'team': team,
            'success': success,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_team_details_get.html')
        self.response.out.write(template.render(path, template_values))

class UsfirstTeamsTpidsGet(webapp.RequestHandler):
    """
    A run-as-needed function that instantiates new Team objects based on
    FIRST's full team list.
    """
    def get(self, year):
        df = DatafeedUsfirst()
        skip = 0

        try:
            skip = self.request.get("skip")
            if skip == '':
                skip = 0
        except Exception, detail:
            logging.error('Failed to get skip value')

        logging.info("YEAR: %s", year)
        df.getTeamsTpids(int(year), skip)

        team_count = Team.query().count()

        template_values = {
            'team_count': team_count
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_teams_tpids.html')
        self.response.out.write(template.render(path, template_values))

class OffseasonMatchesGet(webapp.RequestHandler):
    """
    Handles reading an offseason match results page and updating the datastore as needed.
    """
    def get(self, event_key):
        df = DatafeedOffseason()

        event = Event.get_by_id(event_key)
        url = self.request.get('url')

        new_matches = MatchManipulator.createOrUpdate(df.getMatches(event, url))

        template_values = {
            'matches': new_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/offseason_matches_get.html')
        self.response.out.write(template.render(path, template_values))

class TwitterFrcfmsMatchesGet(webapp.RequestHandler):
    """
    Handles getting matches from @FRCFMS on Twitter, and returns a table of
    matches that can be manually manipulated and added
    """
    def get(self):
        df = DatafeedTwitter()

        event_matches = df.getMatches()

        template_values = {
            'event_matches': event_matches,
        }

        path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/twitter_frcfms_matches_get.html')
        self.response.out.write(template.render(path, template_values))
<|endoftext|>


<commit_before>"""Module for building models using ISAMBARD."""

import enum
import itertools
import sys

import isambard
import isambard.add_ons.knobs_into_holes as kihs

REGISTER_ADJUST = {
    'a': 0,
    'b': 102.8,
    'c': 205.6,
    'd': 308.4,
    'e': 51.4,
    'f': 154.2,
    'g': 257
}

def build_coiled_coil(parameters, debug=False):
    """Builds a model of a coiled coil using the input parameters.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil
        i.e. oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    coiled_coil = isambard.specifications.CoiledCoil(
        len(parameters), auto_build=False)
    coiled_coil.aas = [len(p['Sequence']) for p in parameters]
    coiled_coil.major_radii = [p['Radius'] for p in parameters]
    coiled_coil.major_pitches = [p['Pitch'] for p in parameters]
    raw_phi = [p['Interface Angle'] for p in parameters]
    registers = [p['Register'] for p in parameters]
    coiled_coil.phi_c_alphas = [ia + REGISTER_ADJUST[r]
                                for ia, r in zip(raw_phi, registers)]
    sequences = [p['Sequence'] for p in parameters]
    coiled_coil.z_shifts = [p['Z-Shift'] for p in parameters]
    lshr_adjust = [(p['Z-Shift'] / p['Pitch']) * 360 if p['Linked SHR'] else 0
                   for p in parameters]
    coiled_coil.rotational_offsets = [d + p['Super-Helical Rotation'] - a
                                      for d, p, a in zip(
                                          coiled_coil.rotational_offsets,
                                          parameters,
                                          lshr_adjust)]
    coiled_coil.orientations = [-1 if p['Orientation']
                                else 1 for p in parameters]
    coiled_coil.build()
    coiled_coil.pack_new_sequences(sequences)
    mean_rpt_value = calculate_average_rpt(coiled_coil)
    score = coiled_coil.buff_interaction_energy.total_energy
    knobs = [k.knob_residue for k in kihs.find_kihs(coiled_coil)]
    knob_ids = [[k.ampal_parent.id, k.id] for k in knobs]
    return coiled_coil.pdb, score, mean_rpt_value, knob_ids

def build_collagen(parameters, debug=False):
    """Builds a model of a collagen triple-helix using the input parameters.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil
        i.e. radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    collagen = isambard.specifications.CoiledCoil.tropocollagen(
        auto_build=False)
    collagen.aas = [len(p['Sequence']) for p in parameters]
    collagen.major_radii = [p['Radius'] for p in parameters]
    collagen.major_pitches = [p['Pitch'] for p in parameters]
    collagen.phi_c_alphas = [p['Interface Angle'] for p in parameters]
    original_sequences = [p['Sequence'] for p in parameters]
    proline_sequences = [seq.replace('O', 'P') for seq in original_sequences]
    collagen.z_shifts = [
        d + p['Z-Shift'] for d, p in zip(
            collagen.z_shifts, parameters)]
    lshr_adjust = [(z / p['Pitch']) * 360 if p['Linked SHR'] else 0
                   for z, p in zip(collagen.z_shifts, parameters)]
    collagen.rotational_offsets = [d + p['Super-Helical Rotation'] + a
                                   for d, p, a in zip(
                                       collagen.rotational_offsets,
                                       parameters,
                                       lshr_adjust)]
    collagen.orientations = [-1 if p['Orientation']
                             else 1 for p in parameters]
    collagen.build()
    collagen.pack_new_sequences(proline_sequences)
    score = collagen.buff_interaction_energy.total_energy
    mean_rpt_value = calculate_average_rpt(collagen)
    for res, ml in zip(collagen.get_monomers(), ''.join(original_sequences)):
        if ml == 'O':
            isambard.ampal.non_canonical.convert_pro_to_hyp(res)
    knob_ids = []  # Collagen can't have KIHs
    return collagen.pdb, score, mean_rpt_value, knob_ids

def optimise_coiled_coil(parameters, debug=False):
    """Optimises the parameters for a given structure.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil i.e.
        oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    oligomer_state = len(parameters)
    radius_centre = parameters[0]['Radius']
    phi_centre = parameters[0]['Interface Angle'] + \
        REGISTER_ADJUST[parameters[0]['Register']]
    opt = isambard.optimisation.optimizer.GA_Opt_Internal(
        isambard.specifications.CoiledCoil.from_parameters)
    opt.parameters(
        [parameters[0]['Sequence']] * oligomer_state,
        [radius_centre, 300, phi_centre],
        [2, 150, 20],
        [oligomer_state, len(parameters[0]['Sequence']),
         'var0', 'var1', 'var2']
    )
    opt.run_opt(20, 5, 2)
    top_model = opt.best_model
    knobs = [k.knob_residue for k in kihs.find_kihs(top_model)]
    knob_ids = [[k.ampal_parent.id, k.id] for k in knobs]
    optimised_parameters = {
        'radius': top_model.major_radii[0],
        'pitch': top_model.major_pitches[0],
        'phiCA': top_model.phi_c_alphas[0],
        'sequence': top_model[0].sequence,
        'register': parameters[0]['Register']
    }
    model_and_info = {
        'pdb': top_model.pdb,
        'mean_rpt_value': calculate_average_rpt(top_model),
        'score': top_model.buff_interaction_energy.total_energy,
        'knob_ids': knob_ids
    }
    return optimised_parameters, model_and_info

def optimise_collagen(parameters, debug=False):
    """Optimises the parameters for a collagen structure.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil i.e.
        oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    radius_centre = parameters[0]['Radius']
    phi_centre = parameters[0]['Interface Angle']
    opt = isambard.optimisation.optimizer.GA_Opt_Internal(OptCollagen)
    opt.parameters(
        [parameters[0]['Sequence']] * 3,
        [radius_centre, 150, phi_centre],
        [2, 100, 20],
        [len(parameters[0]['Sequence']), 'var0', 'var1', 'var2']
    )
    opt.run_opt(20, 5, 2)
    top_model = opt.best_model
    knob_ids = []
    optimised_parameters = {
        'radius': top_model.major_radii[0],
        'pitch': top_model.major_pitches[0],
        'phiCA': top_model.phi_c_alphas[0],
        'sequence': top_model[0].sequence,
        'register': parameters[0]['Register']
    }
    model_and_info = {
        'pdb': top_model.pdb,
        'mean_rpt_value': calculate_average_rpt(top_model),
        'score': top_model.buff_interaction_energy.total_energy,
        'knob_ids': knob_ids
    }
    return optimised_parameters, model_and_info

def calculate_average_rpt(ampal):
    """Returns the mean residues per turn value for an AMPAL object."""
    rpt_lists = [isambard.analyse_protein.residues_per_turn(ch)[
        :-1] for ch in ampal]
    rpt_values = list(itertools.chain(*rpt_lists))
    average_rpt = sum(rpt_values) / len(rpt_values)
    return average_rpt

class OptCollagen(isambard.specifications.CoiledCoil):
    """A wrapper class for building collagens with simple parameters."""
    def __init__(self, aas, major_radius, major_pitch, interface_angle):
        n = 3
        super().__init__(n, auto_build=False)
        self.aas = [aas] * n
        self.phi_c_alphas = [interface_angle] * n
        self.minor_helix_types = ['PPII'] * n
        self.major_pitches = [major_pitch] * n
        self.major_radii = [major_radius] * n
        self.major_handedness = ['r'] * n
        self.minor_repeats = [None] * n
        rpr_ppii = 3.1
        self.z_shifts = [-rpr_ppii * 2, 0.0, -rpr_ppii]
        rot_adj = [(z / p) * 360 for z,
                   p in zip(self.z_shifts, self.major_pitches)]
        self.rotational_offsets = [r + ra for r, ra in zip(
            self.rotational_offsets, rot_adj)]
        self.build()

class HelixType(enum.Enum):
    """Possible helix types that can be encountered."""
    ALPHA = 1
    COLLAGEN = 2
<commit_msg>Fixes returned phiCA not being corrected for register.<commit_after>"""Module for building models using ISAMBARD."""

import enum
import itertools
import sys

import isambard
import isambard.add_ons.knobs_into_holes as kihs

REGISTER_ADJUST = {
    'a': 0,
    'b': 102.8,
    'c': 205.6,
    'd': 308.4,
    'e': 51.4,
    'f': 154.2,
    'g': 257
}

def build_coiled_coil(parameters, debug=False):
    """Builds a model of a coiled coil using the input parameters.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil
        i.e. oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    coiled_coil = isambard.specifications.CoiledCoil(
        len(parameters), auto_build=False)
    coiled_coil.aas = [len(p['Sequence']) for p in parameters]
    coiled_coil.major_radii = [p['Radius'] for p in parameters]
    coiled_coil.major_pitches = [p['Pitch'] for p in parameters]
    raw_phi = [p['Interface Angle'] for p in parameters]
    registers = [p['Register'] for p in parameters]
    coiled_coil.phi_c_alphas = [ia + REGISTER_ADJUST[r]
                                for ia, r in zip(raw_phi, registers)]
    sequences = [p['Sequence'] for p in parameters]
    coiled_coil.z_shifts = [p['Z-Shift'] for p in parameters]
    lshr_adjust = [(p['Z-Shift'] / p['Pitch']) * 360 if p['Linked SHR'] else 0
                   for p in parameters]
    coiled_coil.rotational_offsets = [d + p['Super-Helical Rotation'] - a
                                      for d, p, a in zip(
                                          coiled_coil.rotational_offsets,
                                          parameters,
                                          lshr_adjust)]
    coiled_coil.orientations = [-1 if p['Orientation']
                                else 1 for p in parameters]
    coiled_coil.build()
    coiled_coil.pack_new_sequences(sequences)
    mean_rpt_value = calculate_average_rpt(coiled_coil)
    score = coiled_coil.buff_interaction_energy.total_energy
    knobs = [k.knob_residue for k in kihs.find_kihs(coiled_coil)]
    knob_ids = [[k.ampal_parent.id, k.id] for k in knobs]
    return coiled_coil.pdb, score, mean_rpt_value, knob_ids

def build_collagen(parameters, debug=False):
    """Builds a model of a collagen triple-helix using the input parameters.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil
        i.e. radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    collagen = isambard.specifications.CoiledCoil.tropocollagen(
        auto_build=False)
    collagen.aas = [len(p['Sequence']) for p in parameters]
    collagen.major_radii = [p['Radius'] for p in parameters]
    collagen.major_pitches = [p['Pitch'] for p in parameters]
    collagen.phi_c_alphas = [p['Interface Angle'] for p in parameters]
    original_sequences = [p['Sequence'] for p in parameters]
    proline_sequences = [seq.replace('O', 'P') for seq in original_sequences]
    collagen.z_shifts = [
        d + p['Z-Shift'] for d, p in zip(
            collagen.z_shifts, parameters)]
    lshr_adjust = [(z / p['Pitch']) * 360 if p['Linked SHR'] else 0
                   for z, p in zip(collagen.z_shifts, parameters)]
    collagen.rotational_offsets = [d + p['Super-Helical Rotation'] + a
                                   for d, p, a in zip(
                                       collagen.rotational_offsets,
                                       parameters,
                                       lshr_adjust)]
    collagen.orientations = [-1 if p['Orientation']
                             else 1 for p in parameters]
    collagen.build()
    collagen.pack_new_sequences(proline_sequences)
    score = collagen.buff_interaction_energy.total_energy
    mean_rpt_value = calculate_average_rpt(collagen)
    for res, ml in zip(collagen.get_monomers(), ''.join(original_sequences)):
        if ml == 'O':
            isambard.ampal.non_canonical.convert_pro_to_hyp(res)
    knob_ids = []  # Collagen can't have KIHs
    return collagen.pdb, score, mean_rpt_value, knob_ids

def optimise_coiled_coil(parameters, debug=False):
    """Optimises the parameters for a given structure.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil i.e.
        oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    oligomer_state = len(parameters)
    radius_centre = parameters[0]['Radius']
    phi_centre = parameters[0]['Interface Angle'] + \
        REGISTER_ADJUST[parameters[0]['Register']]
    opt = isambard.optimisation.optimizer.GA_Opt_Internal(
        isambard.specifications.CoiledCoil.from_parameters)
    opt.parameters(
        [parameters[0]['Sequence']] * oligomer_state,
        [radius_centre, 300, phi_centre],
        [2, 150, 20],
        [oligomer_state, len(parameters[0]['Sequence']),
         'var0', 'var1', 'var2']
    )
    opt.run_opt(20, 5, 2)
    top_model = opt.best_model
    knobs = [k.knob_residue for k in kihs.find_kihs(top_model)]
    knob_ids = [[k.ampal_parent.id, k.id] for k in knobs]
    optimised_parameters = {
        'radius': top_model.major_radii[0],
        'pitch': top_model.major_pitches[0],
        'phiCA': top_model.phi_c_alphas[0] - \
            REGISTER_ADJUST[parameters[0]['Register']],
        'sequence': top_model[0].sequence,
        'register': parameters[0]['Register']
    }
    model_and_info = {
        'pdb': top_model.pdb,
        'mean_rpt_value': calculate_average_rpt(top_model),
        'score': top_model.buff_interaction_energy.total_energy,
        'knob_ids': knob_ids
    }
    return optimised_parameters, model_and_info

def optimise_collagen(parameters, debug=False):
    """Optimises the parameters for a collagen structure.

    Parameters
    ----------
    parameter : [ dict[str, int/float/str] ]
        List of parameter dictionaries required for building a coiled coil i.e.
        oligomer state, radius, pitch, phiCA, sequence.

    Returns
    -------
    model_data : dict
        A dictionary containing information about the model that has
        been produced.
    """
    if debug:
        print(parameters, file=sys.stderr)
    radius_centre = parameters[0]['Radius']
    phi_centre = parameters[0]['Interface Angle']
    opt = isambard.optimisation.optimizer.GA_Opt_Internal(OptCollagen)
    opt.parameters(
        [parameters[0]['Sequence']] * 3,
        [radius_centre, 150, phi_centre],
        [2, 100, 20],
        [len(parameters[0]['Sequence']), 'var0', 'var1', 'var2']
    )
    opt.run_opt(20, 5, 2)
    top_model = opt.best_model
    knob_ids = []
    optimised_parameters = {
        'radius': top_model.major_radii[0],
        'pitch': top_model.major_pitches[0],
        'phiCA': top_model.phi_c_alphas[0],
        'sequence': top_model[0].sequence,
        'register': parameters[0]['Register']
    }
    model_and_info = {
        'pdb': top_model.pdb,
        'mean_rpt_value': calculate_average_rpt(top_model),
        'score': top_model.buff_interaction_energy.total_energy,
        'knob_ids': knob_ids
    }
    return optimised_parameters, model_and_info

def calculate_average_rpt(ampal):
    """Returns the mean residues per turn value for an AMPAL object."""
    rpt_lists = [isambard.analyse_protein.residues_per_turn(ch)[
        :-1] for ch in ampal]
    rpt_values = list(itertools.chain(*rpt_lists))
    average_rpt = sum(rpt_values) / len(rpt_values)
    return average_rpt

class OptCollagen(isambard.specifications.CoiledCoil):
    """A wrapper class for building collagens with simple parameters."""
    def __init__(self, aas, major_radius, major_pitch, interface_angle):
        n = 3
        super().__init__(n, auto_build=False)
        self.aas = [aas] * n
        self.phi_c_alphas = [interface_angle] * n
        self.minor_helix_types = ['PPII'] * n
        self.major_pitches = [major_pitch] * n
        self.major_radii = [major_radius] * n
        self.major_handedness = ['r'] * n
        self.minor_repeats = [None] * n
        rpr_ppii = 3.1
        self.z_shifts = [-rpr_ppii * 2, 0.0, -rpr_ppii]
        rot_adj = [(z / p) * 360 for z,
                   p in zip(self.z_shifts, self.major_pitches)]
        self.rotational_offsets = [r + ra for r, ra in zip(
            self.rotational_offsets, rot_adj)]
        self.build()

class HelixType(enum.Enum):
    """Possible helix types that can be encountered."""
    ALPHA = 1
    COLLAGEN = 2
<|endoftext|>


<commit_before>from werkzeug.utils import redirect
from mailu import models, utils
from mailu.sso import sso, forms
from mailu.ui import access

from flask import current_app as app
import flask
import flask_login

@sso.route('/login', methods=['GET', 'POST'])
def login():
    client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
    form = forms.LoginForm()
    form.submitAdmin.label.text = form.submitAdmin.label.text + ' Admin'
    form.submitWebmail.label.text = form.submitWebmail.label.text + ' Webmail'

    fields = []
    if str(app.config["ADMIN"]).upper() != "FALSE":
        fields.append(form.submitAdmin)
    if str(app.config["WEBMAIL"]).upper() != "NONE":
        fields.append(form.submitWebmail)

    if form.validate_on_submit():
        if form.submitAdmin.data:
            destination = app.config['WEB_ADMIN']
        elif form.submitWebmail.data:
            destination = app.config['WEB_WEBMAIL']
        device_cookie, device_cookie_username = utils.limiter.parse_device_cookie(flask.request.cookies.get('rate_limit'))
        username = form.email.data
        if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip):
            flask.flash('Too many attempts from your IP (rate-limit)', 'error')
            return flask.render_template('login.html', form=form)
        if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username):
            flask.flash('Too many attempts for this user (rate-limit)', 'error')
            return flask.render_template('login.html', form=form)
        user = models.User.login(username, form.pw.data)
        if user:
            flask.session.regenerate()
            flask_login.login_user(user)
            response = flask.redirect(destination)
            response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'))
            flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.')
            return response
        else:
            utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip)
            flask.current_app.logger.warn(f'Login failed for {username} from {client_ip}.')
            flask.flash('Wrong e-mail or password', 'error')
    return flask.render_template('login.html', form=form, fields=fields)

@sso.route('/logout', methods=['GET'])
@access.authenticated
def logout():
    flask_login.logout_user()
    flask.session.destroy()
    return flask.redirect(flask.url_for('.login'))

<commit_msg>Set the right flags on the rate_limit cookie<commit_after>from werkzeug.utils import redirect
from mailu import models, utils
from mailu.sso import sso, forms
from mailu.ui import access

from flask import current_app as app
import flask
import flask_login

@sso.route('/login', methods=['GET', 'POST'])
def login():
    client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
    form = forms.LoginForm()
    form.submitAdmin.label.text = form.submitAdmin.label.text + ' Admin'
    form.submitWebmail.label.text = form.submitWebmail.label.text + ' Webmail'

    fields = []
    if str(app.config["ADMIN"]).upper() != "FALSE":
        fields.append(form.submitAdmin)
    if str(app.config["WEBMAIL"]).upper() != "NONE":
        fields.append(form.submitWebmail)

    if form.validate_on_submit():
        if form.submitAdmin.data:
            destination = app.config['WEB_ADMIN']
        elif form.submitWebmail.data:
            destination = app.config['WEB_WEBMAIL']
        device_cookie, device_cookie_username = utils.limiter.parse_device_cookie(flask.request.cookies.get('rate_limit'))
        username = form.email.data
        if username != device_cookie_username and utils.limiter.should_rate_limit_ip(client_ip):
            flask.flash('Too many attempts from your IP (rate-limit)', 'error')
            return flask.render_template('login.html', form=form)
        if utils.limiter.should_rate_limit_user(username, client_ip, device_cookie, device_cookie_username):
            flask.flash('Too many attempts for this user (rate-limit)', 'error')
            return flask.render_template('login.html', form=form)
        user = models.User.login(username, form.pw.data)
        if user:
            flask.session.regenerate()
            flask_login.login_user(user)
            response = flask.redirect(destination)
            response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=True, httponly=True)
            flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.')
            return response
        else:
            utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip)
            flask.current_app.logger.warn(f'Login failed for {username} from {client_ip}.')
            flask.flash('Wrong e-mail or password', 'error')
    return flask.render_template('login.html', form=form, fields=fields)

@sso.route('/logout', methods=['GET'])
@access.authenticated
def logout():
    flask_login.logout_user()
    flask.session.destroy()
    return flask.redirect(flask.url_for('.login'))

<|endoftext|>


<commit_before><commit_msg>sync-webkit-git: fixes to work on Windows<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

#  Copyright 2008 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Robot Framework Test Data Documentation Tool

Usage:  testdoc.py [options] test_data

This tool generates a high level test documentation from a given test data.
Generated documentation includes the names, documentations and other metadata
of each test suite and test case, as well as the top-level keywords and their
arguments. Most of the options accepted by this tool have exactly same
semantics as same options have when executing test cases.

Options:
  -o --output path       Where to write the generated documentation. If the
                         path is a directory, the documentation is
                         generated there using name '<suitename>-doc.html'.
  -T --title title       Set the title of the generated documentation.
                         Underscores in the title are converted to spaces.
  -N --name name         Set the name of the top level test suite.
  -D --doc document      Set the document of the top level test suite.
  -M --metadata name:value *  Set metadata of the top level test suite.
  -G --settag tag *      Set given tag(s) to all test cases.
  -t --test name *       Include test cases by name.
  -s --suite name *      Include test suites by name.
  -i --include tag *     Include test cases by tags.
  -e --exclude tag *     Exclude test cases by tags.
  -h --help              Print this help.

Examples:
  $ testdoc.py mytestcases.html
  $ testdoc.py --name smoke_test_plan --include smoke path/to/my_tests/

Note that this tool requires Robot Framework 2.0.3 or newer to be installed.
"""

import sys
import os
import time

from robot.common import BaseKeyword, BaseTestSuite
from robot.running import TestSuite, Keyword
from robot.conf import RobotSettings
from robot.output.abstractlogger import AbstractLogger
from robot.serializing.serializer import LogSuiteSerializer
from robot.serializing import templates
from robot.serializing.templating import Namespace, Template
from robot import utils
from robot.errors import DataError

class _FakeVariableScopes:

    def replace_from_meta(self, name, item, errors):
        return item

    def replace_string(self, item):
        return item

class _SystemLogger(AbstractLogger):

    def write(self, msg, level):
        pass

class MySerializer(LogSuiteSerializer):

    def __init__(self, output, suite):
        self._writer = utils.HtmlWriter(output)
        self._idgen = utils.IdGenerator()
        self._suite_level = 0

    def start_suite(self, suite):
        suite._init_suite(_FakeVariableScopes())
        LogSuiteSerializer.start_suite(self, suite)

    def start_test(self, test):
        test._init_test(_FakeVariableScopes())
        LogSuiteSerializer.start_test(self, test)

    def start_keyword(self, kw):
        if isinstance(kw, Keyword):  # Doesn't match For or Parallel
            kw.name = kw._get_name(kw.name, _FakeVariableScopes())
        LogSuiteSerializer.start_keyword(self, kw)

    def _is_element_open(self, item):
        return isinstance(item, BaseTestSuite)

    def _write_times(self, item):
        pass

    def _write_suite_metadata(self, suite):
        self._start_suite_or_test_metadata(suite)
        for name, value in suite.get_metadata(html=True):
            self._write_metadata_row(name, value, escape=False, write_empty=True)
        self._write_metadata_row('Number of Tests', suite.get_test_count())
        self._writer.end_element('table')

    def _write_test_metadata(self, test):
        self._start_suite_or_test_metadata(test)
        if test.timeout.secs < 0:
            tout = ''
        else:
            tout = utils.secs_to_timestr(test.timeout.secs)
            if test.timeout.message:
                tout += ' | ' + test.timeout.message
        self._write_metadata_row('Timeout', tout)
        self._write_metadata_row('Tags', ', '.join(test.tags))
        self._writer.end_element('table')

    def _write_folding_button(self, item):
        if not isinstance(item, BaseKeyword):
            LogSuiteSerializer._write_folding_button(self, item)

    def _write_expand_all(self, item):
        if isinstance(item, BaseTestSuite):
            LogSuiteSerializer._write_expand_all(self, item)

def generate_test_doc(args):
    opts, datasources = process_arguments(args)
    suite = TestSuite(datasources, RobotSettings(opts), _SystemLogger())
    outpath = get_outpath(opts['output'], suite.name)
    serialize_test_doc(suite, outpath, opts['title'])
    print outpath

def serialize_test_doc(suite, outpath, title):
    outfile = open(outpath, 'w')
    serializer = MySerializer(outfile, suite)
    ttuple = time.localtime()
    str_time = '%s %s' % (utils.format_time(ttuple, daytimesep=' '),
                          utils.get_diff_to_gmt())
    int_time = long(time.mktime(ttuple))
    if title:
        title = title.replace('_', ' ')
    else:
        title = 'Documentation for %s' % suite.name
    namespace = Namespace(gentime_str=str_time, gentime_int=int_time,
                          version=utils.get_full_version('testdoc.py'),
                          suite=suite, title=title)
    Template(template=templates.LOG).generate(namespace, outfile)
    suite.serialize(serializer)
    outfile.write('\n\n')
    outfile.close()

def process_arguments(args_list):
    argparser = utils.ArgumentParser(__doc__)
    try:
        opts, args = argparser.parse_args(args_list)
    except DataError, err:
        exit(error=str(err))
    if opts['help'] or not args:
        exit(msg=__doc__)
    return opts, args

def exit(msg=None, error=None):
    if msg:
        sys.stdout.write(msg + '\n')
    if error:
        sys.stderr.write(error + '\n\nTry --help for usage information.\n')
        sys.exit(1)
    sys.exit(0)

def get_outpath(path, suite_name):
    if not path:
        path = '.'
    if os.path.isdir(path):
        path = os.path.join(path, '%s-doc.html' % suite_name.replace(' ', '_'))
    return os.path.abspath(path)

if __name__ == '__main__':
    generate_test_doc(sys.argv[1:])
<commit_msg>renamed some helpers and moved classes around<commit_after>#!/usr/bin/env python

#  Copyright 2008 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

"""Robot Framework Test Data Documentation Tool

Usage:  testdoc.py [options] test_data

This tool generates a high level test documentation from a given test data.
Generated documentation includes the names, documentations and other metadata
of each test suite and test case, as well as the top-level keywords and their
arguments. Most of the options accepted by this tool have exactly same
semantics as same options have when executing test cases.

Options:
  -o --output path       Where to write the generated documentation. If the
                         path is a directory, the documentation is
                         generated there using name '<suitename>-doc.html'.
  -T --title title       Set the title of the generated documentation.
                         Underscores in the title are converted to spaces.
  -N --name name         Set the name of the top level test suite.
  -D --doc document      Set the document of the top level test suite.
  -M --metadata name:value *  Set metadata of the top level test suite.
  -G --settag tag *      Set given tag(s) to all test cases.
  -t --test name *       Include test cases by name.
  -s --suite name *      Include test suites by name.
  -i --include tag *     Include test cases by tags.
  -e --exclude tag *     Exclude test cases by tags.
  -h --help              Print this help.

Examples:
  $ testdoc.py mytestcases.html
  $ testdoc.py --name smoke_test_plan --include smoke path/to/my_tests/

Note that this tool requires Robot Framework 2.0.3 or newer to be installed.
"""

import sys
import os
import time

from robot.common import BaseKeyword, BaseTestSuite
from robot.running import TestSuite, Keyword
from robot.conf import RobotSettings
from robot.output.abstractlogger import AbstractLogger
from robot.serializing.serializer import LogSuiteSerializer
from robot.serializing import templates
from robot.serializing.templating import Namespace, Template
from robot import utils
from robot.errors import DataError

def generate_test_doc(args):
    opts, datasources = process_arguments(args)
    suite = TestSuite(datasources, RobotSettings(opts), SilentSystemLogger())
    outpath = get_outpath(opts['output'], suite.name)
    serialize_test_doc(suite, outpath, opts['title'])
    print outpath

def serialize_test_doc(suite, outpath, title):
    outfile = open(outpath, 'w')
    serializer = MySerializer(outfile, suite)
    ttuple = time.localtime()
    str_time = '%s %s' % (utils.format_time(ttuple, daytimesep=' '),
                          utils.get_diff_to_gmt())
    int_time = long(time.mktime(ttuple))
    if title:
        title = title.replace('_', ' ')
    else:
        title = 'Documentation for %s' % suite.name
    namespace = Namespace(gentime_str=str_time, gentime_int=int_time,
                          version=utils.get_full_version('testdoc.py'),
                          suite=suite, title=title)
    Template(template=templates.LOG).generate(namespace, outfile)
    suite.serialize(serializer)
    outfile.write('\n\n')
    outfile.close()

def process_arguments(args_list):
    argparser = utils.ArgumentParser(__doc__)
    try:
        opts, args = argparser.parse_args(args_list)
    except DataError, err:
        exit(error=str(err))
    if opts['help'] or not args:
        exit(msg=__doc__)
    return opts, args

def exit(msg=None, error=None):
    if msg:
        sys.stdout.write(msg + '\n')
    if error:
        sys.stderr.write(error + '\n\nTry --help for usage information.\n')
        sys.exit(1)
    sys.exit(0)

def get_outpath(path, suite_name):
    if not path:
        path = '.'
    if os.path.isdir(path):
        path = os.path.join(path, '%s-doc.html' % suite_name.replace(' ', '_'))
    return os.path.abspath(path)

class MySerializer(LogSuiteSerializer):

    def __init__(self, output, suite):
        self._writer = utils.HtmlWriter(output)
        self._idgen = utils.IdGenerator()
        self._suite_level = 0

    def start_suite(self, suite):
        suite._init_suite(NonResolvingVariableScopes())
        LogSuiteSerializer.start_suite(self, suite)

    def start_test(self, test):
        test._init_test(NonResolvingVariableScopes())
        LogSuiteSerializer.start_test(self, test)

    def start_keyword(self, kw):
        if isinstance(kw, Keyword):  # Doesn't match For or Parallel
            kw.name = kw._get_name(kw.name, NonResolvingVariableScopes())
        LogSuiteSerializer.start_keyword(self, kw)

    def _is_element_open(self, item):
        return isinstance(item, BaseTestSuite)

    def _write_times(self, item):
        pass

    def _write_suite_metadata(self, suite):
        self._start_suite_or_test_metadata(suite)
        for name, value in suite.get_metadata(html=True):
            self._write_metadata_row(name, value, escape=False, write_empty=True)
        self._write_metadata_row('Number of Tests', suite.get_test_count())
        self._writer.end_element('table')

    def _write_test_metadata(self, test):
        self._start_suite_or_test_metadata(test)
        if test.timeout.secs < 0:
            tout = ''
        else:
            tout = utils.secs_to_timestr(test.timeout.secs)
            if test.timeout.message:
                tout += ' | ' + test.timeout.message
        self._write_metadata_row('Timeout', tout)
        self._write_metadata_row('Tags', ', '.join(test.tags))
        self._writer.end_element('table')

    def _write_folding_button(self, item):
        if not isinstance(item, BaseKeyword):
            LogSuiteSerializer._write_folding_button(self, item)

    def _write_expand_all(self, item):
        if isinstance(item, BaseTestSuite):
            LogSuiteSerializer._write_expand_all(self, item)

class NonResolvingVariableScopes:

    def replace_from_meta(self, name, item, errors):
        return item

    def replace_string(self, item):
        return item

class SilentSystemLogger(AbstractLogger):

    def write(self, msg, level):
        pass

if __name__ == '__main__':
    generate_test_doc(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Extract messages from the source files, update all the .po files if needed,
and compile the .po files to produce .mo files.

This script first runs makemessages to produce an English .po file template.
If this new .po file has a different set of messages than the existing
English .po file, then this script updates all the .po files for the other
languages with the newly extracted messages, preserving the translations from
the existing files.

To minimize unnecessary changes from version to version, the resulting .po
files have no "#: filename:line" comments and the messages are sorted by msgid.

Usage:
    ../tools/update_messages

The arguments should be options to pass to the django "makemessages" command.
If no arguments are specified, -e '.py,.html,.txt,.template' is assumed.
"""

import datetime
import os
import StringIO
import sys
import re

from babel.messages import pofile
from django.core import management

LOCALE_DIR = os.path.join(os.environ['APP_DIR'], 'locale')

def print_flush(text):
    sys.stdout.write(text)
    sys.stdout.flush()

def get_po_filename(locale):
    return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES', 'django.po')

def read_po(file):
    """pofile.read_po() with a bug workaround.

    pofile.read_po() produces an incorrect string if either msgid or msgstr
    spans multiple lines and the first line content string is not empty e.g.
      msgid "hoge "
      "foo"
    This function workarounds this bug by converting this to:
      msgid ""
      "hoge "
      "foo"
    """
    converted = ''
    for line in file:
        line = line.rstrip('\n')
        m = re.search(r'^(msgid|msgstr) "(.+)"$', line)
        if m:
            converted += '%s ""\n' % m.group(1)
            converted += '"%s"\n' % m.group(2)
        else:
            converted += '%s\n' % line
    return pofile.read_po(StringIO.StringIO(converted))

def write_clean_po(filename, catalog):
    """Writes out a .po file in a canonical way, to minimize spurious diffs."""
    catalog.creation_date = datetime.datetime(2000, 1, 1, 0, 0, 0)
    file = open(filename, 'w')
    pofile.write_po(file, catalog,
                    no_location=True, sort_output=True, ignore_obsolete=True)
    file.close()

def update_po(locale, new_msgids, header_comment):
    """Updates a .po file to contain the given set of messages and the given
    header comment, and returns the number of missing translations."""
    filename = get_po_filename(locale)
    translations = read_po(open(filename))

    # Remove unused messages.
    for message in translations:
        if message.id and message.id not in new_msgids:
            del translations[message.id]

    # Add new messages.
    for id in new_msgids:
        if id and id not in translations:
            translations.add(id, '')

    # Fix cases where message.id starts with '\n' but message.string doesn't.
    # django_admin('compilemessages') fails with an error for such cases.
    for message in translations:
        if (message.id.startswith(u'\n') and
                message.string and
                not message.string.startswith(u'\n')):
            message.string = u'\n' + message.string

    translations.header_comment = header_comment
    write_clean_po(filename, translations)
    return len([message for message in translations
                if message.fuzzy or not message.string])

def django_admin(*args):
    """Executes a django admin tool from the app directory."""
    cwd = os.getcwd()
    os.chdir(os.environ['APP_DIR'])
    management.execute_from_command_line(['django-admin.py'] + list(args))
    os.chdir(cwd)

if __name__ == '__main__':
    args = sys.argv[1:]
    if '-h' in args or '--help' in args:
        print __doc__
        sys.exit(1)
    force = '-f' in args or '--force' in args
    makemessages_args = [arg for arg in args if arg not in ['-f', '--force']]
    if not makemessages_args:
        makemessages_args = ['-e', '.py,.html,.txt,.template']

    # Get the set of messages in the existing English .po file.
    en_filename = get_po_filename('en')
    old_template = read_po(open(en_filename))
    old_msgids = set(message.id for message in old_template)

    # Run makemessages to update the English .po file.
    django_admin('makemessages', '-l', 'en', *makemessages_args)

    # Get the set of messages in the new English .po file.
    new_template = read_po(open(en_filename))
    new_msgids = set(message.id for message in new_template)

    # Write the English .po file in sorted order without file references.
    write_clean_po(en_filename, new_template)

    # Update the other .po files if necessary.
    if new_msgids != old_msgids or force:
        print '%d messages added, %d removed.' % (
            len(new_msgids - old_msgids), len(old_msgids - new_msgids))
        print_flush('Updating:')
        missing = {}
        for locale in sorted(os.listdir(LOCALE_DIR)):
            if locale != 'en':
                print_flush(' ' + locale)
                missing[locale] = update_po(
                    locale, new_msgids, new_template.header_comment)
        print
        for locale in sorted(missing):
            print '%s: %d missing' % (locale, missing[locale])
    else:
        print 'No messages have changed.'

    # Run compilemessages to generate all the .mo files.
    sys.stderr = open('/dev/null', 'w')  # suppress the listing of all files
    django_admin('compilemessages')
<commit_msg>Fix line break inconsistencies in update_messages to suppress errors.<commit_after>#!/usr/bin/env python
# Copyright 2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Extract messages from the source files, update all the .po files if needed,
and compile the .po files to produce .mo files.

This script first runs makemessages to produce an English .po file template.
If this new .po file has a different set of messages than the existing
English .po file, then this script updates all the .po files for the other
languages with the newly extracted messages, preserving the translations from
the existing files.

To minimize unnecessary changes from version to version, the resulting .po
files have no "#: filename:line" comments and the messages are sorted by msgid.

Usage:
    ../tools/update_messages

The arguments should be options to pass to the django "makemessages" command.
If no arguments are specified, -e '.py,.html,.txt,.template' is assumed.
"""

import datetime
import os
import StringIO
import sys
import re

from babel.messages import pofile
from django.core import management

LOCALE_DIR = os.path.join(os.environ['APP_DIR'], 'locale')

def print_flush(text):
    sys.stdout.write(text)
    sys.stdout.flush()

def get_po_filename(locale):
    return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES', 'django.po')

def read_po(file):
    """pofile.read_po() with a bug workaround.

    pofile.read_po() produces an incorrect string if either msgid or msgstr
    spans multiple lines and the first line content string is not empty e.g.
      msgid "hoge "
      "foo"
    This function workarounds this bug by converting this to:
      msgid ""
      "hoge "
      "foo"
    """
    converted = ''
    for line in file:
        line = line.rstrip('\n')
        m = re.search(r'^(msgid|msgstr) "(.+)"$', line)
        if m:
            converted += '%s ""\n' % m.group(1)
            converted += '"%s"\n' % m.group(2)
        else:
            converted += '%s\n' % line
    return pofile.read_po(StringIO.StringIO(converted))

def write_clean_po(filename, catalog):
    """Writes out a .po file in a canonical way, to minimize spurious diffs."""
    catalog.creation_date = datetime.datetime(2000, 1, 1, 0, 0, 0)
    file = open(filename, 'w')
    pofile.write_po(file, catalog,
                    no_location=True, sort_output=True, ignore_obsolete=True)
    file.close()

def update_po(locale, new_msgids, header_comment):
    """Updates a .po file to contain the given set of messages and the given
    header comment, and returns the number of missing translations."""
    filename = get_po_filename(locale)
    translations = read_po(open(filename))

    # Remove unused messages.
    for message in translations:
        if message.id and message.id not in new_msgids:
            del translations[message.id]

    # Add new messages.
    for id in new_msgids:
        if id and id not in translations:
            translations.add(id, '')

    # django_admin('compilemessages') fails if message.id starts/ends with '\n'
    # but message.string doesn't, or vice versa.
    # Fix message.string to make it consistent.
    for message in translations:
        if not message.string: continue
        if (message.id.startswith(u'\n') and
                not message.string.startswith(u'\n')):
            message.string = u'\n' + message.string
        if (not message.id.startswith(u'\n') and
                message.string.startswith(u'\n')):
            message.string = message.string.lstrip('\n')
        if (message.id.endswith(u'\n') and
                not message.string.endswith(u'\n')):
            message.string = message.string + u'\n'
        if (not message.id.endswith(u'\n') and
                message.string.endswith(u'\n')):
            message.string = message.string.rstrip('\n')

    translations.header_comment = header_comment
    write_clean_po(filename, translations)
    return len([message for message in translations
                if message.fuzzy or not message.string])

def django_admin(*args):
    """Executes a django admin tool from the app directory."""
    cwd = os.getcwd()
    os.chdir(os.environ['APP_DIR'])
    management.execute_from_command_line(['django-admin.py'] + list(args))
    os.chdir(cwd)

if __name__ == '__main__':
    args = sys.argv[1:]
    if '-h' in args or '--help' in args:
        print __doc__
        sys.exit(1)
    force = '-f' in args or '--force' in args
    makemessages_args = [arg for arg in args if arg not in ['-f', '--force']]
    if not makemessages_args:
        makemessages_args = ['-e', '.py,.html,.txt,.template']

    # Get the set of messages in the existing English .po file.
    en_filename = get_po_filename('en')
    old_template = read_po(open(en_filename))
    old_msgids = set(message.id for message in old_template)

    # Run makemessages to update the English .po file.
    django_admin('makemessages', '-l', 'en', *makemessages_args)

    # Get the set of messages in the new English .po file.
    new_template = read_po(open(en_filename))
    new_msgids = set(message.id for message in new_template)

    # Write the English .po file in sorted order without file references.
    write_clean_po(en_filename, new_template)

    # Update the other .po files if necessary.
    if new_msgids != old_msgids or force:
        print '%d messages added, %d removed.' % (
            len(new_msgids - old_msgids), len(old_msgids - new_msgids))
        print_flush('Updating:')
        missing = {}
        for locale in sorted(os.listdir(LOCALE_DIR)):
            if locale != 'en':
                print_flush(' ' + locale)
                missing[locale] = update_po(
                    locale, new_msgids, new_template.header_comment)
        print
        for locale in sorted(missing):
            print '%s: %d missing' % (locale, missing[locale])
    else:
        print 'No messages have changed.'

    # Run compilemessages to generate all the .mo files.
    sys.stderr = open('/dev/null', 'w')  # suppress the listing of all files
    django_admin('compilemessages')
<|endoftext|>