<commit_before>import datetime
import logging
import random
import time
import traceback
from apscheduler.triggers.cron import CronTrigger

class ScheduleMixin(object):

    def times_key(self, periodic_list=False):
        if periodic_list:
            return "will_periodic_times_list"

        return "will_schedule_times_list"

    def schedule_key(self, periodic_list=False):
        if periodic_list:
            return "will_periodic_list"

        return "will_schedule_list"

    def get_schedule_list(self, periodic_list=False):
        # TODO: Clean this up.
        return self.load(self.schedule_key(periodic_list=periodic_list), [])

    def save_schedule_list(self, new_list, periodic_list=False):
        print "save_schedule_list for (%s)" % periodic_list
        print new_list

        self.save(self.schedule_key(periodic_list=periodic_list), new_list)
        print "Saved"
        print self.get_schedule_list(periodic_list=periodic_list)

    def get_times_list(self, periodic_list=False):
        # TODO: Clean this up.
        return self.load(self.times_key(periodic_list=periodic_list), [])

    def save_times_list(self, new_list, periodic_list=False):
        return self.save(self.times_key(periodic_list=periodic_list), new_list)

    def add_direct_message_to_schedule(self, when, content, message, *args, **kwargs):
        target_user = self.get_user_from_message(message)
        self.add_to_schedule(when, {
            "type": "direct_message",
            "content": content,
            "target_jid": target_user["jid"],
            "args": args,
            "kwargs": kwargs,
        })

    def add_room_message_to_schedule(self, when, content, room, *args, **kwargs):
        self.add_to_schedule(when, {
            "type": "room_message",
            "content": content,
            "room": room,
            "args": args,
            "kwargs": kwargs,
        })

    def add_to_schedule(self, when, item, periodic_list=False, ignore_scheduler_lock=False):
        try:
            while (ignore_scheduler_lock == False and self.load("scheduler_lock", False)) or self.load("scheduler_add_lock", False):
                import sys
                sys.stdout.write("waiting for lock to clear\n")
                time.sleep(random.random())

            self.save("scheduler_add_lock", True)
            sched_list = self.get_schedule_list(periodic_list=periodic_list)
            print "adding: "
            print periodic_list
            print item
            print sched_list
            times_list = self.get_times_list(periodic_list=periodic_list)
            item["when"] = when
            sched_list.append(item)
            times_list.append(when)
            self.save_schedule_list(sched_list, periodic_list=periodic_list)
            self.save_times_list(times_list, periodic_list=periodic_list)

        except:
            logging.critical("Error adding to schedule at %s.  \n\n%s\nContinuing...\n" % (when, traceback.format_exc() ))
        self.save("scheduler_add_lock", False)

    def remove_from_schedule(self, index, periodic_list=False):
        # If this is ever called from anywhere outside the scheduler_lock, it needs its own lock.
        sched_list = self.get_schedule_list(periodic_list=periodic_list)
        times_list = self.get_times_list(periodic_list=periodic_list)
        del sched_list[index]
        del times_list[index]
        self.save_schedule_list(sched_list, periodic_list=periodic_list)
        self.save_times_list(times_list, periodic_list=periodic_list)

    def add_periodic_task(self, cls, sched_args, sched_kwargs, fn, ignore_scheduler_lock=False):
        now = datetime.datetime.now()
        ct = CronTrigger(*sched_args, **sched_kwargs)
        when = ct.get_next_fire_time(now)
        item = {
            "type": "periodic_task",
            "class": cls,
            "function": fn,
            "sched_args": sched_args,
            "sched_kwargs": sched_kwargs,
        }
        print "add_periodic_task"
        print when
        print item
        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_periodic_task_directly(self, item, ignore_scheduler_lock=False):
        now = datetime.datetime.now()
        ct = CronTrigger(item["sched_args"], item["sched_kwargs"])
        when = ct.get_next_fire_time(now)
        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_single_random_task(self, when, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=False):
        item = {
            "type": "random_task",
            "class": cls,
            "function": fn,
            "start_hour": start_hour,
            "end_hour": end_hour,
            "day_of_week": day_of_week,
            "num_times_per_day": num_times_per_day,
        }

        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_random_tasks(self, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=False):
        # This function is fired at startup, and every day at midnight.

        if end_hour < start_hour:
            raise Exception("start_hour is after end_hour!")

        # Get the next appropriate date
        now = datetime.datetime.now()

        # Work around crontab bug where if the hour has started, it's skipped.
        adjusted_start_hour = start_hour
        if adjusted_start_hour != 23:
            adjusted_start_hour += 1
        adjusted_start_hour = "%s" % adjusted_start_hour

        ct = CronTrigger(hour=adjusted_start_hour, day_of_week=day_of_week)
        fire_time = ct.get_next_fire_time(now)

        # If it's today, schedule it. Otherwise, it'll be scheduled at midnight of its run day.
        if fire_time.day == now.day:

            # There are more efficient ways to do this, but this supports almost any n for num_times_per_day,
            # and that seems more useful.
            possible_times = []
            for i in range(start_hour, end_hour):
                for j in range(0, 59):
                    possible_times.append((i,j))

            times = random.sample(possible_times, num_times_per_day)
            for hour, minute in times:
                when = datetime.datetime(now.year, now.month, now.day, hour, minute)

                # If we're starting up mid-day, this may not be true.
                if when >= now:
                    self.add_single_random_task(when, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=ignore_scheduler_lock)
<commit_msg>* and **<commit_after>import datetime
import logging
import random
import time
import traceback
from apscheduler.triggers.cron import CronTrigger

class ScheduleMixin(object):

    def times_key(self, periodic_list=False):
        if periodic_list:
            return "will_periodic_times_list"

        return "will_schedule_times_list"

    def schedule_key(self, periodic_list=False):
        if periodic_list:
            return "will_periodic_list"

        return "will_schedule_list"

    def get_schedule_list(self, periodic_list=False):
        # TODO: Clean this up.
        return self.load(self.schedule_key(periodic_list=periodic_list), [])

    def save_schedule_list(self, new_list, periodic_list=False):
        print "save_schedule_list for (%s)" % periodic_list
        print new_list

        self.save(self.schedule_key(periodic_list=periodic_list), new_list)
        print "Saved"
        print self.get_schedule_list(periodic_list=periodic_list)

    def get_times_list(self, periodic_list=False):
        # TODO: Clean this up.
        return self.load(self.times_key(periodic_list=periodic_list), [])

    def save_times_list(self, new_list, periodic_list=False):
        return self.save(self.times_key(periodic_list=periodic_list), new_list)

    def add_direct_message_to_schedule(self, when, content, message, *args, **kwargs):
        target_user = self.get_user_from_message(message)
        self.add_to_schedule(when, {
            "type": "direct_message",
            "content": content,
            "target_jid": target_user["jid"],
            "args": args,
            "kwargs": kwargs,
        })

    def add_room_message_to_schedule(self, when, content, room, *args, **kwargs):
        self.add_to_schedule(when, {
            "type": "room_message",
            "content": content,
            "room": room,
            "args": args,
            "kwargs": kwargs,
        })

    def add_to_schedule(self, when, item, periodic_list=False, ignore_scheduler_lock=False):
        try:
            while (ignore_scheduler_lock == False and self.load("scheduler_lock", False)) or self.load("scheduler_add_lock", False):
                import sys
                sys.stdout.write("waiting for lock to clear\n")
                time.sleep(random.random())

            self.save("scheduler_add_lock", True)
            sched_list = self.get_schedule_list(periodic_list=periodic_list)
            print "adding: "
            print periodic_list
            print item
            print sched_list
            times_list = self.get_times_list(periodic_list=periodic_list)
            item["when"] = when
            sched_list.append(item)
            times_list.append(when)
            self.save_schedule_list(sched_list, periodic_list=periodic_list)
            self.save_times_list(times_list, periodic_list=periodic_list)

        except:
            logging.critical("Error adding to schedule at %s.  \n\n%s\nContinuing...\n" % (when, traceback.format_exc() ))
        self.save("scheduler_add_lock", False)

    def remove_from_schedule(self, index, periodic_list=False):
        # If this is ever called from anywhere outside the scheduler_lock, it needs its own lock.
        sched_list = self.get_schedule_list(periodic_list=periodic_list)
        times_list = self.get_times_list(periodic_list=periodic_list)
        del sched_list[index]
        del times_list[index]
        self.save_schedule_list(sched_list, periodic_list=periodic_list)
        self.save_times_list(times_list, periodic_list=periodic_list)

    def add_periodic_task(self, cls, sched_args, sched_kwargs, fn, ignore_scheduler_lock=False):
        now = datetime.datetime.now()
        ct = CronTrigger(*sched_args, **sched_kwargs)
        when = ct.get_next_fire_time(now)
        item = {
            "type": "periodic_task",
            "class": cls,
            "function": fn,
            "sched_args": sched_args,
            "sched_kwargs": sched_kwargs,
        }
        print "add_periodic_task"
        print when
        print item
        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_periodic_task_directly(self, item, ignore_scheduler_lock=False):
        now = datetime.datetime.now()
        ct = CronTrigger(*item["sched_args"], **item["sched_kwargs"])
        when = ct.get_next_fire_time(now)
        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_single_random_task(self, when, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=False):
        item = {
            "type": "random_task",
            "class": cls,
            "function": fn,
            "start_hour": start_hour,
            "end_hour": end_hour,
            "day_of_week": day_of_week,
            "num_times_per_day": num_times_per_day,
        }

        self.add_to_schedule(when, item, periodic_list=True, ignore_scheduler_lock=ignore_scheduler_lock)

    def add_random_tasks(self, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=False):
        # This function is fired at startup, and every day at midnight.

        if end_hour < start_hour:
            raise Exception("start_hour is after end_hour!")

        # Get the next appropriate date
        now = datetime.datetime.now()

        # Work around crontab bug where if the hour has started, it's skipped.
        adjusted_start_hour = start_hour
        if adjusted_start_hour != 23:
            adjusted_start_hour += 1
        adjusted_start_hour = "%s" % adjusted_start_hour

        ct = CronTrigger(hour=adjusted_start_hour, day_of_week=day_of_week)
        fire_time = ct.get_next_fire_time(now)

        # If it's today, schedule it. Otherwise, it'll be scheduled at midnight of its run day.
        if fire_time.day == now.day:

            # There are more efficient ways to do this, but this supports almost any n for num_times_per_day,
            # and that seems more useful.
            possible_times = []
            for i in range(start_hour, end_hour):
                for j in range(0, 59):
                    possible_times.append((i,j))

            times = random.sample(possible_times, num_times_per_day)
            for hour, minute in times:
                when = datetime.datetime(now.year, now.month, now.day, hour, minute)

                # If we're starting up mid-day, this may not be true.
                if when >= now:
                    self.add_single_random_task(when, cls, fn, start_hour, end_hour, day_of_week, num_times_per_day, ignore_scheduler_lock=ignore_scheduler_lock)
<|endoftext|>


<commit_before>import urllib.parse
from bs4 import BeautifulSoup
import re
import os
import syscmd
import sysErrorLog
import sqlite3

def fmi( self ):

	if len(self.msg) == 5 and "set" in self.msg[4]:
		self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
		return
	## For saving the city
	if len(self.msg) == 6:
		if "set" in self.msg:
			city = self.msg[5]
			setCity( self, city )
			self.send_notice( "Your city {0} has been saved!".format(city.title().strip()))
		else:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
	else:
		try:
			if len(self.msg) == 4:	## when called only with !fmi, see if the city is saved
				city = getCity( self )
				if not city and len(self.msg) == 5:
					city = self.msg[4]
			if len(self.msg) == 5:
				city = self.msg[4].strip() #With !fmi City
			## If no city saved and no parameter
			if not city:
				self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
				return
		except IndexError:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
			raise

		if "set" not in city and syscmd.checkCity( self, city ): # We don't want to look for a city named "set"

			city = city.title().strip()
			parameter = urllib.parse.quote(city)
			url = "http://ilmatieteenlaitos.fi/saa/" + parameter

			if city == "Oulu":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=101799"
			if city == "Helsinki":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=100971"
			if city == "Kilpisjärvi":
				url = "http://ilmatieteenlaitos.fi/saa/enonteki%C3%B6/" + parameter

			html = syscmd.getHtml(self, url, True )

			try:
				soup = BeautifulSoup(html)
				text = ""
				if "Varasivu" in soup.title.string:
					self.send_chan("~ Ilmatieteen laitoksen sivustolla on ongelmia - http://vara.fmi.fi/")
				else:
					## When the weather was updated
					time = soup.find_all("span", class_="time-stamp")
					if time:
						time = time[0].string.split(" ")
						time = time[1][:-7]

					string = soup.findAll("span", {"class" : "parameter-name-value"})
					feels = soup.findAll("div", {"class" : "apparent-temperature-indifferent"})
					rain = soup.findAll("tr", {"class" : "meteogram-probabilities-of-precipitation"})
					rainAmount = soup.findAll("tr", {"class" : "meteogram-hourly-precipitation-values"})
					warnings = soup.findAll("span", {"class" : "advisory-type"})

					## If FMI happens to return the feel temperatures..
					if feels:
					 feels = "{0}".format(feels[0].get('title'))
					else:
						feels = "0°C"

					## Loop the reusts into a string
					for index, element in enumerate(string):
						if index == 1 and rainAmount and feels and rain:
							text += "{0}{1} - ".format(feels[4:-1].capitalize(),"C")
							text += "Sateen todennäköisyys {0} ({1} mm) - ".format(rain[0].td.div.span.string, rainAmount[0].td.span.strong.string)

						text += "{0} - ".format(element)
					text = text[:-2]

					## See if there's warnings
					if warnings:
						text += "- Varoitus: {0}".format(warnings[0].string)

					## Remove the Html tags
					if text:
						trimmed = syscmd.delHtml(text)
						output = "{0} klo {1}: {2}".format(city.strip(), time, trimmed)
						self.send_chan( output )
					else:
						self.send_chan("An error occured getting the weather data")
			except Exception as e:
				self.errormsg = "[ERROR]-[fmi] fmi() stating: {0}".format(e)
				sysErrorLog.log( self ) ## LOG the error
				if self.config["debug"]:
					print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		else:
			self.send_chan( "City {0} doesn't exist!".format(city.title().strip()) )

## Return saved city for the nick
def getCity ( self ):

	nick = self.get_nick()
	try:
		db = sqlite3.connect("modules/data/fmiCities.db")

		cursor = db.cursor()

		## Check if the city is saved in the db
		cursor.execute("""
				SELECT city FROM nicks WHERE nick=?
				COLLATE NOCASE """, (nick.strip(),))
		try:
			city = cursor.fetchone()[0]
		except TypeError:
			city = None
		return(city)
	except Exception as e:
		db.rollback()
		self.errormsg = "[ERROR]-[fmi] getCity() stating: {0}".format(e)
		sysErrorLog.log() ## LOG the error
		if self.config["debug"]:
			print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		raise e
	finally:
		db.close()

## Save user city
def setCity ( self, city ):

	nick = self.get_nick()
	city = city.title().strip()

	try:
		db = sqlite3.connect("modules/data/fmiCities.db")

		cursor = db.cursor()

		## Check if nick is in the db
		cursor.execute("""
				SELECT id FROM nicks WHERE nick=?
				""", (nick.strip(),))
		try:
			nickId = cursor.fetchone()[0]
		except TypeError:
			nickId = None
		## If a nick is found, update the row
		if nickId:
			cursor.execute("""
					UPDATE nicks SET city=? WHERE id=?
					""", (city, nickId))
			db.commit()
			return(True)
		## If not found, insert it
		else:
			cursor.execute("""
					INSERT INTO nicks(nick, city) VALUES(?, ?)
					""", (nick.strip(), city))
			db.commit()
			return(True)
	except Exception as e:
		db.rollback()
		self.errormsg = "[ERROR]-[fmi] setCity() stating: {0}".format(e)
		sysErrorLog.log() ## LOG the error
		if self.config["debug"]:
			print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		raise e
	finally:
		db.close()

<commit_msg>Maybe this works better<commit_after>import urllib.parse
from bs4 import BeautifulSoup
import re
import os
import syscmd
import sysErrorLog
import sqlite3

def fmi( self ):

	if len(self.msg) == 5 and "set" in self.msg[4]:
		self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
		return
	## For saving the city
	if len(self.msg) == 6:
		if "set" in self.msg:
			city = self.msg[5]
			setCity( self, city )
			self.send_notice( "Your city {0} has been saved!".format(city.title().strip()))
		else:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
	else:
		try:
			if len(self.msg) == 4:	## when called only with !fmi, see if the city is saved
				city = getCity( self )
				if not city and len(self.msg) == 5:
					city = self.msg[4]
			if len(self.msg) == 5:
				city = self.msg[4].strip() #With !fmi City
			## If no city saved and no parameter
			if not city:
				self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
				return
		except IndexError:
			self.send_chan( "Usage: !fmi <city> | !fmi set <city>" )
			raise

		if "set" not in city and syscmd.checkCity( self, city ): # We don't want to look for a city named "set"

			city = city.title().strip()
			parameter = urllib.parse.quote(city)
			url = "http://ilmatieteenlaitos.fi/saa/" + parameter

			if city == "Oulu":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=101799"
			if city == "Helsinki":
				url = "http://ilmatieteenlaitos.fi/saa/" + parameter + "?&station=100971"
			if city == "Kilpisjärvi":
				url = "http://ilmatieteenlaitos.fi/saa/enonteki%C3%B6/" + parameter

			html = syscmd.getHtml(self, url, True )

			try:
				soup = BeautifulSoup(html)
				text = ""
				if "Varasivu" in soup.title.string:
					self.send_chan("~ Ilmatieteen laitoksen sivustolla on ongelmia - http://vara.fmi.fi/")
				else:
					## When the weather was updated
					time = soup.find_all("span", class_="time-stamp")
					if time:
						time = time[0].string.split(" ")
						time = time[1][:-7]

					string = soup.findAll("span", {"class" : "parameter-name-value"})
					feels = soup.findAll("tr", {"class" : "meteogram-apparent-temperatures"})
					rain = soup.findAll("tr", {"class" : "meteogram-probabilities-of-precipitation"})
					rainAmount = soup.findAll("tr", {"class" : "meteogram-hourly-precipitation-values"})
					warnings = soup.findAll("span", {"class" : "advisory-type"})

					## If FMI happens to return the feel temperatures..
					if feels:
					 feels = "{0}".format(feels[0].td.div.get('title'))
					else:
						feels = "0°C"

					## Loop the reusts into a string
					for index, element in enumerate(string):
						if index == 1 and rainAmount and feels and rain:
							text += "{0}{1} - ".format(feels[4:-1].capitalize(),"C")
							text += "Sateen todennäköisyys {0} ({1} mm) - ".format(rain[0].td.div.span.string, rainAmount[0].td.span.strong.string)

						text += "{0} - ".format(element)
					text = text[:-2]

					## See if there's warnings
					if warnings:
						text += "- Varoitus: {0}".format(warnings[0].string)

					## Remove the Html tags
					if text:
						trimmed = syscmd.delHtml(text)
						output = "{0} klo {1}: {2}".format(city.strip(), time, trimmed)
						self.send_chan( output )
					else:
						self.send_chan("An error occured getting the weather data")
			except Exception as e:
				self.errormsg = "[ERROR]-[fmi] fmi() stating: {0}".format(e)
				sysErrorLog.log( self ) ## LOG the error
				if self.config["debug"]:
					print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		else:
			self.send_chan( "City {0} doesn't exist!".format(city.title().strip()) )

## Return saved city for the nick
def getCity ( self ):

	nick = self.get_nick()
	try:
		db = sqlite3.connect("modules/data/fmiCities.db")

		cursor = db.cursor()

		## Check if the city is saved in the db
		cursor.execute("""
				SELECT city FROM nicks WHERE nick=?
				COLLATE NOCASE """, (nick.strip(),))
		try:
			city = cursor.fetchone()[0]
		except TypeError:
			city = None
		return(city)
	except Exception as e:
		db.rollback()
		self.errormsg = "[ERROR]-[fmi] getCity() stating: {0}".format(e)
		sysErrorLog.log() ## LOG the error
		if self.config["debug"]:
			print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		raise e
	finally:
		db.close()

## Save user city
def setCity ( self, city ):

	nick = self.get_nick()
	city = city.title().strip()

	try:
		db = sqlite3.connect("modules/data/fmiCities.db")

		cursor = db.cursor()

		## Check if nick is in the db
		cursor.execute("""
				SELECT id FROM nicks WHERE nick=?
				""", (nick.strip(),))
		try:
			nickId = cursor.fetchone()[0]
		except TypeError:
			nickId = None
		## If a nick is found, update the row
		if nickId:
			cursor.execute("""
					UPDATE nicks SET city=? WHERE id=?
					""", (city, nickId))
			db.commit()
			return(True)
		## If not found, insert it
		else:
			cursor.execute("""
					INSERT INTO nicks(nick, city) VALUES(?, ?)
					""", (nick.strip(), city))
			db.commit()
			return(True)
	except Exception as e:
		db.rollback()
		self.errormsg = "[ERROR]-[fmi] setCity() stating: {0}".format(e)
		sysErrorLog.log() ## LOG the error
		if self.config["debug"]:
			print("{0}{1}{2}".format(self.color("red"), self.errormsg, self.color("end")))
		raise e
	finally:
		db.close()

<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

## Note: This file is very trim now because we've broken wsgi_fileserver, wsgi_proxy, wsgi_jsonrpc, and wsgi_xmlrpc
## in to their own libraries which are now distributed on their own and treated as dependencies

import httplib
import copy
import socket
import random
import os
from urlparse import urlparse
import logging
import threading
from time import sleep
logger = logging.getLogger(__name__)

import windmill
from windmill.server import proxy
import wsgi_jsonrpc
import wsgi_xmlrpc
import wsgi_fileserver

import jsmin

START_DST_PORT = 32000
CURRENT_DST_PORT = [random.randint(32000, 34000)]

def reconstruct_url(environ):
    # From WSGI spec, PEP 333
    from urllib import quote
    url = environ['wsgi.url_scheme']+'://'
    if environ.get('HTTP_HOST'): url += environ['HTTP_HOST']
    else:
        url += environ['SERVER_NAME']
        if environ['wsgi.url_scheme'] == 'https':
            if environ['SERVER_PORT'] != '443':
               url += ':' + environ['SERVER_PORT']
        else:
            if environ['SERVER_PORT'] != '80':
               url += ':' + environ['SERVER_PORT']
    url += environ.get('SCRIPT_NAME','')
    url += environ.get('PATH_INFO','')
    # Fix ;arg=value in url
    if url.find('%3B') is not -1:
        url, arg = url.split('%3B', 1)
        url = ';'.join([url, arg.replace('%3D', '=')])
    # Stick query string back in
    if environ.get('QUERY_STRING'):
        url += '?' + environ['QUERY_STRING']
    # Stick it in environ for convenience
    environ['reconstructed_url'] = url
    print url
    return url

HTTPConnection = httplib.HTTPConnection

WindmillProxyApplication = proxy.WindmillProxyApplication
WindmillProxyApplication.ConnectionClass = HTTPConnection

add_namespace = None

class WindmillChooserApplication(object):
    """Application to handle choosing the proper application to handle each request"""
    def __init__(self, apps, proxy):
        self.namespaces = dict([ (arg.ns, arg) for arg in apps ])
        self.proxy = proxy

    def add_namespace(self, name, application):
        """Add an application to a specific url namespace in windmill"""
        self.namespaces[name] = application

    def handler(self, environ, start_response):
        """Windmill app chooser"""
        sleep(.2)
        reconstruct_url(environ)

        for key in self.namespaces.keys():
            if environ['PATH_INFO'].find('/'+key+'/') is not -1:
                logger.debug('dispatching request %s to %s' % (environ['reconstructed_url'], key))
                return self.namespaces[key](environ, start_response)

        logger.debug('dispatching request %s to WindmillProxyApplication' % reconstruct_url(environ))
        response = self.proxy(environ, start_response)
        return response

    def __call__(self, environ, start_response):
        response = self.handler(environ, start_response)
        for x in response:
            yield x

class WindmillCompressor(object):
    """Full JavaScript Compression Library"""
    js_file_list = [
        ('lib', 'fleegix.js',),
        ('lib', 'hash.js',),
        ('lib', 'firebug', 'pi.js',),
        ('lib', 'firebug', 'firebug-lite.js',),
        ('lib', 'json2.js',),
        ('lib', 'browserdetect.js',),
        ('wm', 'windmill.js',),
        ('lib', 'getXPath.js',),
        ('lib', 'elementslib.js',),
        ('lib', 'js-xpath.js',),
        ('controller', 'controller.js',),
        ('controller', 'commands.js',),
        ('controller', 'asserts.js',),
        ('controller', 'waits.js',),
        ('extensions', 'extensions.js',),
        ('wm', 'registry.js',),
        ('wm', 'utils.js',),
        ('wm', 'ide', 'ui.js',),
        ('wm', 'ide', 'recorder.js',),
        ('wm', 'ide', 'remote.js',),
        ('wm', 'ide', 'dx.js',),
        ('wm', 'ide', 'ax.js',),
        ('wm', 'ide', 'results.js',),
        ('wm', 'xhr.js',),
        ('wm', 'metrics.js',),
        ('wm', 'events.js',),
        ('wm', 'global.js',),
        ('wm', 'jstest.js',),
        ('wm', 'load.js',),
    ]

    def __init__(self, js_path, enabled=True):
        self.enabled = enabled
        self.js_path = js_path
        self.compressed_windmill = None
        if enabled:
            self._thread = threading.Thread(target=self.compress_file)
            self._thread.start()

    def compress_file(self):
        compressed_windmill = ''
        for filename in self.js_file_list:
            compressed_windmill += jsmin.jsmin(open(os.path.join(self.js_path, *filename), 'r').read())
        self.compressed_windmill = compressed_windmill

    def __call__(self, environ, start_response):
        if not self.enabled:
            start_response('404 Not Found', [('Content-Type', 'text/plain',), ('Content-Length', '0',)])
            return ['']
        # if self.compressed_windmill is None:
        #     self.compressed_windmill = ''
        #     for filename in self.js_file_list:
        #         self.compressed_windmill += jsmin.jsmin(open(os.path.join(self.js_path, *filename), 'r').read())

        while not self.compressed_windmill:
            sleep(.15)

        start_response('200 Ok', [('Content-Type', 'application/x-javascript',),
                                  ('Content-Length', str(len(self.compressed_windmill)),)])
        return [self.compressed_windmill]

def make_windmill_server(http_port=None, js_path=None, compression_enabled=None):
    if http_port is None:
        http_port = windmill.settings['SERVER_HTTP_PORT']
    if js_path is None:
        js_path = windmill.settings['JS_PATH']
    if compression_enabled is None:
        compression_enabled = not windmill.settings['DISABLE_JS_COMPRESS']

    # Start up all the convergence objects
    import convergence
    test_resolution_suite = convergence.TestResolutionSuite()
    command_resolution_suite = convergence.CommandResolutionSuite()
    queue = convergence.ControllerQueue(command_resolution_suite, test_resolution_suite)
    xmlrpc_methods_instance = convergence.XMLRPCMethods(queue, test_resolution_suite, command_resolution_suite)
    jsonrpc_methods_instance = convergence.JSONRPCMethods(queue, test_resolution_suite, command_resolution_suite)

    # Start up all the wsgi applications
    windmill_serv_app = wsgi_fileserver.WSGIFileServerApplication(root_path=js_path, mount_point='/windmill-serv/')
    windmill_proxy_app = WindmillProxyApplication()
    windmill_xmlrpc_app =  wsgi_xmlrpc.WSGIXMLRPCApplication(instance=xmlrpc_methods_instance)
    windmill_jsonrpc_app = wsgi_jsonrpc.WSGIJSONRPCApplication(instance=jsonrpc_methods_instance)
    windmill_compressor_app = WindmillCompressor(os.path.join(js_path, 'js'), compression_enabled)
    windmill_serv_app.ns = 'windmill-serv'
    windmill_xmlrpc_app.ns = 'windmill-xmlrpc'
    windmill_jsonrpc_app.ns = 'windmill-jsonrpc'
    windmill_compressor_app.ns = 'windmill-compressor'
    windmill_chooser_app = WindmillChooserApplication(apps=[windmill_serv_app, windmill_jsonrpc_app,
                                                            windmill_xmlrpc_app, windmill_compressor_app],
                                                      proxy=windmill_proxy_app)

    # Make add_namespace available at the module level
    global add_namespace
    add_namespace = windmill_chooser_app.add_namespace

    from cherrypy import wsgiserver
    httpd = wsgiserver.CherryPyWSGIServer(('0.0.0.0', http_port),
                       windmill_chooser_app, server_name='windmill-http', numthreads=50)

    # Attach some objects to httpd for convenience
    httpd.controller_queue = queue
    httpd.test_resolution_suite = test_resolution_suite
    httpd.command_resolution_suite = command_resolution_suite
    httpd.xmlrpc_methods_instance = xmlrpc_methods_instance
    httpd.jsonrpc_methods_instance = jsonrpc_methods_instance

    return httpd

<commit_msg>Forgot to remove debugging print statement.<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

## Note: This file is very trim now because we've broken wsgi_fileserver, wsgi_proxy, wsgi_jsonrpc, and wsgi_xmlrpc
## in to their own libraries which are now distributed on their own and treated as dependencies

import httplib
import copy
import socket
import random
import os
from urlparse import urlparse
import logging
import threading
from time import sleep
logger = logging.getLogger(__name__)

import windmill
from windmill.server import proxy
import wsgi_jsonrpc
import wsgi_xmlrpc
import wsgi_fileserver

import jsmin

START_DST_PORT = 32000
CURRENT_DST_PORT = [random.randint(32000, 34000)]

def reconstruct_url(environ):
    # From WSGI spec, PEP 333
    from urllib import quote
    url = environ['wsgi.url_scheme']+'://'
    if environ.get('HTTP_HOST'): url += environ['HTTP_HOST']
    else:
        url += environ['SERVER_NAME']
        if environ['wsgi.url_scheme'] == 'https':
            if environ['SERVER_PORT'] != '443':
               url += ':' + environ['SERVER_PORT']
        else:
            if environ['SERVER_PORT'] != '80':
               url += ':' + environ['SERVER_PORT']
    url += environ.get('SCRIPT_NAME','')
    url += environ.get('PATH_INFO','')
    # Fix ;arg=value in url
    if url.find('%3B') is not -1:
        url, arg = url.split('%3B', 1)
        url = ';'.join([url, arg.replace('%3D', '=')])
    # Stick query string back in
    if environ.get('QUERY_STRING'):
        url += '?' + environ['QUERY_STRING']
    # Stick it in environ for convenience
    environ['reconstructed_url'] = url
    return url

HTTPConnection = httplib.HTTPConnection

WindmillProxyApplication = proxy.WindmillProxyApplication
WindmillProxyApplication.ConnectionClass = HTTPConnection

add_namespace = None

class WindmillChooserApplication(object):
    """Application to handle choosing the proper application to handle each request"""
    def __init__(self, apps, proxy):
        self.namespaces = dict([ (arg.ns, arg) for arg in apps ])
        self.proxy = proxy

    def add_namespace(self, name, application):
        """Add an application to a specific url namespace in windmill"""
        self.namespaces[name] = application

    def handler(self, environ, start_response):
        """Windmill app chooser"""
        sleep(.2)
        reconstruct_url(environ)

        for key in self.namespaces.keys():
            if environ['PATH_INFO'].find('/'+key+'/') is not -1:
                logger.debug('dispatching request %s to %s' % (environ['reconstructed_url'], key))
                return self.namespaces[key](environ, start_response)

        logger.debug('dispatching request %s to WindmillProxyApplication' % reconstruct_url(environ))
        response = self.proxy(environ, start_response)
        return response

    def __call__(self, environ, start_response):
        response = self.handler(environ, start_response)
        for x in response:
            yield x

class WindmillCompressor(object):
    """Full JavaScript Compression Library"""
    js_file_list = [
        ('lib', 'fleegix.js',),
        ('lib', 'hash.js',),
        ('lib', 'firebug', 'pi.js',),
        ('lib', 'firebug', 'firebug-lite.js',),
        ('lib', 'json2.js',),
        ('lib', 'browserdetect.js',),
        ('wm', 'windmill.js',),
        ('lib', 'getXPath.js',),
        ('lib', 'elementslib.js',),
        ('lib', 'js-xpath.js',),
        ('controller', 'controller.js',),
        ('controller', 'commands.js',),
        ('controller', 'asserts.js',),
        ('controller', 'waits.js',),
        ('extensions', 'extensions.js',),
        ('wm', 'registry.js',),
        ('wm', 'utils.js',),
        ('wm', 'ide', 'ui.js',),
        ('wm', 'ide', 'recorder.js',),
        ('wm', 'ide', 'remote.js',),
        ('wm', 'ide', 'dx.js',),
        ('wm', 'ide', 'ax.js',),
        ('wm', 'ide', 'results.js',),
        ('wm', 'xhr.js',),
        ('wm', 'metrics.js',),
        ('wm', 'events.js',),
        ('wm', 'global.js',),
        ('wm', 'jstest.js',),
        ('wm', 'load.js',),
    ]

    def __init__(self, js_path, enabled=True):
        self.enabled = enabled
        self.js_path = js_path
        self.compressed_windmill = None
        if enabled:
            self._thread = threading.Thread(target=self.compress_file)
            self._thread.start()

    def compress_file(self):
        compressed_windmill = ''
        for filename in self.js_file_list:
            compressed_windmill += jsmin.jsmin(open(os.path.join(self.js_path, *filename), 'r').read())
        self.compressed_windmill = compressed_windmill

    def __call__(self, environ, start_response):
        if not self.enabled:
            start_response('404 Not Found', [('Content-Type', 'text/plain',), ('Content-Length', '0',)])
            return ['']
        # if self.compressed_windmill is None:
        #     self.compressed_windmill = ''
        #     for filename in self.js_file_list:
        #         self.compressed_windmill += jsmin.jsmin(open(os.path.join(self.js_path, *filename), 'r').read())

        while not self.compressed_windmill:
            sleep(.15)

        start_response('200 Ok', [('Content-Type', 'application/x-javascript',),
                                  ('Content-Length', str(len(self.compressed_windmill)),)])
        return [self.compressed_windmill]

def make_windmill_server(http_port=None, js_path=None, compression_enabled=None):
    if http_port is None:
        http_port = windmill.settings['SERVER_HTTP_PORT']
    if js_path is None:
        js_path = windmill.settings['JS_PATH']
    if compression_enabled is None:
        compression_enabled = not windmill.settings['DISABLE_JS_COMPRESS']

    # Start up all the convergence objects
    import convergence
    test_resolution_suite = convergence.TestResolutionSuite()
    command_resolution_suite = convergence.CommandResolutionSuite()
    queue = convergence.ControllerQueue(command_resolution_suite, test_resolution_suite)
    xmlrpc_methods_instance = convergence.XMLRPCMethods(queue, test_resolution_suite, command_resolution_suite)
    jsonrpc_methods_instance = convergence.JSONRPCMethods(queue, test_resolution_suite, command_resolution_suite)

    # Start up all the wsgi applications
    windmill_serv_app = wsgi_fileserver.WSGIFileServerApplication(root_path=js_path, mount_point='/windmill-serv/')
    windmill_proxy_app = WindmillProxyApplication()
    windmill_xmlrpc_app =  wsgi_xmlrpc.WSGIXMLRPCApplication(instance=xmlrpc_methods_instance)
    windmill_jsonrpc_app = wsgi_jsonrpc.WSGIJSONRPCApplication(instance=jsonrpc_methods_instance)
    windmill_compressor_app = WindmillCompressor(os.path.join(js_path, 'js'), compression_enabled)
    windmill_serv_app.ns = 'windmill-serv'
    windmill_xmlrpc_app.ns = 'windmill-xmlrpc'
    windmill_jsonrpc_app.ns = 'windmill-jsonrpc'
    windmill_compressor_app.ns = 'windmill-compressor'
    windmill_chooser_app = WindmillChooserApplication(apps=[windmill_serv_app, windmill_jsonrpc_app,
                                                            windmill_xmlrpc_app, windmill_compressor_app],
                                                      proxy=windmill_proxy_app)

    # Make add_namespace available at the module level
    global add_namespace
    add_namespace = windmill_chooser_app.add_namespace

    from cherrypy import wsgiserver
    httpd = wsgiserver.CherryPyWSGIServer(('0.0.0.0', http_port),
                       windmill_chooser_app, server_name='windmill-http', numthreads=50)

    # Attach some objects to httpd for convenience
    httpd.controller_queue = queue
    httpd.test_resolution_suite = test_resolution_suite
    httpd.command_resolution_suite = command_resolution_suite
    httpd.xmlrpc_methods_instance = xmlrpc_methods_instance
    httpd.jsonrpc_methods_instance = jsonrpc_methods_instance

    return httpd

<|endoftext|>


<commit_before>"""
Implements an abstraction to the database.
"""
import sqlite3
import logging
import time
import pyres.episode as mod_episode
import pyres.utils as utils

class PodcastDatabase(object):
    """ Class to encapsulate access to database """
    def __init__(self, file_name):
        if not file_name:
            raise AttributeError()

        self.connection = sqlite3.connect(file_name)
        self.connection.text_factory = str
        self.cursor = self.connection.cursor()
        # test to ensure that the main podcasts table exists
        # Create it if not
        try:
            self.connection.execute("CREATE TABLE podcasts (name text, url "
                                    "text unique)")
        except sqlite3.OperationalError:
            pass

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, traceback):
        if not exception_type:
            self.connection.commit()
        else:
            self.connection.rollback()
        self.connection.close()

    def add_podcast(self, name, url):
        """Add a new podcast url to the database and set up a table to track
           episodes.
        """
        if not name or not url:
            raise AttributeError()

        with self.connection:
            cursor = self.connection.cursor()

            # make sure this podcast isn't already there
            cursor.execute("Select * from podcasts where name = ?", (name, ))
            check1 = cursor.fetchone()
            if check1 is not None:
                return  # already exists

            cursor.execute("INSERT INTO podcasts VALUES (?, ?)", (name, url))
            cursor.execute("CREATE TABLE '%s' (date text, title text unique, "
                           "file text, url text, size integer, state integer)"
                           % name)

    def add_new_episode_data(self, table, episode):
        """Add the episode data if not already present.  If the episode is
        present, do nothing.
        """
        if not episode.date or not episode.title or not episode.url:
            print("Got a bad episode from server, will not add:", episode.date,
                  episode.title, episode.url)
            return False
        with self.connection:
            cursor = self.connection.cursor()
            cursor.execute("SELECT * from '%s' where title = '%s'" %
                           (table, episode.title))
            check1 = cursor.fetchone()
            if check1 is None:
                cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" %
                               table, episode.as_list())
                logging.debug("Added %s", episode.title)
                return True
            else:
                logging.debug("Didn't add %s as it was already there!",
                              episode.title)
                return False

    def find_episodes_to_download(self, table):
        """ returns a list of episodes read to download """
        return self.find_episodes(table, 0)

    def find_episodes_to_copy(self, table):
        """ returns a list of episodes read to copy to mp3 player """
        return self.find_episodes(table, 1)

    def find_episodes(self, table, state):
        """Return a list of (url, filename) tuples for each file to be
        downloaded.
        """
        episodes = list()
        with self.connection:
            cursor = self.connection.cursor()
            for row in cursor.execute("SELECT * from '%s' where "
                                      "state = %s" % (table, state)):
                row_list = list(row)
                episodes.append(
                    mod_episode.Episode(date=utils.string_to_date(row_list[0]),
                                        title=row_list[1], podcast=table,
                                        file_name=row_list[2], url=row_list[3],
                                        size=row_list[4], state=row_list[5]))
        return episodes

    def _update_size(self, table, title, size):
        """ change state of podcast """
        logging.debug("In update size with %s %s %d", table, title, size)
        with self.connection:
            self.connection.execute("UPDATE '%s' SET size=? where title = ?"
                                    % table, (size, title))

    def _update_state(self, table, title, state):
        """ change state of podcast """
        with self.connection:
            self.connection.execute("UPDATE '%s' SET state=? where title = ?"
                                    % table, (state, title))

    def mark_episode_downloaded(self, episode):
        """ update state to downloaded and update size """
        logging.debug("in mark with %s %s", episode.podcast, episode.title)
        self._update_state(episode.podcast, episode.title, 1)
        self._update_size(episode.podcast, episode.title, episode.size)

    def mark_episode_on_mp3_player(self, episode):
        """ update state to downloaded and update size """
        logging.debug("in on player : %s %s", episode.podcast, episode.title)
        self._update_state(episode.podcast, episode.title, 2)

    def delete_podcast(self, name):
        """Delete a podcast from the main table.  Also drops the episode table
           for this podcast.
        """
        with self.connection:
            cursor = self.connection.cursor()
            cursor.execute("DELETE FROM podcasts WHERE name='%s'" % name)
            cursor.execute("DROP TABLE '%s'" % name)

    def get_podcast_urls(self):
        """Return a list of urls.  Goes through the table list and produces a
        list of [name, url] tuples.  Walks this list and produces a list of
        [url, latest_date] tuples from the podcast tables specified by names.
        I suspect someone better at SQL than I could get this in a single
        query, but this will work quickly enough for the small data sets I'm
        expecting.
        """
        urls = list()
        tuples = list()
        with self.connection:
            cursor = self.connection.cursor()
            urls = list(cursor.execute('SELECT url,name FROM podcasts '
                                       'ORDER BY name'))
            for _tuple in urls:
                url = _tuple[0]
                name = _tuple[1]
                cursor.execute("Select date from '%s' ORDER BY date DESC" %
                               name)
                check1 = cursor.fetchone()
                if check1 is not None:
                    latest_date = utils.string_to_date(check1[0])
                else:
                    latest_date = time.gmtime()
                tuples.append([url, latest_date])
        return tuples

    def get_podcast_names(self):
        """Return a list of podcasts
        """
        names = list()
        with self.connection:
            cursor = self.connection.cursor()
            for name in cursor.execute('SELECT name FROM podcasts ORDER BY '
                                       'name'):
                names.append(name[0])
        return names

    def show_podcasts(self):
        """Display information from database.
        """
        names = self.get_podcast_names()
        with self.connection:
            cursor = self.connection.cursor()
            for name in names:
                print
                print name
                #if "memory" not in name:
                    #continue
                for row in cursor.execute("SELECT * FROM '%s'" % name):
                    row_list = list(row)
                    print row_list[0], row_list[1], row_list[5],
                    if row_list[3]:
                        print "URL OK"
                    else:
                        print "BAD URL"

    def clean_table_of_dups(self, name):
        """ gets all episodes from a table, only keeping unique dates.  Then
        drops the table and re-creates it with only unique data. """
        uniques = dict()
        with self.connection:
            cursor = self.connection.cursor()
            for row in cursor.execute("SELECT * FROM '%s'" % name):
                row_list = list(row)
                print row_list[0], row_list[1], row_list[5]
                if not row_list[0] in uniques:
                    uniques[row_list[0]] = row_list
            print
            for key in uniques:
                print key, uniques[key][1]

            # now drop that table entirely!
            cursor.execute("DROP TABLE '%s'" % name)
            # and re-create it
            cursor.execute("CREATE TABLE '%s' (date text, title text unique, "
                           "file text, url text, size integer, state integer)"
                           % name)

            for key in uniques:
                cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" %
                               name, uniques[key])

    def convert_tables(self):
        """ Utility to change format of the podcast tables. """
        # JHA this  returned names, too podcasts = self.jima_get_podcast_urls()
        #for _tuple in sorted(podcasts):
            #print "%-49s" % (_tuple[1])
            #self.clean_table_of_dups(_tuple[1])

        #name = 'NPR Fresh Air'
        #name = 'How To Do Everything'
        #name = 'TED Radio Hour'
        #name = 'This American Life'
        #name = 'Wait Wait... Dont Tell Me!'
        #self.clean_table_of_dups(name)
        #pass
        # use old version of string to date function
        #utils.string_to_date = lambda x: time.strptime(x, "%x:%X")

        #names = self.get_podcast_names()
        #with self.connection:
            #cursor = self.connection.cursor()
            #for name in names:
                #if "Invisible" not in name:
                    #continue
                #print "GOT IT %s" % name
                #title = "Kickstart Radiotopia- A Storytelling Revolution"
                #cursor.execute("DELETE FROM '%s' WHERE title='%s'" % (name,
                                                                      #title))
                #newtable = list()
                #print "----------------------------------------"
                #print name
                #print "----------------------------------------"
                #for row in cursor.execute("SELECT * FROM '%s'" % name):
                    # right now we're converting date from mm/dd/yy to
                    # yyyy/mm/dd to it sorts correctly
                    #row_list = list(row)
                    #date = utils.string_to_date(row_list[0])
                    #row_list[0] = utils.date_as_string(date)
                    #newtable.append(row_list)
                #print newtable
                #cursor.execute("DROP TABLE '%s'" % name)
                #cursor.execute("CREATE TABLE '%s' (date text, title text, "
                               #"file text, url text, size integer, state "
                               #"integer)" % name)
                #for podcast in newtable:
                    #cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?" \
                                   #", ?)" % name, podcast)

if __name__ == "__main__":
    print "Not Implemented"
<commit_msg>Catch exception and swallow it on duplicate title entry.  Fresh air giving me grief<commit_after>"""
Implements an abstraction to the database.
"""
import sqlite3
import logging
import time
import pyres.episode as mod_episode
import pyres.utils as utils

class PodcastDatabase(object):
    """ Class to encapsulate access to database """
    def __init__(self, file_name):
        if not file_name:
            raise AttributeError()

        self.connection = sqlite3.connect(file_name)
        self.connection.text_factory = str
        self.cursor = self.connection.cursor()
        # test to ensure that the main podcasts table exists
        # Create it if not
        try:
            self.connection.execute("CREATE TABLE podcasts (name text, url "
                                    "text unique)")
        except sqlite3.OperationalError:
            pass

    def __enter__(self):
        return self

    def __exit__(self, exception_type, exception_value, traceback):
        if not exception_type:
            self.connection.commit()
        else:
            self.connection.rollback()
        self.connection.close()

    def add_podcast(self, name, url):
        """Add a new podcast url to the database and set up a table to track
           episodes.
        """
        if not name or not url:
            raise AttributeError()

        with self.connection:
            cursor = self.connection.cursor()

            # make sure this podcast isn't already there
            cursor.execute("Select * from podcasts where name = ?", (name, ))
            check1 = cursor.fetchone()
            if check1 is not None:
                return  # already exists

            cursor.execute("INSERT INTO podcasts VALUES (?, ?)", (name, url))
            cursor.execute("CREATE TABLE '%s' (date text, title text unique, "
                           "file text, url text, size integer, state integer)"
                           % name)

    def add_new_episode_data(self, table, episode):
        """Add the episode data if not already present.  If the episode is
        present, do nothing.
        """
        if not episode.date or not episode.title or not episode.url:
            print("Got a bad episode from server, will not add:", episode.date,
                  episode.title, episode.url)
            return False
        with self.connection:
            cursor = self.connection.cursor()
            cursor.execute("SELECT * from '%s' where title = '%s'" %
                           (table, episode.title))
            check1 = cursor.fetchone()
            if check1 is None:
                try:
                    cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)"
                                   % table, episode.as_list())
                    logging.debug("Added %s", episode.title)
                    return True
                except:
                    # Fresh air is giving me duplicate titles for some reason
                    # The table will throw on a duplicate name.  We'll ignore
                    # it for now.  Would be good to figure out what's going
                    # on there
                    pass

            else:
                logging.debug("Didn't add %s as it was already there!",
                              episode.title)
                return False

    def find_episodes_to_download(self, table):
        """ returns a list of episodes read to download """
        return self.find_episodes(table, 0)

    def find_episodes_to_copy(self, table):
        """ returns a list of episodes read to copy to mp3 player """
        return self.find_episodes(table, 1)

    def find_episodes(self, table, state):
        """Return a list of (url, filename) tuples for each file to be
        downloaded.
        """
        episodes = list()
        with self.connection:
            cursor = self.connection.cursor()
            for row in cursor.execute("SELECT * from '%s' where "
                                      "state = %s" % (table, state)):
                row_list = list(row)
                episodes.append(
                    mod_episode.Episode(date=utils.string_to_date(row_list[0]),
                                        title=row_list[1], podcast=table,
                                        file_name=row_list[2], url=row_list[3],
                                        size=row_list[4], state=row_list[5]))
        return episodes

    def _update_size(self, table, title, size):
        """ change state of podcast """
        logging.debug("In update size with %s %s %d", table, title, size)
        with self.connection:
            self.connection.execute("UPDATE '%s' SET size=? where title = ?"
                                    % table, (size, title))

    def _update_state(self, table, title, state):
        """ change state of podcast """
        with self.connection:
            self.connection.execute("UPDATE '%s' SET state=? where title = ?"
                                    % table, (state, title))

    def mark_episode_downloaded(self, episode):
        """ update state to downloaded and update size """
        logging.debug("in mark with %s %s", episode.podcast, episode.title)
        self._update_state(episode.podcast, episode.title, 1)
        self._update_size(episode.podcast, episode.title, episode.size)

    def mark_episode_on_mp3_player(self, episode):
        """ update state to downloaded and update size """
        logging.debug("in on player : %s %s", episode.podcast, episode.title)
        self._update_state(episode.podcast, episode.title, 2)

    def delete_podcast(self, name):
        """Delete a podcast from the main table.  Also drops the episode table
           for this podcast.
        """
        with self.connection:
            cursor = self.connection.cursor()
            cursor.execute("DELETE FROM podcasts WHERE name='%s'" % name)
            cursor.execute("DROP TABLE '%s'" % name)

    def get_podcast_urls(self):
        """Return a list of urls.  Goes through the table list and produces a
        list of [name, url] tuples.  Walks this list and produces a list of
        [url, latest_date] tuples from the podcast tables specified by names.
        I suspect someone better at SQL than I could get this in a single
        query, but this will work quickly enough for the small data sets I'm
        expecting.
        """
        urls = list()
        tuples = list()
        with self.connection:
            cursor = self.connection.cursor()
            urls = list(cursor.execute('SELECT url,name FROM podcasts '
                                       'ORDER BY name'))
            for _tuple in urls:
                url = _tuple[0]
                name = _tuple[1]
                cursor.execute("Select date from '%s' ORDER BY date DESC" %
                               name)
                check1 = cursor.fetchone()
                if check1 is not None:
                    latest_date = utils.string_to_date(check1[0])
                else:
                    latest_date = time.gmtime()
                tuples.append([url, latest_date])
        return tuples

    def get_podcast_names(self):
        """Return a list of podcasts
        """
        names = list()
        with self.connection:
            cursor = self.connection.cursor()
            for name in cursor.execute('SELECT name FROM podcasts ORDER BY '
                                       'name'):
                names.append(name[0])
        return names

    def show_podcasts(self):
        """Display information from database.
        """
        names = self.get_podcast_names()
        with self.connection:
            cursor = self.connection.cursor()
            for name in names:
                print
                print name
                #if "memory" not in name:
                    #continue
                for row in cursor.execute("SELECT * FROM '%s'" % name):
                    row_list = list(row)
                    print row_list[0], row_list[1], row_list[5],
                    if row_list[3]:
                        print "URL OK"
                    else:
                        print "BAD URL"

    def clean_table_of_dups(self, name):
        """ gets all episodes from a table, only keeping unique dates.  Then
        drops the table and re-creates it with only unique data. """
        uniques = dict()
        with self.connection:
            cursor = self.connection.cursor()
            for row in cursor.execute("SELECT * FROM '%s'" % name):
                row_list = list(row)
                print row_list[0], row_list[1], row_list[5]
                if not row_list[0] in uniques:
                    uniques[row_list[0]] = row_list
            print
            for key in uniques:
                print key, uniques[key][1]

            # now drop that table entirely!
            cursor.execute("DROP TABLE '%s'" % name)
            # and re-create it
            cursor.execute("CREATE TABLE '%s' (date text, title text unique, "
                           "file text, url text, size integer, state integer)"
                           % name)

            for key in uniques:
                cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?, ?)" %
                               name, uniques[key])

    def convert_tables(self):
        """ Utility to change format of the podcast tables. """
        # JHA this  returned names, too podcasts = self.jima_get_podcast_urls()
        #for _tuple in sorted(podcasts):
            #print "%-49s" % (_tuple[1])
            #self.clean_table_of_dups(_tuple[1])

        #name = 'NPR Fresh Air'
        #name = 'How To Do Everything'
        #name = 'TED Radio Hour'
        #name = 'This American Life'
        #name = 'Wait Wait... Dont Tell Me!'
        #self.clean_table_of_dups(name)
        #pass
        # use old version of string to date function
        #utils.string_to_date = lambda x: time.strptime(x, "%x:%X")

        #names = self.get_podcast_names()
        #with self.connection:
            #cursor = self.connection.cursor()
            #for name in names:
                #if "Invisible" not in name:
                    #continue
                #print "GOT IT %s" % name
                #title = "Kickstart Radiotopia- A Storytelling Revolution"
                #cursor.execute("DELETE FROM '%s' WHERE title='%s'" % (name,
                                                                      #title))
                #newtable = list()
                #print "----------------------------------------"
                #print name
                #print "----------------------------------------"
                #for row in cursor.execute("SELECT * FROM '%s'" % name):
                    # right now we're converting date from mm/dd/yy to
                    # yyyy/mm/dd to it sorts correctly
                    #row_list = list(row)
                    #date = utils.string_to_date(row_list[0])
                    #row_list[0] = utils.date_as_string(date)
                    #newtable.append(row_list)
                #print newtable
                #cursor.execute("DROP TABLE '%s'" % name)
                #cursor.execute("CREATE TABLE '%s' (date text, title text, "
                               #"file text, url text, size integer, state "
                               #"integer)" % name)
                #for podcast in newtable:
                    #cursor.execute("INSERT INTO '%s' VALUES (?, ?, ?, ?, ?" \
                                   #", ?)" % name, podcast)

if __name__ == "__main__":
    print "Not Implemented"
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Created by: Lee Bergstrand (2017)

Description: A set of helper functions.
"""

from modules.database_reference import DatabaseReference
from modules.genome_property import GenomeProperty
from modules.literature_reference import LiteratureReference
from modules.step import Step

def create_marker_and_content(genome_property_flat_file_line):
    """
    Splits a list of lines from a genome property file into marker, content pairs.
    :param genome_property_flat_file_line: A line from a genome property flat file line.
    :return: A tuple containing a marker, content pair.
    """
    columns = genome_property_flat_file_line.split('  ')
    marker = columns[0].strip()
    content = ''.join(columns[1:]).rstrip()
    return marker, content

def collapse_genome_property_record(genome_property_record):
    """
    The standard genome property record wraps every 80 lines. This function unwraps the record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return:    A list of reduced redundancy markers, content tuples representing genome property flat file lines.
                Consecutive markers (often 'CC' and '**') markers are collapsed to one tuple.
                Inside steps, multiple 'EV' and 'TG' markers are piled up to two markers.
    """
    collapsed_genome_property_record = []

    trailing_marker_content = []
    previous_marker = genome_property_record[0][0]
    for marker, content in genome_property_record:
        if marker == previous_marker:
            trailing_marker_content.append(content)
        else:
            collapsed_marker_content = ' '.join(trailing_marker_content)
            new_collapsed_marker = (previous_marker, collapsed_marker_content)

            collapsed_genome_property_record.append(new_collapsed_marker)

            previous_marker = marker
            trailing_marker_content = [content]

    final_genome_property_record = collapse_step_evidence_and_gene_ontologys(collapsed_genome_property_record)
    return final_genome_property_record

def collapse_step_evidence_and_gene_ontologys(genome_property_record):
    """
    Pile up multiple consecutive 'EV' and 'TG' pairs to a single set of 'EV' and 'TG' pairs.
    Example:
        EV  C1
        TG  C2  ==== Goes To ====>> EV C1; C3;
        EV  C3                      TG C2; C4;
        EV  C4
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return:    A list of reduced redundancy markers, content tuples representing genome property flat file lines.
                Inside steps, multiple 'EV' and 'TG' markers are piled up to two markers. See above for final layout.

    TODO:   There may be an issue with this and the sufficient key word. The sufficient key word may be only for one
            piece of evidence only.
    """
    current_evidence = []
    current_go_terms = []
    final_genome_property_record = []
    for marker, content in genome_property_record:
        if marker == '--':
            if current_evidence:
                final_genome_property_record.append(('EV', ' '.join(current_evidence)))
                current_evidence = []
            if current_go_terms:
                final_genome_property_record.append(('TG', ' '.join(current_go_terms)))
                current_go_terms = []
            final_genome_property_record.append(('--', ''))
        else:
            if marker == 'EV':
                current_evidence.append(content)
            elif marker == 'TG':
                current_go_terms.append(content)
            else:
                final_genome_property_record.append((marker, content))

    return final_genome_property_record

def parse_genome_property(genome_property_record):
    """
    Parses a single genome property from a genome property record.
    :param genome_property_record:  A list of marker, content tuples representing genome property flat file lines.
    :return: A single genome property object.
    """
    # A list of record markers related to the genome property.
    core_genome_property_markers = ['AC', 'DE', 'TP', 'TH', 'PN', 'CC', '**']
    gathered_core_genome_property_markers = {}

    has_references = False
    has_databases = False
    has_steps = False

    for marker, content in genome_property_record:
        if marker == 'RN':
            has_references = True
        elif marker == 'DC':
            has_databases = True
        elif marker == 'SN':
            has_steps = True
        elif marker in core_genome_property_markers:
            if marker == 'TH':
                content = int(content)
            gathered_core_genome_property_markers[marker] = content

    if has_references:
        references = parse_literature_references(genome_property_record)
    else:
        references = []

    if has_databases:
        databases = parse_database_references(genome_property_record)
    else:
        databases = []

    if has_steps:
        steps = parse_steps(genome_property_record)
    else:
        steps = []

    new_genome_property = GenomeProperty(accession_id=gathered_core_genome_property_markers.get('AC'),
                                         name=gathered_core_genome_property_markers.get('DE'),
                                         property_type=gathered_core_genome_property_markers.get('TP'),
                                         threshold=gathered_core_genome_property_markers.get('TH'),
                                         parent=gathered_core_genome_property_markers.get('PN'),
                                         description=gathered_core_genome_property_markers.get('CC'),
                                         private_notes=gathered_core_genome_property_markers.get('**'),
                                         references=references,
                                         databases=databases,
                                         steps=steps)
    return new_genome_property

def parse_literature_references(genome_property_record):
    """
    Parses literature references from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of LiteratureReference objects.
    """
    # A list of record markers related to literature references.
    literature_reference_markers = ['RN', 'RM', 'RT', 'RA', 'RL']

    literature_references = []
    current_literature_reference = {}
    for marker, content in genome_property_record:
        if marker in literature_reference_markers:
            if marker in current_literature_reference:
                literature_references.append(LiteratureReference(number=current_literature_reference.get('RN'),
                                                                 pubmed_id=current_literature_reference.get('RM'),
                                                                 title=current_literature_reference.get('RT'),
                                                                 authors=current_literature_reference.get('RA'),
                                                                 citation=current_literature_reference.get('RL')))
                if marker == 'RN':
                    content = int(content.strip('[]'))

                current_literature_reference = {marker: content}
            else:
                if marker == 'RN':
                    content = int(content.strip('[]'))

                current_literature_reference[marker] = content

    literature_references.append(LiteratureReference(number=current_literature_reference.get('RN'),
                                                     pubmed_id=current_literature_reference.get('RM'),
                                                     title=current_literature_reference.get('RT'),
                                                     authors=current_literature_reference.get('RA'),
                                                     citation=current_literature_reference.get('RL')))
    return literature_references

def parse_steps(genome_property_record):
    """
    Parses steps from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of Step objects.
    """
    step_markers = ['SN', 'ID', 'DN', 'RQ', 'EV', 'TG']
    steps = []
    try:
        current_step = {}
        for marker, content in genome_property_record:
            if marker in step_markers:
                if marker in current_step:
                    steps.append(Step(number=current_step.get('SN'), identifier=current_step.get('ID'),
                                      name=current_step.get('DN'), evidence=current_step.get('EV'),
                                      gene_ontology_id=current_step.get('TG'), required=current_step.get('RQ'),
                                      sufficient=current_step.get('SF')))

                    if marker == 'SN':
                        content = int(content)

                    current_step = {marker: content}
                else:
                    if marker == 'SN':
                        content = int(content)
                    elif marker == 'EV':
                        split_content = filter(None, content.split(';'))
                        cleaned_content = list(map(lambda evidence: evidence.strip(), split_content))
                        if 'sufficient' in cleaned_content:
                            current_step['SF'] = True
                        else:
                            current_step['SF'] = False
                        content = [evidence for evidence in cleaned_content if evidence != 'sufficient']
                    elif marker == 'RQ':
                        if int(content) == 1:
                            content = True
                        else:
                            content = False

                    current_step[marker] = content

        steps.append(Step(number=current_step.get('SN'), identifier=current_step.get('ID'),
                          name=current_step.get('DN'), evidence=current_step.get('EV'),
                          gene_ontology_id=current_step.get('TG'), required=current_step.get('RQ'),
                          sufficient=current_step.get('SF')))
    except TypeError:
        print('yolo')
    finally:
        return steps

def parse_database_references(genome_property_record):
    """
    Parses database reference from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of DatabaseReference objects.
    """
    database_reference_markers = ['DC', 'DR']

    database_references = []
    current_database_reference = {}
    for marker, content in genome_property_record:
        if marker in database_reference_markers:
            if marker in current_database_reference:
                database_references.append(DatabaseReference(record_title=current_database_reference.get('DC'),
                                                             database_name=current_database_reference.get('DN'),
                                                             record_ids=current_database_reference.get('DI')))

                current_database_reference = {marker: content}
            else:
                if marker == 'DR':
                    split_content = filter(None, content.split(';'))
                    cleaned_content = list(map(lambda evidence: evidence.strip(), split_content))
                    database_name = cleaned_content[0]
                    database_records = cleaned_content[1:]
                    current_database_reference['DN'] = database_name
                    current_database_reference['DI'] = database_records

                current_database_reference[marker] = content

    database_references.append(DatabaseReference(record_title=current_database_reference.get('DC'),
                                                 database_name=current_database_reference.get('DN'),
                                                 record_ids=current_database_reference.get('DI')))
    return database_references

def parse_genome_property_file(gen_prop_file):
    """
    A parses a genome property flat file.
    :param gen_prop_file: A genome property file object.
    :return: A list of GenomeProperty objects.
    """
    genome_properties = []
    current_genome_property_record = []
    for line in gen_prop_file:
        if not line.strip() == '//':
            current_genome_property_record.append(create_marker_and_content(line))
        else:
            collapsed_genome_property_record = collapse_genome_property_record(current_genome_property_record)
            new_genome_property = parse_genome_property(collapsed_genome_property_record)
            genome_properties.append(new_genome_property)
            current_genome_property_record = []

    return genome_properties
<commit_msg>Only send row subset to parsing functions.<commit_after>#!/usr/bin/env python

"""
Created by: Lee Bergstrand (2017)

Description: A set of helper functions.
"""

from modules.database_reference import DatabaseReference
from modules.genome_property import GenomeProperty
from modules.literature_reference import LiteratureReference
from modules.step import Step

def create_marker_and_content(genome_property_flat_file_line):
    """
    Splits a list of lines from a genome property file into marker, content pairs.
    :param genome_property_flat_file_line: A line from a genome property flat file line.
    :return: A tuple containing a marker, content pair.
    """
    columns = genome_property_flat_file_line.split('  ')
    marker = columns[0].strip()
    content = ''.join(columns[1:]).rstrip()
    return marker, content

def collapse_genome_property_record(genome_property_record):
    """
    The standard genome property record wraps every 80 lines. This function unwraps the record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return:    A list of reduced redundancy markers, content tuples representing genome property flat file lines.
                Consecutive markers (often 'CC' and '**') markers are collapsed to one tuple.
                Inside steps, multiple 'EV' and 'TG' markers are piled up to two markers.
    """
    collapsed_genome_property_record = []

    trailing_marker_content = []
    previous_marker = genome_property_record[0][0]
    for marker, content in genome_property_record:
        if marker == previous_marker:
            trailing_marker_content.append(content)
        else:
            collapsed_marker_content = ' '.join(trailing_marker_content)
            new_collapsed_marker = (previous_marker, collapsed_marker_content)

            collapsed_genome_property_record.append(new_collapsed_marker)

            previous_marker = marker
            trailing_marker_content = [content]

    final_genome_property_record = collapse_step_evidence_and_gene_ontologys(collapsed_genome_property_record)
    return final_genome_property_record

def collapse_step_evidence_and_gene_ontologys(genome_property_record):
    """
    Pile up multiple consecutive 'EV' and 'TG' pairs to a single set of 'EV' and 'TG' pairs.
    Example:
        EV  C1
        TG  C2  ==== Goes To ====>> EV C1; C3;
        EV  C3                      TG C2; C4;
        EV  C4
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return:    A list of reduced redundancy markers, content tuples representing genome property flat file lines.
                Inside steps, multiple 'EV' and 'TG' markers are piled up to two markers. See above for final layout.

    TODO:   There may be an issue with this and the sufficient key word. The sufficient key word may be only for one
            piece of evidence only.
    """
    current_evidence = []
    current_go_terms = []
    final_genome_property_record = []
    for marker, content in genome_property_record:
        if marker == '--':
            if current_evidence:
                final_genome_property_record.append(('EV', ' '.join(current_evidence)))
                current_evidence = []
            if current_go_terms:
                final_genome_property_record.append(('TG', ' '.join(current_go_terms)))
                current_go_terms = []
            final_genome_property_record.append(('--', ''))
        else:
            if marker == 'EV':
                current_evidence.append(content)
            elif marker == 'TG':
                current_go_terms.append(content)
            else:
                final_genome_property_record.append((marker, content))

    return final_genome_property_record

def parse_genome_property(genome_property_record):
    """
    Parses a single genome property from a genome property record.
    :param genome_property_record:  A list of marker, content tuples representing genome property flat file lines.
    :return: A single genome property object.
    """
    # A list of record markers related to the genome property.
    core_genome_property_markers = ['AC', 'DE', 'TP', 'TH', 'PN', 'CC', '**']
    gathered_core_genome_property_markers = {}

    reference_index = False
    database_index = False
    step_index = False

    current_index = 0
    for marker, content in genome_property_record:
        if marker == 'RN':
            if not reference_index:
                reference_index = current_index
        elif marker == 'DC':
            if not database_index:
                database_index = current_index
        elif marker == 'SN':
            if not step_index:
                step_index = current_index
        elif marker in core_genome_property_markers:
            if marker == 'TH':
                content = int(content)
            gathered_core_genome_property_markers[marker] = content

        current_index = current_index + 1

    if reference_index:
        reference_rows = genome_property_record[reference_index:database_index]
        references = parse_literature_references(reference_rows)
    else:
        references = []

    if database_index:
        database_rows = genome_property_record[database_index:step_index - 1]
        databases = parse_database_references(database_rows)
    else:
        databases = []

    if step_index:
        step_rows = genome_property_record[step_index:]
        steps = parse_steps(step_rows)
    else:
        steps = []

    new_genome_property = GenomeProperty(accession_id=gathered_core_genome_property_markers.get('AC'),
                                         name=gathered_core_genome_property_markers.get('DE'),
                                         property_type=gathered_core_genome_property_markers.get('TP'),
                                         threshold=gathered_core_genome_property_markers.get('TH'),
                                         parent=gathered_core_genome_property_markers.get('PN'),
                                         description=gathered_core_genome_property_markers.get('CC'),
                                         private_notes=gathered_core_genome_property_markers.get('**'),
                                         references=references,
                                         databases=databases,
                                         steps=steps)
    return new_genome_property

def parse_literature_references(genome_property_record):
    """
    Parses literature references from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of LiteratureReference objects.
    """
    # A list of record markers related to literature references.
    literature_reference_markers = ['RN', 'RM', 'RT', 'RA', 'RL']

    literature_references = []
    current_literature_reference = {}
    for marker, content in genome_property_record:
        if marker in literature_reference_markers:
            if marker in current_literature_reference:
                literature_references.append(LiteratureReference(number=current_literature_reference.get('RN'),
                                                                 pubmed_id=current_literature_reference.get('RM'),
                                                                 title=current_literature_reference.get('RT'),
                                                                 authors=current_literature_reference.get('RA'),
                                                                 citation=current_literature_reference.get('RL')))
                if marker == 'RN':
                    content = int(content.strip('[]'))

                current_literature_reference = {marker: content}
            else:
                if marker == 'RN':
                    content = int(content.strip('[]'))

                current_literature_reference[marker] = content

    literature_references.append(LiteratureReference(number=current_literature_reference.get('RN'),
                                                     pubmed_id=current_literature_reference.get('RM'),
                                                     title=current_literature_reference.get('RT'),
                                                     authors=current_literature_reference.get('RA'),
                                                     citation=current_literature_reference.get('RL')))
    return literature_references

def parse_steps(genome_property_record):
    """
    Parses steps from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of Step objects.
    """
    step_markers = ['SN', 'ID', 'DN', 'RQ', 'EV', 'TG']
    steps = []
    try:
        current_step = {}
        for marker, content in genome_property_record:
            if marker in step_markers:
                if marker in current_step:
                    steps.append(Step(number=current_step.get('SN'), identifier=current_step.get('ID'),
                                      name=current_step.get('DN'), evidence=current_step.get('EV'),
                                      gene_ontology_id=current_step.get('TG'), required=current_step.get('RQ'),
                                      sufficient=current_step.get('SF')))

                    if marker == 'SN':
                        content = int(content)

                    current_step = {marker: content}
                else:
                    if marker == 'SN':
                        content = int(content)
                    elif marker == 'EV':
                        split_content = filter(None, content.split(';'))
                        cleaned_content = list(map(lambda evidence: evidence.strip(), split_content))
                        if 'sufficient' in cleaned_content:
                            current_step['SF'] = True
                        else:
                            current_step['SF'] = False
                        content = [evidence for evidence in cleaned_content if evidence != 'sufficient']
                    elif marker == 'RQ':
                        if int(content) == 1:
                            content = True
                        else:
                            content = False

                    current_step[marker] = content

        steps.append(Step(number=current_step.get('SN'), identifier=current_step.get('ID'),
                          name=current_step.get('DN'), evidence=current_step.get('EV'),
                          gene_ontology_id=current_step.get('TG'), required=current_step.get('RQ'),
                          sufficient=current_step.get('SF')))
    except TypeError:
        print('yolo')
    finally:
        return steps

def parse_database_references(genome_property_record):
    """
    Parses database reference from a genome properties record.
    :param genome_property_record: A list of marker, content tuples representing genome property flat file lines.
    :return: A list of DatabaseReference objects.
    """
    database_reference_markers = ['DC', 'DR']

    database_references = []
    current_database_reference = {}
    for marker, content in genome_property_record:
        if marker in database_reference_markers:
            if marker in current_database_reference:
                database_references.append(DatabaseReference(record_title=current_database_reference.get('DC'),
                                                             database_name=current_database_reference.get('DN'),
                                                             record_ids=current_database_reference.get('DI')))

                current_database_reference = {marker: content}
            else:
                if marker == 'DR':
                    split_content = filter(None, content.split(';'))
                    cleaned_content = list(map(lambda evidence: evidence.strip(), split_content))
                    database_name = cleaned_content[0]
                    database_records = cleaned_content[1:]
                    current_database_reference['DN'] = database_name
                    current_database_reference['DI'] = database_records

                current_database_reference[marker] = content

    database_references.append(DatabaseReference(record_title=current_database_reference.get('DC'),
                                                 database_name=current_database_reference.get('DN'),
                                                 record_ids=current_database_reference.get('DI')))
    return database_references

def parse_genome_property_file(gen_prop_file):
    """
    A parses a genome property flat file.
    :param gen_prop_file: A genome property file object.
    :return: A list of GenomeProperty objects.
    """
    genome_properties = []
    current_genome_property_record = []
    for line in gen_prop_file:
        if not line.strip() == '//':
            current_genome_property_record.append(create_marker_and_content(line))
        else:
            collapsed_genome_property_record = collapse_genome_property_record(current_genome_property_record)
            new_genome_property = parse_genome_property(collapsed_genome_property_record)
            genome_properties.append(new_genome_property)
            current_genome_property_record = []

    return genome_properties
<|endoftext|>


<commit_before>from django.conf.urls import url

from . import views

urlpatterns = [
    #url(r'^$', views.review_list, name='review_list'),
    url(r'^$', views.new_wine, name='home'),
    url(r'^new_review/$', views.new_review, name='new_review'),
    url(r'^review/user/(?P<username>\w+)/$', views.user_review_list, name='user_review_list'),
    url(r'^review/edit/(?P<review_id>[0-9]+)/$', views.edit_review_form, name='edit_review_form'),
    url(r'^review/edit_review/(?P<review_id>[0-9]+)/$', views.edit_review, name='edit_review'),
    url(r'^review/delete_review/(?P<review_id>[0-9]+)/$', views.delete_review, name='delete_review'),
    url(r'^detail/(?P<review_id>[0-9]+)/$', views.wine_detail, name='wine_detail'),

    #url(r'^wine/(?P<wine_id>[0-9]+)/add_review/$', views.add_review, name='add_review'),

    url(r'^wine$', views.wine_list, name='wine_list'),
    url(r'^recommendation/$', views.user_reccommendation_list, name = 'user_recommendation_list'),
]
<commit_msg>testing<commit_after>from django.conf.urls import url

from . import views

urlpatterns = [
    #url(r'^$', views.review_list, name='review_list'),
    url(r'^$', views.new_wine, name='home'),
    url(r'^new_review/$', views.new_review, name='new_review'),
    url(r'^review/user/(?P<username>\w+)/$', views.user_review_list, name='user_review_list'),
    url(r'^review/edit/(?P<review_id>[0-9]+)/$', views.edit_review_form, name='edit_review_form'),
    url(r'^review/edit_review/(?P<review_id>[0-9]+)/$', views.edit_review, name='edit_review'),
    url(r'^review/delete_review/(?P<review_id>[0-9]+)/$', views.delete_review, name='delete_review'),
    url(r'^detail/(?P<review_id>[0-9]+)/$', views.wine_detail, name='wine_detail'),
]
<|endoftext|>


<commit_before>import logging
from urllib.request import Request, urlopen

from .renderers import XMLRenderer
from .models import SendResponse, LabelResponse
from .parsers import XMLParser

logger = logging.getLogger(__name__)

def soap(tag):
    return '{http://www.w3.org/2003/05/soap-envelope}%s' % tag

def mrw(tag):
    return '{http://www.mrw.es/}%s' % tag

class Client:
    def __init__(self, hostname, auth_info):
        self.base_url = 'http://{}/MRWEnvio.asmx'.format(hostname)
        self.auth_info = auth_info
        self.parser = XMLParser()
        self.renderer = XMLRenderer()

    def make_http_request(self, tag, request_data, extra_headers=None):
        # TODO: insert header 'SOAPAction: "http://www.mrw.es/GetEtiquetaEnvio"
        extra_headers = extra_headers or {}
        data = [
            {'soap:Header': self.auth_info.as_dict()},
            {'soap:Body': {
                tag: {
                    'mrw:request': request_data
                },
            }},
        ]
        namespaces = [
            ('soap', 'http://www.w3.org/2003/05/soap-envelope'),
            ('mrw', 'http://www.mrw.es/'),
        ]
        data = self.renderer.render({'soap:Envelope': data}, namespaces)
        logger.debug('Sending %s', data)
        headers = {'Content-type': 'application/soap+xml; charset=utf-8'}
        headers.update(extra_headers)
        return Request(self.base_url, data, headers, method='POST')

    def send(self, pickup_info, service_info):
        request_data = [pickup_info.as_dict(), service_info.as_dict()]
        req = self.make_http_request('mrw:TransmEnvio', request_data)
        with urlopen(req) as response:
            body = response.read()
            logger.debug('Received %s', body)

        response = self.parser.parse(body)
        message = response[soap('Envelope')][0][soap('Body')][0]
        response_obj = SendResponse.from_raw(message)
        return response_obj.result

    def get_label(self, get_label_request):
        request_data = [get_label_request.as_dict()[None]]
        req = self.make_http_request('mrw:GetEtiquetaEnvio', request_data)
        with urlopen(req) as response:
            body = response.read()
            logger.debug('Received %s', body)

        response = self.parser.parse(body)
        message = response[soap('Envelope')][0][soap('Body')][0]
        response_obj = LabelResponse.from_raw(message)
        return response_obj.result
<commit_msg>Remove unused code<commit_after>import logging
from urllib.request import Request, urlopen

from .renderers import XMLRenderer
from .models import SendResponse, LabelResponse
from .parsers import XMLParser

logger = logging.getLogger(__name__)

def soap(tag):
    return '{http://www.w3.org/2003/05/soap-envelope}%s' % tag

class Client:
    def __init__(self, hostname, auth_info):
        self.base_url = 'http://{}/MRWEnvio.asmx'.format(hostname)
        self.auth_info = auth_info
        self.parser = XMLParser()
        self.renderer = XMLRenderer()

    def make_http_request(self, tag, request_data, extra_headers=None):
        # TODO: insert header 'SOAPAction: "http://www.mrw.es/GetEtiquetaEnvio"
        extra_headers = extra_headers or {}
        data = [
            {'soap:Header': self.auth_info.as_dict()},
            {'soap:Body': {
                tag: {
                    'mrw:request': request_data
                },
            }},
        ]
        namespaces = [
            ('soap', 'http://www.w3.org/2003/05/soap-envelope'),
            ('mrw', 'http://www.mrw.es/'),
        ]
        data = self.renderer.render({'soap:Envelope': data}, namespaces)
        logger.debug('Sending %s', data)
        headers = {'Content-type': 'application/soap+xml; charset=utf-8'}
        headers.update(extra_headers)
        return Request(self.base_url, data, headers, method='POST')

    def send(self, pickup_info, service_info):
        request_data = [pickup_info.as_dict(), service_info.as_dict()]
        req = self.make_http_request('mrw:TransmEnvio', request_data)
        with urlopen(req) as response:
            body = response.read()
            logger.debug('Received %s', body)

        response = self.parser.parse(body)
        message = response[soap('Envelope')][0][soap('Body')][0]
        response_obj = SendResponse.from_raw(message)
        return response_obj.result

    def get_label(self, get_label_request):
        request_data = [get_label_request.as_dict()[None]]
        req = self.make_http_request('mrw:GetEtiquetaEnvio', request_data)
        with urlopen(req) as response:
            body = response.read()
            logger.debug('Received %s', body)

        response = self.parser.parse(body)
        message = response[soap('Envelope')][0][soap('Body')][0]
        response_obj = LabelResponse.from_raw(message)
        return response_obj.result
<|endoftext|>