<commit_before>#!/usr/bin/env python

import requests
import urllib
import json
import os
import re

from flask import Flask
from flask import request
from flask import make_response
from datetime import datetime as DateTime, timedelta as TimeDelta

# Flask app should start in global layout
app = Flask(__name__)

@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = makeWebhookResult(req)

    res = json.dumps(res, indent=4)
    print(res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def makeWebhookResult(req):
    if req.get("result").get("action") == "browse.search.products":
        result = req.get("result")
        parameters = result.get("parameters")
        color = parameters.get("color")
        cat = parameters.get("catalog-category")
        if (((color is None) or (color is "")) and (req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            return {
                "data": {
                    "facebook": {
                        "text": "Pick a color:",
                        "quick_replies": [
                            {
                                "content_type": "text",
                                "title": "Red",
                                "payload": "red"
                            },
                            {
                                "content_type": "text",
                                "title": "Green",
                                "payload": "green"
                            }
                        ]
                    }
                }
            }
        else:
            rq = requests.get("http://www.lanebryant.com/lanebryant/search?Ntt=" + color + " " + cat + "&format=JSON")
            jdata = json.loads(rq.text)
            speech = "I found " + str(jdata["contents"][0]["MainContent"][0]["MainContent"][0]["contents"][0]["totalNumRecs"]) + " " + color + " " + cat + " products."
    elif req.get("result").get("action") == "promos":
        result = req.get("result")
        headers = {'HOST': 'sit.catherines.com'}
        rq = requests.get("https://23.34.4.174/static/promo_01?format=json", headers=headers, verify=False)
        jdata = json.loads(rq.text)
        speech = "Promos are "
        for mc in jdata["MainContent"]:
            speech = speech + str(mc["freeFormContent"]) + "... "
    elif ((req.get("result").get("action") == "Order_Status_yes") or (req.get("result").get("action") == "checkout.order.status")):
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        ordernum = parameters.get("order-number")

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn’t find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if status == 'Shipped':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        elif status == 'Canceled':
            speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
        elif status == 'Processing':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        else:
            speech = status
    elif req.get("result").get("action") == "Order_Status_no":
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        ordernum = 'OJTW022967052'

        print ("Zipcode is ", zipcode)

        if zipcode == '20166':
            ordernum = 'OJTW022967052'
        elif zipcode == '37122':
            ordernum = 'OJTW027567667'
        elif zipcode == '19148':
            ordernum = 'OJTW027678055'

        print ("Order number is ", ordernum)

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn’t find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if status == 'Shipped':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        elif status == 'Canceled':
            speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
        elif status == 'Processing':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        else:
            speech = status
    else:
        return{}
    print("Response:")
    print(speech)
    return {
        "speech": speech,
        "displayText": speech,
        #"data": {},
        # "contextOut": [],
        "source": "apiai-onlinestore-search"
    }

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print ("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<commit_msg>Updating the NO logic for order details<commit_after>#!/usr/bin/env python

import requests
import urllib
import json
import os
import re

from flask import Flask
from flask import request
from flask import make_response
from datetime import datetime as DateTime, timedelta as TimeDelta

# Flask app should start in global layout
app = Flask(__name__)

@app.route('/webhook', methods=['POST'])
def webhook():
    req = request.get_json(silent=True, force=True)

    print("Request:")
    print(json.dumps(req, indent=4))

    res = makeWebhookResult(req)

    res = json.dumps(res, indent=4)
    print(res)
    r = make_response(res)
    r.headers['Content-Type'] = 'application/json'
    return r

def makeWebhookResult(req):
    if req.get("result").get("action") == "browse.search.products":
        result = req.get("result")
        parameters = result.get("parameters")
        color = parameters.get("color")
        cat = parameters.get("catalog-category")
        if (((color is None) or (color is "")) and (req.get("originalRequest") is not None) and (req.get("originalRequest").get("source") == "facebook")):
            return {
                "data": {
                    "facebook": {
                        "text": "Pick a color:",
                        "quick_replies": [
                            {
                                "content_type": "text",
                                "title": "Red",
                                "payload": "red"
                            },
                            {
                                "content_type": "text",
                                "title": "Green",
                                "payload": "green"
                            }
                        ]
                    }
                }
            }
        else:
            rq = requests.get("http://www.lanebryant.com/lanebryant/search?Ntt=" + color + " " + cat + "&format=JSON")
            jdata = json.loads(rq.text)
            speech = "I found " + str(jdata["contents"][0]["MainContent"][0]["MainContent"][0]["contents"][0]["totalNumRecs"]) + " " + color + " " + cat + " products."
    elif req.get("result").get("action") == "promos":
        result = req.get("result")
        headers = {'HOST': 'sit.catherines.com'}
        rq = requests.get("https://23.34.4.174/static/promo_01?format=json", headers=headers, verify=False)
        jdata = json.loads(rq.text)
        speech = "Promos are "
        for mc in jdata["MainContent"]:
            speech = speech + str(mc["freeFormContent"]) + "... "
    elif ((req.get("result").get("action") == "Order_Status_yes") or (req.get("result").get("action") == "checkout.order.status")):
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        ordernum = parameters.get("order-number")

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn’t find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if status == 'Shipped':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        elif status == 'Canceled':
            speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
        elif status == 'Processing':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        else:
            speech = status
    elif req.get("result").get("action") == "Order_Status_no":
        result = req.get("result")
        parameters = result.get("parameters")
        zipcode = parameters.get("zipcode")
        email = parameters.get("email")
        ordertime = parameters.get("order-time")

        #TODO - Need to put here the API to get order number using email, zipcode and the timeframe
        if zipcode == '20166':
            ordernum = 'OJTW022967052'
        elif zipcode == '37122':
            ordernum = 'OJTW027567667'
        elif zipcode == '19148':
            ordernum = 'OJTW027678055'
        else
            ordernum = 'OJTW022967052'

        rq = requests.post("https://www.shopjustice.com/justice/homepage/includes/order-response-html.jsp", data={'orderNum': ordernum, 'billingZip': zipcode, 'Action': 'fetchODDetails'})
        #print rq.text
        matchObj = rq.text[rq.text.find("mar-status")+12:rq.text.find("<", rq.text.find("mar-status"))]
        matchDate = rq.text[rq.text.find("mar-date")+10:rq.text.find("<", rq.text.find("mar-date"))]
        if len(matchObj) < 50:
            print ("matchObj : ", matchObj)
            print ("matchDate : ", matchDate)
            status = matchObj
            date = DateTime.strptime(matchDate, '%m/%d/%Y') + TimeDelta(days=5)
        else:
            status = "I couldn’t find that order. Either the number or the zipcode is not correct."
            print ("No match!!")

        if status == 'Shipped':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        elif status == 'Canceled':
            speech = "Order status is " + status + ". Please reach out to the customer support for more details about the order."
        elif status == 'Processing':
            speech = "Order status is " + status + ". You shall receive the package by " + date.strftime('%m/%d/%Y') + "."
        else:
            speech = status
    else:
        return{}
    print("Response:")
    print(speech)
    return {
        "speech": speech,
        "displayText": speech,
        #"data": {},
        # "contextOut": [],
        "source": "apiai-onlinestore-search"
    }

if __name__ == '__main__':
    port = int(os.getenv('PORT', 5000))

    print ("Starting app on port %d" % port)

    app.run(debug=True, port=port, host='0.0.0.0')
<|endoftext|>


<commit_before>"""spark-error-selector

This app will look for keywords in the messages from a kafka topic,
and then take those messages and send them to another topic.
"""
import argparse
import json

import kafka
import pyspark
from pyspark import streaming
from pyspark.streaming import kafka as kstreaming

class FilterStreamProcessor():
    """A class to filter Kafka stream data

    This class wraps the Spark and Kafka specific logic for creating a
    DStream and applying processing functions to the RDDs contained
    therein.

    If any message contains a word from the FILTER_LIST, those messages
    will be copied to the output topic.
    """
    FILTER_LIST = (
        'ERROR',
        'Error',
        'error',
        'WARN',
        'warn',
        'failed',
        'failure',
    )

    def __init__(self, input_topic, output_topic, count_topic, servers, duration):
        """Create a new StreamProcessor

        Keyword arguments:
        input_topic -- Kafka topic to read messages from
        output_topic -- Kafka topic to write message to
        count_topic -- Kafka topic to write message counts to
        servers -- A list of Kafka brokers
        duration -- The window duration to sample the Kafka stream in
                    seconds
        """
        self.input_topic = input_topic
        self.output_topic = output_topic
        self.count_topic = count_topic
        self.servers = servers
        self.spark_context = pyspark.SparkContext(
            appName='spark-error-selector')
        self.streaming_context = streaming.StreamingContext(
            self.spark_context, duration)
        self.kafka_stream = kstreaming.KafkaUtils.createDirectStream(
            self.streaming_context,
            [self.input_topic],
            {'bootstrap.servers': self.servers})

    def configure_processing(self):
        """Configure the processing pipeline

        This function contains all the stream processing
        configuration that will affect how each RDD is processed.
        It will be called before the stream listener is started.
        """
        def send_filtered(rdd):
            """A function to publish an RDD to a Kafka topic"""
            producer = kafka.KafkaProducer(bootstrap_servers=self.servers)
            for r in rdd.collect():
                try:
                    totalcount.add(1)
                    record = r.encode('ascii', 'backslashreplace')
                    if any(word in record for word in FilterStreamProcessor.FILTER_LIST):
                        producer.send(self.output_topic, record)
                        filtercount.add(1)
                except Exception as e:
                    print('Error sending collected RDD')
                    print('Original exception: {}'.format(e))
            counts = {'counts':{'total': totalcount.value, 'filter': filtercount.value}}
            producer.send(self.count_topic, json.dumps(counts))
            producer.flush()

        totalcount = self.spark_context.accumulator(0)
        filtercount = self.spark_context.accumulator(0)
        messages = self.kafka_stream.map(lambda m: m[1])
        messages.foreachRDD(send_filtered)
        #totalcount = messages.count()
        #filtermessages = messages.filter(
        #    lambda r: any(word in r for word in FilterStreamProcessor.FILTER_LIST))
        #filtercount = filtermessages.count()
        #filtermessages.foreachRDD(send_filtered)

    def start_and_await_termination(self):
        """Start the stream processor

        This function will start the Spark-based stream processor,
        it will run until `stop` is called or an exception is
        thrown.
        """
        self.configure_processing()
        self.streaming_context.start()
        self.streaming_context.awaitTermination()

    def stop(self):
        """Stop the stream processor"""
        self.streaming_context.stop()

def main():
    """The main function

    This will process the command line arguments and launch the main
    Spark/Kafka monitor class.
    """
    parser = argparse.ArgumentParser(
        description='process data with Spark, using Kafka as the transport')
    parser.add_argument('--in', dest='input_topic',
        help='the kafka topic to read data from')
    parser.add_argument('--out', dest='output_topic',
        help='the kafka topic to publish data to')
    parser.add_argument('--count', dest='count_topic',
        help='the kafka topic to publish count information to')
    parser.add_argument('--servers', help='the kafka brokers')
    args = parser.parse_args()

    processor = FilterStreamProcessor(
        input_topic = args.input_topic,
        output_topic = args.output_topic,
        count_topic = args.count_topic,
        servers = args.servers,
        duration = 3)

    processor.start_and_await_termination()

if __name__ == '__main__':
    main()

<commit_msg>clean up comments<commit_after>"""spark-error-selector

This app will look for keywords in the messages from a kafka topic,
and then take those messages and send them to another topic.
"""
import argparse
import json

import kafka
import pyspark
from pyspark import streaming
from pyspark.streaming import kafka as kstreaming

class FilterStreamProcessor():
    """A class to filter Kafka stream data

    This class wraps the Spark and Kafka specific logic for creating a
    DStream and applying processing functions to the RDDs contained
    therein.

    If any message contains a word from the FILTER_LIST, those messages
    will be copied to the output topic.
    """
    FILTER_LIST = (
        'ERROR',
        'Error',
        'error',
        'WARN',
        'warn',
        'failed',
        'failure',
    )

    def __init__(self, input_topic, output_topic, count_topic, servers, duration):
        """Create a new StreamProcessor

        Keyword arguments:
        input_topic -- Kafka topic to read messages from
        output_topic -- Kafka topic to write message to
        count_topic -- Kafka topic to write message counts to
        servers -- A list of Kafka brokers
        duration -- The window duration to sample the Kafka stream in
                    seconds
        """
        self.input_topic = input_topic
        self.output_topic = output_topic
        self.count_topic = count_topic
        self.servers = servers
        self.spark_context = pyspark.SparkContext(
            appName='spark-error-selector')
        self.streaming_context = streaming.StreamingContext(
            self.spark_context, duration)
        self.kafka_stream = kstreaming.KafkaUtils.createDirectStream(
            self.streaming_context,
            [self.input_topic],
            {'bootstrap.servers': self.servers})

    def configure_processing(self):
        """Configure the processing pipeline

        This function contains all the stream processing
        configuration that will affect how each RDD is processed.
        It will be called before the stream listener is started.
        """
        def send_filtered(rdd):
            """A function to publish an RDD to a Kafka topic"""
            producer = kafka.KafkaProducer(bootstrap_servers=self.servers)
            for r in rdd.collect():
                try:
                    totalcount.add(1)
                    # this encode is helpful for strings that come in with
                    # characters that mess up the kafka send, it could
                    # be improved to handle unicode more efficiently.
                    record = r.encode('ascii', 'backslashreplace')
                    if any(word in record for word in FilterStreamProcessor.FILTER_LIST):
                        producer.send(self.output_topic, record)
                        filtercount.add(1)
                except Exception as e:
                    print('Error sending collected RDD')
                    print('Original exception: {}'.format(e))
            counts = {'counts':{'total': totalcount.value, 'filter': filtercount.value}}
            producer.send(self.count_topic, json.dumps(counts))
            producer.flush()

        totalcount = self.spark_context.accumulator(0)
        filtercount = self.spark_context.accumulator(0)
        messages = self.kafka_stream.map(lambda m: m[1])
        messages.foreachRDD(send_filtered)

    def start_and_await_termination(self):
        """Start the stream processor

        This function will start the Spark-based stream processor,
        it will run until `stop` is called or an exception is
        thrown.
        """
        self.configure_processing()
        self.streaming_context.start()
        self.streaming_context.awaitTermination()

    def stop(self):
        """Stop the stream processor"""
        self.streaming_context.stop()

def main():
    """The main function

    This will process the command line arguments and launch the main
    Spark/Kafka monitor class.
    """
    parser = argparse.ArgumentParser(
        description='process data with Spark, using Kafka as the transport')
    parser.add_argument('--in', dest='input_topic',
        help='the kafka topic to read data from')
    parser.add_argument('--out', dest='output_topic',
        help='the kafka topic to publish data to')
    parser.add_argument('--count', dest='count_topic',
        help='the kafka topic to publish count information to')
    parser.add_argument('--servers', help='the kafka brokers')
    args = parser.parse_args()

    processor = FilterStreamProcessor(
        input_topic = args.input_topic,
        output_topic = args.output_topic,
        count_topic = args.count_topic,
        servers = args.servers,
        duration = 3)

    processor.start_and_await_termination()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from math import ceil
import json
import datetime
import os

import requests
from flask import Flask, render_template, request, g, redirect, url_for
from peewee import *

SECRET_KEY = 'hin6bab8ge25*r=x&+5$0kn=-#log$pt^#@vrqjld!^2ci@g*b'

app = Flask(__name__)
app.config.from_object(__name__)

db = os.path.dirname(os.path.abspath(__file__)) + '/zhihudaily.db'
database = SqliteDatabase(db)

class BaseModel(Model):
    class Meta:
        database = database

class Zhihudaily(BaseModel):
    date = IntegerField()
    json_news = CharField()
    display_date = CharField()

def create_tables():
    database.connect()
    database.create_tables([Zhihudaily])

@app.before_request
def before_request():
    g.db = database
    g.db.connect()

@app.after_request
def after_request(response):
    g.db.close()
    return response

@app.route('/before/<date>')
def before(date):
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get(
        'http://news.at.zhihu.com/api/1.2/news/before/{0}'.format(date))
    display_date = r.json()['display_date']
    today = datetime.date.today().strftime('%Y%m%d')
    strdate = r.json()["date"]
    day_after = (datetime.datetime.strptime(date, '%Y%m%d') + datetime.timedelta(1)).strftime('%Y%m%d')
    if int(today) < int(date):
        if request.args['image']:
            return redirect(url_for('with_image'))
        else:
            return redirect(url_for('index'))
    is_today = r.json().get('is_today', False)
    news_list = [item for item in r.json()['news']]
    if request.args['image']:
        return render_template('with_image.html', lists=news_list,
                               display_date=display_date, date=strdate,
                               is_today=is_today, day_after=day_after)
    else:
        return render_template("index.html", lists=news_list,
                               display_date=display_date, date=strdate,
                               is_today=is_today, day_after=day_after)

@app.route('/')
def index():
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()['date']
    news_list = [item for item in r.json()['news']]
    return render_template("index.html", lists=news_list,
                           display_date=display_date, date=date,
                           is_today=True)

@app.route('/withimage')
def with_image():
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()["date"]
    news_list = [item for item in r.json()['news']]
    request.environ['Referer'] = 'http://daily.zhihu.com/'
    return render_template('with_image.html', lists=news_list,
                           display_date=display_date, date=date)

@app.route('/pages')
@app.route('/pages/<int:page>')
def pages(page=1):
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()["date"]
    news_list = [item for item in r.json()['news']]
    request.environ['Referer'] = 'http://daily.zhihu.com/'
    news = Zhihudaily.select().paginate(page, 7)
    records = []
    for i in news:
        temp = json.loads(i.json_news)
        records.append({"date": i.date, "news": temp, "display_date": i.display_date})
    pages = int(ceil(Zhihudaily.select().count() / 7))
    return render_template('pages.html', lists=news_list,
                           display_date=display_date, date=date,
                           page=page, records=records, pages=pages)

@app.route('/pagination')
def pagination():
    pages = int(ceil(Zhihudaily.select().count() / 7))
    return render_template('pagination.html', pages=pages)

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>Pep8 check; bugfix for the before view<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from math import ceil
import json
import datetime
import os

import requests
from flask import Flask, render_template, request, g, redirect, url_for
from peewee import *

SECRET_KEY = 'hin6bab8ge25*r=x&+5$0kn=-#log$pt^#@vrqjld!^2ci@g*b'

app = Flask(__name__)
app.config.from_object(__name__)

db = os.path.dirname(os.path.abspath(__file__)) + '/zhihudaily.db'
database = SqliteDatabase(db)

class BaseModel(Model):
    class Meta:
        database = database

class Zhihudaily(BaseModel):
    date = IntegerField()
    json_news = CharField()
    display_date = CharField()

def create_tables():
    database.connect()
    database.create_tables([Zhihudaily])

@app.before_request
def before_request():
    g.db = database
    g.db.connect()

@app.after_request
def after_request(response):
    g.db.close()
    return response

@app.route('/before/<date>')
def before(date):
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get(
        'http://news.at.zhihu.com/api/1.2/news/before/{0}'.format(date))
    display_date = r.json()['display_date']
    today = datetime.date.today().strftime('%Y%m%d')
    strdate = r.json()["date"]
    day_after = (
        datetime.datetime.strptime(date, '%Y%m%d') + datetime.timedelta(1)
    ).strftime('%Y%m%d')
    if int(today) < int(date):
        if request.args['image'] == 'True':
            return redirect(url_for('with_image'))
        else:
            return redirect(url_for('index'))
    is_today = r.json().get('is_today', False)
    news_list = [item for item in r.json()['news']]
    if request.args['image'] == 'True':
        return render_template('with_image.html', lists=news_list,
                               display_date=display_date, date=strdate,
                               is_today=is_today, day_after=day_after)
    else:
        return render_template("index.html", lists=news_list,
                               display_date=display_date, date=strdate,
                               is_today=is_today, day_after=day_after)

@app.route('/')
def index():
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()['date']
    news_list = [item for item in r.json()['news']]
    return render_template("index.html", lists=news_list,
                           display_date=display_date, date=date,
                           is_today=True)

@app.route('/withimage')
def with_image():
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()["date"]
    news_list = [item for item in r.json()['news']]
    request.environ['Referer'] = 'http://daily.zhihu.com/'
    return render_template('with_image.html', lists=news_list,
                           display_date=display_date, date=date,
                           is_today=True)

@app.route('/pages')
@app.route('/pages/<int:page>')
def pages(page=1):
    session = requests.Session()
    session.headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux \
                            x86_64; rv:28.0) Gecko/20100101 Firefox/28.0'})
    r = session.get('http://news.at.zhihu.com/api/1.2/news/latest')
    display_date = r.json()['display_date']
    date = r.json()["date"]
    news_list = [item for item in r.json()['news']]
    request.environ['Referer'] = 'http://daily.zhihu.com/'
    news = Zhihudaily.select().paginate(page, 7)
    records = []
    for i in news:
        temp = json.loads(i.json_news)
        records.append({"date": i.date, "news": temp,
                        "display_date": i.display_date})
    pages = int(ceil(Zhihudaily.select().count() / 7))
    return render_template('pages.html', lists=news_list,
                           display_date=display_date, date=date,
                           page=page, records=records, pages=pages)

@app.route('/pagination')
def pagination():
    pages = int(ceil(Zhihudaily.select().count() / 7))
    return render_template('pagination.html', pages=pages)

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import division
import os
import sys
import time
import logging
import tempfile
import argparse
import subprocess
import distutils.spawn
import consensus

REQUIRED_COMMANDS = ['mafft']
OPT_DEFAULTS = {'min_reads':3, 'processes':1}
USAGE = "%(prog)s [options]"
DESCRIPTION = """Build consensus sequences from read families. Pipe sorted reads into stdin. Prints
single-strand consensus sequences in FASTA to stdout. The sequence names are BARCODE.MATE.SEQS, e.g.
"CTCAGATAACATACCTTATATGCA.1:7", where "BARCODE" is the input barcode, "MATE" is "1" for the first
read in the pair and "2" for the second, and "SEQS" is the number of reads in the family."""

def main(argv):

  parser = argparse.ArgumentParser(description=DESCRIPTION)
  parser.set_defaults(**OPT_DEFAULTS)

  parser.add_argument('infile', metavar='read-families.tsv', nargs='?',
    help='The input reads, sorted into families. One line per read pair, 8 tab-delimited columns: '
         '1. canonical barcode, 2. barcode order ("ab" for alpha+beta, "ba" for beta-alpha) 3. '
         'read 1 name, 4. read 1 sequence, 5. read 1 quality scores, 6. read 2 name, 7. read 2 '
         'sequence, 8. read 2 quality scores.')
  parser.add_argument('-r', '--min-reads', type=int,
    help='The minimum number of reads (from each strand) required to form a family. Families with '
         'fewer reads will be skipped. Default: %(default)s.')
  parser.add_argument('-m', '--msa', action='store_true',
    help='Print the multiple sequence alignment as well as the consensus. Instead of printing '
         'FASTA, it will print a tab-delimited format with 3 columns: 1. barcode, 2. read name, 3. '
         'aligned read sequence. One line will be the consensus sequence, named "CONSENSUS". One '
         'alignment and consensus will be printed for each read in the pair.')
  parser.add_argument('-s', '--stats-file',
    help='Print statistics on the run to this file. Use "-" to print to stderr.')
  parser.add_argument('-p', '--processes', type=int,
    help='Number of processes to use. If > 1, launches this many worker subprocesses. '
         'Default: %(default)s.')
  parser.add_argument('-S', '--slurm', action='store_true',
    help='If -p > 1, prepend sub-commands with "srun -C new".')

  args = parser.parse_args(argv[1:])

  assert args.processes > 0, '-p must be greater than zero'

  # Check for required commands.
  missing_commands = []
  if args.slurm:
    REQUIRED_COMMANDS.append('srun')
  for command in REQUIRED_COMMANDS:
    if not distutils.spawn.find_executable(command):
      missing_commands.append(command)
  if missing_commands:
    fail('Error: Missing commands: "'+'", "'.join(missing_commands)+'".')

  if args.infile:
    infile = open(args.infile)
  else:
    infile = sys.stdin

  if args.stats_file:
    if args.stats_file == '-':
      logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(message)s')
    else:
      logging.basicConfig(filename=args.stats_file, filemode='w', level=logging.INFO,
                          format='%(message)s')
  else:
    logging.disable(logging.CRITICAL)

  # Open all the worker processes, if we're using more than one.
  workers = None
  if args.processes > 1:
    workers = open_workers(args.processes, args)

  stats = {'time':0, 'pairs':0, 'runs':0, 'families':0}
  all_pairs = 0
  family = []
  barcode = None
  order = None
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    if len(fields) != 8:
      continue
    (this_barcode, this_order, name1, seq1, qual1, name2, seq2, qual2) = fields
    # If the barcode has changed, we're in a new family.
    # Process the reads we've previously gathered as one family and start a new family.
    if this_barcode != barcode or this_order != order:
      process_family(family, barcode, order, args, workers=workers, stats=stats)
      barcode = this_barcode
      order = this_order
      family = []
    family.append((name1, seq1, qual1, name2, seq2, qual2))
    all_pairs += 1
  # Process the last family.
  process_family(family, barcode, order, args, workers=workers, stats=stats)

  if args.processes > 1:
    close_workers(workers)
    compile_results(workers)
    delete_tempfiles(workers)

  if infile is not sys.stdin:
    infile.close()

  if not args.stats_file:
    return

  # Final stats on the run.
  logging.info('Processed {} read pairs and {} multi-pair families.'
               .format(all_pairs, stats['runs']))
  per_pair = stats['time'] / stats['pairs']
  per_run = stats['time'] / stats['runs']
  logging.info('{:0.3f}s per pair, {:0.3f}s per run.'.format(per_pair, per_run))

def open_workers(num_workers, args):
  """Open the required number of worker processes."""
  script_path = os.path.realpath(sys.argv[0])
  workers = []
  for i in range(num_workers):
    if args.slurm:
      command = ['srun', '-C', 'new', 'python', script_path]
    else:
      command = ['python', script_path]
    arguments = gather_args(sys.argv, args.infile)
    command.extend(arguments)
    stats_subfile = None
    if args.stats_file:
      if args.stats_file == '-':
        stats_subfile = '-'
      else:
        stats_subfile = "{}.{}.log".format(args.stats_file, i)
      command.extend(['-s', stats_subfile])
    outfile = tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.out.part.')
    process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=outfile)
    worker = {'proc':process, 'outfile':outfile, 'stats':stats_subfile}
    workers.append(worker)
  return workers

def gather_args(args, infile, excluded_flags=('-S', '--slurm'),
                excluded_args=('-p', '--processes', '-s', '--stats-file')):
  """Take the full list of command-line arguments and return only the ones which
  should be passed to worker processes.
  Excludes the 0th argument (the command name), the input filename ("infile"), all
  arguments in "excluded_flags", and all arguments in "excluded_args" plus the
  arguments which follow."""
  out_args = []
  skip = True
  for arg in args:
    if skip:
      skip = False
      continue
    if arg in excluded_flags:
      continue
    if arg in excluded_args:
      skip = True
      continue
    if arg == infile:
      continue
    out_args.append(arg)
  return out_args

def delegate(worker, family, barcode, order):
  """Send a family to a worker process."""
  for pair in family:
    line = '{}\t{}\t{}\n'.format(barcode, order, '\t'.join(pair))
    worker['proc'].stdin.write(line)

def close_workers(workers):
  for worker in workers:
    worker['outfile'].close()
    worker['proc'].stdin.close()

def compile_results(workers):
  for worker in workers:
    worker['proc'].wait()
    with open(worker['outfile'].name, 'r') as outfile:
      for line in outfile:
        sys.stdout.write(line)

def delete_tempfiles(workers):
  for worker in workers:
    os.remove(worker['outfile'].name)
    if worker['stats']:
      os.remove(worker['stats'])

def process_family(family, barcode, order, args, workers=None, stats=None):
  # Pass if family doesn't contain minimum # of reads.
  if len(family) < args.min_reads:
    return
  stats['families'] += 1
  # Are we the controller process or a worker?
  if args.processes > 1:
    i = stats['families'] % len(workers)
    worker = workers[i]
    delegate(worker, family, barcode, order)
    return
  # We're a worker. Actually process the family.
  start = time.time()
  # If order is beta+alpha, reverse it to get the "true" barcode (alpha+beta).
  if order == 'ba':
    half = len(barcode)//2
    barcode = barcode[half:] + barcode[:half]
  pairs = len(family)
  if pairs == 1:
    (name1, seq1, qual1, name2, seq2, qual2) = family[0]
    if args.msa:
      print '{bar}\tCONSENSUS\t{seq}\n{bar}\t{name}\t{seq}'.format(bar=barcode, name=name1, seq=seq1)
      print '{bar}\tCONSENSUS\t{seq}\n{bar}\t{name}\t{seq}'.format(bar=barcode, name=name2, seq=seq2)
    else:
      print '>'+barcode+'.1:1'
      print seq1
      print '>'+barcode+'.2:1'
      print seq2
  else:
    output = 'consensus'
    if args.msa:
      output = 'msa'
    align_and_cons(family, barcode, 1, output=output)
    align_and_cons(family, barcode, 2, output=output)
  end = time.time()
  elapsed = end - start
  logging.info('{} sec for {} read pairs.'.format(elapsed, pairs))
  if stats and pairs > 1:
    stats['time'] += elapsed
    stats['pairs'] += pairs
    stats['runs'] += 1
  return

def align_and_cons(family, barcode, mate, output='consensus'):
  """Do the bioinformatic work: make the multiple sequence alignment and consensus
  sequence.
  "mate" is "1" or "2" and determines which read in the pair to process.
  "output" is "consensus" to print the consensus sequence in FASTA format, or
    "msa" to print the full multiple sequence alignment in tab-delimited format."""
  align_raw = make_msa(family, mate)
  if align_raw is None:
    logging.warning('Error aligning family {} (read {}).'.format(barcode, mate))
    return
  align = read_fasta(align_raw, is_file=False)
  cons = consensus.get_consensus([read['seq'] for read in align])
  if output == 'consensus' and cons is not None:
    print '>{}.{}:{}'.format(barcode, mate, len(family))
    print cons
  elif output == 'msa':
    print '{}\t{}\t{}'.format(barcode, 'CONSENSUS', cons)
    for sequence in align:
      print '{}\t{}\t{}'.format(barcode, sequence['name'], sequence['seq'])

def make_msa(family, mate):
  """Perform a multiple sequence alignment on a set of sequences.
  Uses MAFFT."""
  #TODO: Replace with tempfile.mkstemp()?
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as family_file:
    for pair in family:
      if mate == 1:
        name = pair[0]
        seq = pair[1]
      else:
        name = pair[3]
        seq = pair[4]
      family_file.write('>'+name+'\n')
      family_file.write(seq+'\n')
  with open(os.devnull, 'w') as devnull:
    try:
      command = ['mafft', '--nuc', '--quiet', family_file.name]
      output = subprocess.check_output(command, stderr=devnull)
    except (OSError, subprocess.CalledProcessError):
      return None
  os.remove(family_file.name)
  return output

def read_fasta(fasta, is_file=True):
  """Quick and dirty FASTA parser. Return the sequences and their names.
  Returns a list of sequences. Each is a dict of 'name' and 'seq'.
  Warning: Reads the entire contents of the file into memory at once."""
  sequences = []
  seq_lines = []
  seq_name = None
  if is_file:
    with open(fasta) as fasta_file:
      fasta_lines = fasta_file.readlines()
  else:
    fasta_lines = fasta.splitlines()
  for line in fasta_lines:
    if line.startswith('>'):
      if seq_lines:
        sequences.append({'name':seq_name, 'seq':''.join(seq_lines)})
      seq_lines = []
      seq_name = line.rstrip('\r\n')[1:]
      continue
    seq_lines.append(line.strip())
  if seq_lines:
    sequences.append({'name':seq_name, 'seq':''.join(seq_lines)})
  return sequences

def fail(message):
  sys.stderr.write(message+"\n")
  sys.exit(1)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>sscs.py: Change each pair to a dict, not a tuple. Format confusing msa printing code.<commit_after>#!/usr/bin/env python
from __future__ import division
import os
import sys
import time
import logging
import tempfile
import argparse
import subprocess
import distutils.spawn
import consensus

REQUIRED_COMMANDS = ['mafft']
OPT_DEFAULTS = {'min_reads':3, 'processes':1}
USAGE = "%(prog)s [options]"
DESCRIPTION = """Build consensus sequences from read families. Pipe sorted reads into stdin. Prints
single-strand consensus sequences in FASTA to stdout. The sequence names are BARCODE.MATE.SEQS, e.g.
"CTCAGATAACATACCTTATATGCA.1:7", where "BARCODE" is the input barcode, "MATE" is "1" for the first
read in the pair and "2" for the second, and "SEQS" is the number of reads in the family."""

def main(argv):

  parser = argparse.ArgumentParser(description=DESCRIPTION)
  parser.set_defaults(**OPT_DEFAULTS)

  parser.add_argument('infile', metavar='read-families.tsv', nargs='?',
    help='The input reads, sorted into families. One line per read pair, 8 tab-delimited columns: '
         '1. canonical barcode, 2. barcode order ("ab" for alpha+beta, "ba" for beta-alpha) 3. '
         'read 1 name, 4. read 1 sequence, 5. read 1 quality scores, 6. read 2 name, 7. read 2 '
         'sequence, 8. read 2 quality scores.')
  parser.add_argument('-r', '--min-reads', type=int,
    help='The minimum number of reads (from each strand) required to form a family. Families with '
         'fewer reads will be skipped. Default: %(default)s.')
  parser.add_argument('-m', '--msa', action='store_true',
    help='Print the multiple sequence alignment as well as the consensus. Instead of printing '
         'FASTA, it will print a tab-delimited format with 3 columns: 1. barcode, 2. read name, 3. '
         'aligned read sequence. One line will be the consensus sequence, named "CONSENSUS". One '
         'alignment and consensus will be printed for each read in the pair.')
  parser.add_argument('-s', '--stats-file',
    help='Print statistics on the run to this file. Use "-" to print to stderr.')
  parser.add_argument('-p', '--processes', type=int,
    help='Number of processes to use. If > 1, launches this many worker subprocesses. '
         'Default: %(default)s.')
  parser.add_argument('-S', '--slurm', action='store_true',
    help='If -p > 1, prepend sub-commands with "srun -C new".')

  args = parser.parse_args(argv[1:])

  assert args.processes > 0, '-p must be greater than zero'

  # Check for required commands.
  missing_commands = []
  if args.slurm:
    REQUIRED_COMMANDS.append('srun')
  for command in REQUIRED_COMMANDS:
    if not distutils.spawn.find_executable(command):
      missing_commands.append(command)
  if missing_commands:
    fail('Error: Missing commands: "'+'", "'.join(missing_commands)+'".')

  if args.infile:
    infile = open(args.infile)
  else:
    infile = sys.stdin

  if args.stats_file:
    if args.stats_file == '-':
      logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(message)s')
    else:
      logging.basicConfig(filename=args.stats_file, filemode='w', level=logging.INFO,
                          format='%(message)s')
  else:
    logging.disable(logging.CRITICAL)

  # Open all the worker processes, if we're using more than one.
  workers = None
  if args.processes > 1:
    workers = open_workers(args.processes, args)

  stats = {'time':0, 'pairs':0, 'runs':0, 'families':0}
  all_pairs = 0
  family = []
  barcode = None
  order = None
  for line in infile:
    fields = line.rstrip('\r\n').split('\t')
    if len(fields) != 8:
      continue
    (this_barcode, this_order, name1, seq1, qual1, name2, seq2, qual2) = fields
    # If the barcode has changed, we're in a new family.
    # Process the reads we've previously gathered as one family and start a new family.
    if this_barcode != barcode or this_order != order:
      process_family(family, barcode, order, args, workers=workers, stats=stats)
      barcode = this_barcode
      order = this_order
      family = []
    pair = {'name1': name1, 'seq1':seq1, 'qual1':qual1, 'name2':name2, 'seq2':seq2, 'qual2':qual2}
    family.append(pair)
    all_pairs += 1
  # Process the last family.
  process_family(family, barcode, order, args, workers=workers, stats=stats)

  if args.processes > 1:
    close_workers(workers)
    compile_results(workers)
    delete_tempfiles(workers)

  if infile is not sys.stdin:
    infile.close()

  if not args.stats_file:
    return

  # Final stats on the run.
  logging.info('Processed {} read pairs and {} multi-pair families.'
               .format(all_pairs, stats['runs']))
  per_pair = stats['time'] / stats['pairs']
  per_run = stats['time'] / stats['runs']
  logging.info('{:0.3f}s per pair, {:0.3f}s per run.'.format(per_pair, per_run))

def open_workers(num_workers, args):
  """Open the required number of worker processes."""
  script_path = os.path.realpath(sys.argv[0])
  workers = []
  for i in range(num_workers):
    if args.slurm:
      command = ['srun', '-C', 'new', 'python', script_path]
    else:
      command = ['python', script_path]
    arguments = gather_args(sys.argv, args.infile)
    command.extend(arguments)
    stats_subfile = None
    if args.stats_file:
      if args.stats_file == '-':
        stats_subfile = '-'
      else:
        stats_subfile = "{}.{}.log".format(args.stats_file, i)
      command.extend(['-s', stats_subfile])
    outfile = tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.out.part.')
    process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=outfile)
    worker = {'proc':process, 'outfile':outfile, 'stats':stats_subfile}
    workers.append(worker)
  return workers

def gather_args(args, infile, excluded_flags=('-S', '--slurm'),
                excluded_args=('-p', '--processes', '-s', '--stats-file')):
  """Take the full list of command-line arguments and return only the ones which
  should be passed to worker processes.
  Excludes the 0th argument (the command name), the input filename ("infile"), all
  arguments in "excluded_flags", and all arguments in "excluded_args" plus the
  arguments which follow."""
  out_args = []
  skip = True
  for arg in args:
    if skip:
      skip = False
      continue
    if arg in excluded_flags:
      continue
    if arg in excluded_args:
      skip = True
      continue
    if arg == infile:
      continue
    out_args.append(arg)
  return out_args

def delegate(worker, family, barcode, order):
  """Send a family to a worker process."""
  for pair in family:
    line = '{}\t{}\t{name1}\t{seq1}\t{qual1}\t{name2}\t{seq2}\t{qual2}\n'.format(barcode, order,
                                                                                **pair)
    worker['proc'].stdin.write(line)

def close_workers(workers):
  for worker in workers:
    worker['outfile'].close()
    worker['proc'].stdin.close()

def compile_results(workers):
  for worker in workers:
    worker['proc'].wait()
    with open(worker['outfile'].name, 'r') as outfile:
      for line in outfile:
        sys.stdout.write(line)

def delete_tempfiles(workers):
  for worker in workers:
    os.remove(worker['outfile'].name)
    if worker['stats']:
      os.remove(worker['stats'])

def process_family(family, barcode, order, args, workers=None, stats=None):
  # Pass if family doesn't contain minimum # of reads.
  if len(family) < args.min_reads:
    return
  stats['families'] += 1
  # Are we the controller process or a worker?
  if args.processes > 1:
    i = stats['families'] % len(workers)
    worker = workers[i]
    delegate(worker, family, barcode, order)
    return
  # We're a worker. Actually process the family.
  start = time.time()
  # If order is beta+alpha, reverse it to get the "true" barcode (alpha+beta).
  if order == 'ba':
    half = len(barcode)//2
    barcode = barcode[half:] + barcode[:half]
  pairs = len(family)
  if pairs == 1:
    pair = family[0]
    if args.msa:
      print ('{bar}\tCONSENSUS\t{seq1}\n'
             '{bar}\t{name1}\t{seq1}\n'
             '{bar}\tCONSENSUS\t{seq2}\n'
             '{bar}\t{name2}\t{seq2}').format(bar=barcode, **pair)
    else:
      print '>'+barcode+'.1:1'
      print pair['seq1']
      print '>'+barcode+'.2:1'
      print pair['seq2']
  else:
    output = 'consensus'
    if args.msa:
      output = 'msa'
    align_and_cons(family, barcode, 1, output=output)
    align_and_cons(family, barcode, 2, output=output)
  end = time.time()
  elapsed = end - start
  logging.info('{} sec for {} read pairs.'.format(elapsed, pairs))
  if stats and pairs > 1:
    stats['time'] += elapsed
    stats['pairs'] += pairs
    stats['runs'] += 1
  return

def align_and_cons(family, barcode, mate, output='consensus'):
  """Do the bioinformatic work: make the multiple sequence alignment and consensus
  sequence.
  "mate" is "1" or "2" and determines which read in the pair to process.
  "output" is "consensus" to print the consensus sequence in FASTA format, or
    "msa" to print the full multiple sequence alignment in tab-delimited format."""
  mate = str(mate)
  assert mate == '1' or mate == '2'
  align_raw = make_msa(family, mate)
  if align_raw is None:
    logging.warning('Error aligning family {} (read {}).'.format(barcode, mate))
    return
  align = read_fasta(align_raw, is_file=False)
  cons = consensus.get_consensus([read['seq'] for read in align])
  if output == 'consensus' and cons is not None:
    print '>{}.{}:{}'.format(barcode, mate, len(family))
    print cons
  elif output == 'msa':
    print '{}\t{}\t{}'.format(barcode, 'CONSENSUS', cons)
    for sequence in align:
      print '{}\t{}\t{}'.format(barcode, sequence['name'], sequence['seq'])

def make_msa(family, mate):
  """Perform a multiple sequence alignment on a set of sequences.
  Uses MAFFT."""
  assert mate == '1' or mate == '2'
  #TODO: Replace with tempfile.mkstemp()?
  with tempfile.NamedTemporaryFile('w', delete=False, prefix='sscs.') as family_file:
    for pair in family:
      name = pair['name'+mate]
      seq = pair['seq'+mate]
      family_file.write('>'+name+'\n')
      family_file.write(seq+'\n')
  with open(os.devnull, 'w') as devnull:
    try:
      command = ['mafft', '--nuc', '--quiet', family_file.name]
      output = subprocess.check_output(command, stderr=devnull)
    except (OSError, subprocess.CalledProcessError):
      return None
  os.remove(family_file.name)
  return output

def read_fasta(fasta, is_file=True):
  """Quick and dirty FASTA parser. Return the sequences and their names.
  Returns a list of sequences. Each is a dict of 'name' and 'seq'.
  Warning: Reads the entire contents of the file into memory at once."""
  sequences = []
  seq_lines = []
  seq_name = None
  if is_file:
    with open(fasta) as fasta_file:
      fasta_lines = fasta_file.readlines()
  else:
    fasta_lines = fasta.splitlines()
  for line in fasta_lines:
    if line.startswith('>'):
      if seq_lines:
        sequences.append({'name':seq_name, 'seq':''.join(seq_lines)})
      seq_lines = []
      seq_name = line.rstrip('\r\n')[1:]
      continue
    seq_lines.append(line.strip())
  if seq_lines:
    sequences.append({'name':seq_name, 'seq':''.join(seq_lines)})
  return sequences

def fail(message):
  sys.stderr.write(message+"\n")
  sys.exit(1)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import config
import unsubscribe as unsub
from flask import Flask, render_template, request
app = Flask(__name__)
db = None

@app.route('/unsubscribe', methods=['GET', 'POST'])
def unsubscribe():
      if request.method == 'POST':
        email_address = request.form['email']
        db.add(email_address)
        return "%s unsubscribed!" % email_address
      else:
        return render_template('unsubscribe.html')

if __name__ == '__main__':
      db = unsub.getDatabase(config.DATABASE_URL)
      if db is None:
        print "Failed to connect to unsubscribe database url '%s'" % config.DATABASE_URL
        exit(1)
      app.run()
<commit_msg>s/exit/sys.exit<commit_after>#!/usr/bin/env python

import config
import unsubscribe as unsub
from flask import Flask, render_template, request
app = Flask(__name__)
db = None

@app.route('/unsubscribe', methods=['GET', 'POST'])
def unsubscribe():
      if request.method == 'POST':
        email_address = request.form['email']
        db.add(email_address)
        return "%s unsubscribed!" % email_address
      else:
        return render_template('unsubscribe.html')

if __name__ == '__main__':
      db = unsub.getDatabase(config.DATABASE_URL)
      if db is None:
        print "Failed to connect to unsubscribe database url '%s'" % config.DATABASE_URL
        sys.exit(1)
      app.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse, os, re, select, stat, subprocess, sys, tempfile, time
from util import *

'''
ssub is a simple job submission script

usage:
  cat list_of_commands.txt | ssub -n 100 -q hour -G gscidfolk -m 8 --io 10
  ssub -n 100 -q week -G broadfolk -m 8 --io 10 "command1; command2; command3;"

to use it as a python library:
  import ssub
  ssub.args.n = 100
  ssub.args.q = "hour"
  commands = get_commands()
  job_ids = submit_array(ssub.args, commands)
  wait_for_jobs(job_ids)
  print "done"

to create and run pipelines:
  import ssub
  ssub.args.q = "week"
  ssub.args.G = "broadfolk"
  commands1 = run_blast()
  commands2 = parse_blast()
  pipeline = [commands1, commands2]
  submit_pipeline(args, pipeline)
  print "done"

'''

# set global variables
config = ConfigParser.ConfigParser()
config.read('train.cfg')
username = config.get('User', 'username')
temp_dir = config.get('User', 'tmp_directory')
cluster= config.get('User', 'cluster')

def parse_args():
    # print usage statement
    usage = "cat list_of_commands.txt | ssub -n 100 -q hour -G gscidfolk -m 8 --io 10\n"
    usage +="ssub -n 100 -q week -G broadfolk -m 8 --io 10 'command 1; command 2; command 3;"

    # add command line arguments
    parser = argparse.ArgumentParser(usage = usage)
    parser.add_argument('-n', default = -1, type = int, help = 'number of cpus')
    parser.add_argument('-q', default = 'hour', help = 'queue')
    parser.add_argument('-G', default = 'gscidfolk', help = 'group')
    parser.add_argument('-m', default = 4, help = 'memory (gb)')
    parser.add_argument('-l', default = 200, help = 'job array slot limit')
    parser.add_argument('--io', default = 1, help = 'disk io (units)')
    parser.add_argument('commands', nargs = '?', default = '')

    # parse arguments from stdin
    if __name__ == '__main__':
        args = parser.parse_args()
    else:
        args = parser.parse_args('')
    return args

class Ssub():

    def __init__(self, cluster = 'coyote'):

        # get command line arguments
        args = parse_args()

        # initialize cluster parameters
        self.cluster = cluster
        self.username = username
        self.temp_dir = temp_dir
        self.header = '#!/bin/bash\n'
        self.l = args.l
        self.n = args.n
        self.m = args.m
        self.q = args.q
        self.G = args.G
        self.io = args.io
        self.commands = args.commands

        # broad parameters
        if cluster == 'broad':
            self.submit_cmd = 'bsub'
            self.stat_cmd = 'bjobs -w'
            self.parse_job = lambda x: re.search('Job <(\d+)>', x).group(1)

        # coyote parameters
        elif cluster == 'coyote':
            self.submit_cmd = 'qsub'
            self.stat_cmd = 'qstat -l'
            self.parse_job = lambda x: x.rstrip()

        # unrecognized cluster
        else:
            error('unrecognized cluster %s' %(cluster))

    def __repr__(self):
        return '\ncluster: %s\nusername: %s\ntemp_dir: %s\nn: %d\nm: %d\nq: %s\nG: %s\nio: %s\n' %(self.cluster, self.username, self.temp_dir, self.n, self.m, self.q, self.G, self.io)

    def __str__(self):
        a = self.__repr__()
        a = re.sub('\n', '\n  ', a)
        return a

    def mktemp(self, suffix = '.tmp'):
        # make temporary file and return [fh, fn]
        fh, fn = tempfile.mkstemp(suffix='.sh', dir=self.temp_dir)
        os.close(fh)
        fh = open(fn, 'w')
        fh.write(self.header)
        return fh, fn

    def job_status(self):
        # return job status
        process = subprocess.Popen([self.stat_cmd], stdout = subprocess.PIPE, shell=True)
        [out, err] = process.communicate()
        out = [line for line in out.split('\n') if self.username in line]
        return [out, err]

    def jobs_finished(self, job_ids):
        # check if jobs are finished
        [out, err] = self.job_status()
        for job in out:
            for job_id in job_ids:
                if job_id in job:
                    return False
        return True

    def write_jobs(self, commands):
        # write job scripts from a list of commands

        # initialize output files
        fhs, fns = zip(*[self.mktemp(suffix='.sh') for i in range(min(self.n, len(commands)))])
        fhs_cycle = cycle(fhs)

        # write commands to file
        for command in commands:
            fh = fhs_cycle.next()
            fh.write('%s\n' %(command))

        # close all filehandles
        for fh in fhs:
            fh.close()

        # make executable and print message
        for fn in fns:
            os.chmod(fn, stat.S_IRWXU)
            message('Writing job %s' %(fn))

        return fns

    def submit_jobs(self, fns):
        # submit jobs to the cluster
        job_ids = []
        for fn in fns:
            if self.cluster == 'broad':
                process = subprocess.Popen(['%s < %s' %(self.submit_cmd, fn)], stdout = subprocess.PIPE, shell=True)
            elif self.cluster == 'coyote':
                process = subprocess.Popen(['%s %s' %(self.submit_cmd, fn)], stdout = subprocess.PIPE, shell=True)
            else:
                quit()
            [out, error] = process.communicate()
            job_ids.append(self.parse_job(out))
            message('Submitting job %s' %(fn))
        return job_ids

    def write_LSF_array(self, fns):
        # write an LSF job array from args and filenames

        # initialize output file
        fh, array_fn = self.mktemp(suffix='.sh')
        array_fn = os.path.abspath(array_fn)

        # write header
        fh.write('#BSUB -J "job[1-%d]%%%s"\n' %(len(fns), self.l))
        fh.write('#BSUB -e %s.e.%%I\n' %(array_fn))
        fh.write('#BSUB -o %s.o.%%I\n' %(array_fn))
        fh.write('#BSUB -q %s\n' %(self.q))
        fh.write('#BSUB -G %s\n' %(self.G))
        fh.write('#BSUB -R "rusage[mem=%s:argon_io=%s]"\n' %(self.m, self.io))
        fh.write('#BSUB -P %s\n' %(array_fn))
        fh.write('source /home/unix/%s/.bashrc\n' %(self.username))
        fh.write('cd $LS_SUBCWD\n')

        # write job array
        for i, fn in enumerate(fns):
            os.chmod(fn, stat.S_IRWXU)
            fh.write('job_array[%d]=%s\n' %(i+1, os.path.abspath(fn)))
        fh.write('${job_array[$LSB_JOBINDEX]};\n')
        fh.close()

        # make executable and print message
        os.chmod(array_fn, stat.S_IRWXU)
        message('Writing array %s' %(array_fn))
        return array_fn

    def write_PBS_array(self, fns):
        # write a PBS job array from args and filenames

        # initialize output file
        fh, array_fn = self.mktemp(suffix='.sh')
        array_fn = os.path.abspath(array_fn)

        # write header
        fh.write('#PBS -t 1-%d%%%s\n' %(len(fns), min(len(fns), int(self.l))))
        fh.write('#PBS -e %s.e\n' %(array_fn))
        fh.write('#PBS -o %s.o\n' %(array_fn))
        fh.write('source /home/%s/.bashrc\n' %(self.username))
        fh.write('cd $PBS_O_WORKDIR\n')

        # write job array
        for i, fn in enumerate(fns):
            os.chmod(fn, stat.S_IRWXU)
            fh.write('job_array[%d]=%s\n' %(i+1, os.path.abspath(fn)))
        fh.write('${job_array[$PBS_ARRAYID]};\n')
        fh.close()

        # make executable and print message
        os.chmod(array_fn, stat.S_IRWXU)
        message('Writing array %s' %(array_fn))
        return array_fn

    def write_job_array(self, fns):
        # write a job array (LSF or PBS)
        if self.cluster == 'broad':
            array_fn = self.write_LSF_array(fns)
        elif self.cluster == 'coyote':
            array_fn = self.write_PBS_array(fns)
        return array_fn

    def submit(self, commands, out = False):
        # submit a job array to the cluster
        if out == False:
            fns = self.write_jobs(commands)
            array_fn = self.write_job_array(fns)
            job_ids = self.submit_jobs([array_fn])
            return job_ids
        elif out == True:
            print '\n'.join(commands)
            return []

    def wait(self, job_ids, out = False):
        # wait for jobs to finish
        if out == False:
            while True:
                time.sleep(5)
                if self.jobs_finished(job_ids):
                    break

    def submit_and_wait(self, commands, out = False):
        # submit job array and wait for it to finish
        job_ids = self.submit(commands, out = out)
        self.wait(job_ids, out = out)

    def submit_pipeline(self, pipeline, out = False):
        # a pipeline is a list of lists of commands
        for commands in pipeline:
            self.submit_and_wait(commands, out = out)

    def validate_output(self, fns, out = False):
        # make sure files exist
        if out == False:
            test = [os.path.exists(fn) for fn in fns]
            if False in test:
                error('file %s does not exist' %(fns[test.index(False)]))

    def remove_files(self, fns, out = False, to_queue = False):
        # remove list of files
        cmds = ['rm %s' %(fn) for fn in fns]
        if out == False:
            if to_queue == False:
                for cmd in cmds:
                    os.system(cmd)
            elif to_queue == True:
                self.submit_and_wait(cmds, out = out)
        elif out == True:
            print '\n'.join(cmds)

    def move_files(self, x, y, out = False, to_queue = False):
        # move files from x to y
        if len(x) != len(y):
            error('move_files: len(x) != len(y)')
        cmds = ['mv %s %s' %(a, b) for a, b in zip(x, y)]
        if out == False:
            if to_queue == False:
                for cmd in cmds:
                    os.system(cmd)
            elif to_queue == True:
                self.submit_and_wait(cmds, out = out)
        else:
            print '\n'.join(cmds)

    def gzip_and_validate(self, fns, tgz, out = False, to_queue = False):
        # compress files and validate .gz file
        cmd = 'tar -cvzf %s %s' %(tgz, ' '.join(fns))
        if out == False:
            if to_queue == False:
                os.system(cmd)
            else:
                self.submit_and_wait(args, [cmd])
            try:
                test = subprocess.check_output(['gunzip', '-t', tgz])
                if test != '':
                    error('gunzip -t %s failed' %(tgz))
            except:
                error('gunzip -t %s failed' %(tgz))
        elif out == True:
            print cmd

    def run_local(self, commands, out = False):
        # Run commands locally
        if out == False:
            for command in commands:
                os.system(command)
        elif out == True:
            print '\n'.join(commands)

def initialize():
    # initialize global variables for ssub

    # parse command line args
    ssub = Ssub()

    # get list of commands
    commands = []
    if ssub.commands != '':
        commands += [command.strip() for command in ssub.commands.split(';')]
    if select.select([sys.stdin], [], [], 0)[0]:
        commands += [line.rstrip() for line in sys.stdin.readlines()]

    # calculate number of cpus
    if ssub.n < 0:
        ssub.n = len(commands)

    return ssub, commands

ssub, commands = initialize()

if __name__ == '__main__':
    ssub.submit(commands)
<commit_msg>made ssub sister<commit_after>#!/usr/bin/env python

import argparse, os, re, select, stat, subprocess, sys, tempfile, time
from util import *

'''
ssub is a simple job submission script

usage:
  cat list_of_commands.txt | ssub -n 100 -q hour -G gscidfolk -m 8 --io 10
  ssub -n 100 -q week -G broadfolk -m 8 --io 10 "command1; command2; command3;"

to use it as a python library:
  import ssub
  ssub.args.n = 100
  ssub.args.q = "hour"
  commands = get_commands()
  job_ids = submit_array(ssub.args, commands)
  wait_for_jobs(job_ids)
  print "done"

to create and run pipelines:
  import ssub
  ssub.args.q = "week"
  ssub.args.G = "broadfolk"
  commands1 = run_blast()
  commands2 = parse_blast()
  pipeline = [commands1, commands2]
  submit_pipeline(args, pipeline)
  print "done"

'''

# set global variables
config = ConfigParser.ConfigParser()
config.read("%s/train.cfg" %(os.path.dirname(os.path.abspath(__file__))))
username = config.get('User', 'username')
temp_dir = config.get('User', 'tmp_directory')
cluster= config.get('User', 'cluster')

def parse_args():
    # print usage statement
    usage = "cat list_of_commands.txt | ssub -n 100 -q hour -G gscidfolk -m 8 --io 10\n"
    usage +="ssub -n 100 -q week -G broadfolk -m 8 --io 10 'command 1; command 2; command 3;"

    # add command line arguments
    parser = argparse.ArgumentParser(usage = usage)
    parser.add_argument('-n', default = -1, type = int, help = 'number of cpus')
    parser.add_argument('-q', default = 'hour', help = 'queue')
    parser.add_argument('-G', default = 'gscidfolk', help = 'group')
    parser.add_argument('-m', default = 4, help = 'memory (gb)')
    parser.add_argument('-l', default = 200, help = 'job array slot limit')
    parser.add_argument('--io', default = 1, help = 'disk io (units)')
    parser.add_argument('commands', nargs = '?', default = '')

    # parse arguments from stdin
    if __name__ == '__main__':
        args = parser.parse_args()
    else:
        args = parser.parse_args('')
    return args

class Ssub():

    def __init__(self, cluster = 'coyote'):

        # get command line arguments
        args = parse_args()

        # initialize cluster parameters
        self.cluster = cluster
        self.username = username
        self.temp_dir = temp_dir
        self.header = '#!/bin/bash\n'
        self.l = args.l
        self.n = args.n
        self.m = args.m
        self.q = args.q
        self.G = args.G
        self.io = args.io
        self.commands = args.commands

        # broad parameters
        if cluster == 'broad':
            self.submit_cmd = 'bsub'
            self.stat_cmd = 'bjobs -w'
            self.parse_job = lambda x: re.search('Job <(\d+)>', x).group(1)

        # coyote parameters
        elif cluster == 'coyote':
            self.submit_cmd = 'qsub'
            self.stat_cmd = 'qstat -l'
            self.parse_job = lambda x: x.rstrip()

        # unrecognized cluster
        else:
            error('unrecognized cluster %s' %(cluster))

    def __repr__(self):
        return '\ncluster: %s\nusername: %s\ntemp_dir: %s\nn: %d\nm: %d\nq: %s\nG: %s\nio: %s\n' %(self.cluster, self.username, self.temp_dir, self.n, self.m, self.q, self.G, self.io)

    def __str__(self):
        a = self.__repr__()
        a = re.sub('\n', '\n  ', a)
        return a

    def mktemp(self, suffix = '.tmp'):
        # make temporary file and return [fh, fn]
        fh, fn = tempfile.mkstemp(suffix='.sh', dir=self.temp_dir)
        os.close(fh)
        fh = open(fn, 'w')
        fh.write(self.header)
        return fh, fn

    def job_status(self):
        # return job status
        process = subprocess.Popen([self.stat_cmd], stdout = subprocess.PIPE, shell=True)
        [out, err] = process.communicate()
        out = [line for line in out.split('\n') if self.username in line]
        return [out, err]

    def jobs_finished(self, job_ids):
        # check if jobs are finished
        [out, err] = self.job_status()
        for job in out:
            for job_id in job_ids:
                if job_id in job:
                    return False
        return True

    def write_jobs(self, commands):
        # write job scripts from a list of commands

        # initialize output files
        fhs, fns = zip(*[self.mktemp(suffix='.sh') for i in range(min(self.n, len(commands)))])
        fhs_cycle = cycle(fhs)

        # write commands to file
        for command in commands:
            fh = fhs_cycle.next()
            fh.write('%s\n' %(command))

        # close all filehandles
        for fh in fhs:
            fh.close()

        # make executable and print message
        for fn in fns:
            os.chmod(fn, stat.S_IRWXU)
            message('Writing job %s' %(fn))

        return fns

    def submit_jobs(self, fns):
        # submit jobs to the cluster
        job_ids = []
        for fn in fns:
            if self.cluster == 'broad':
                process = subprocess.Popen(['%s < %s' %(self.submit_cmd, fn)], stdout = subprocess.PIPE, shell=True)
            elif self.cluster == 'coyote':
                process = subprocess.Popen(['%s %s' %(self.submit_cmd, fn)], stdout = subprocess.PIPE, shell=True)
            else:
                quit()
            [out, error] = process.communicate()
            job_ids.append(self.parse_job(out))
            message('Submitting job %s' %(fn))
        return job_ids

    def write_LSF_array(self, fns):
        # write an LSF job array from args and filenames

        # initialize output file
        fh, array_fn = self.mktemp(suffix='.sh')
        array_fn = os.path.abspath(array_fn)

        # write header
        fh.write('#BSUB -J "job[1-%d]%%%s"\n' %(len(fns), self.l))
        fh.write('#BSUB -e %s.e.%%I\n' %(array_fn))
        fh.write('#BSUB -o %s.o.%%I\n' %(array_fn))
        fh.write('#BSUB -q %s\n' %(self.q))
        fh.write('#BSUB -G %s\n' %(self.G))
        fh.write('#BSUB -R "rusage[mem=%s:argon_io=%s]"\n' %(self.m, self.io))
        fh.write('#BSUB -P %s\n' %(array_fn))
        fh.write('source /home/unix/%s/.bashrc\n' %(self.username))
        fh.write('cd $LS_SUBCWD\n')

        # write job array
        for i, fn in enumerate(fns):
            os.chmod(fn, stat.S_IRWXU)
            fh.write('job_array[%d]=%s\n' %(i+1, os.path.abspath(fn)))
        fh.write('${job_array[$LSB_JOBINDEX]};\n')
        fh.close()

        # make executable and print message
        os.chmod(array_fn, stat.S_IRWXU)
        message('Writing array %s' %(array_fn))
        return array_fn

    def write_PBS_array(self, fns):
        # write a PBS job array from args and filenames

        # initialize output file
        fh, array_fn = self.mktemp(suffix='.sh')
        array_fn = os.path.abspath(array_fn)

        # write header
        fh.write('#PBS -t 1-%d%%%s\n' %(len(fns), min(len(fns), int(self.l))))
        fh.write('#PBS -e %s.e\n' %(array_fn))
        fh.write('#PBS -o %s.o\n' %(array_fn))
        fh.write('source /home/%s/.bashrc\n' %(self.username))
        fh.write('cd $PBS_O_WORKDIR\n')

        # write job array
        for i, fn in enumerate(fns):
            os.chmod(fn, stat.S_IRWXU)
            fh.write('job_array[%d]=%s\n' %(i+1, os.path.abspath(fn)))
        fh.write('${job_array[$PBS_ARRAYID]};\n')
        fh.close()

        # make executable and print message
        os.chmod(array_fn, stat.S_IRWXU)
        message('Writing array %s' %(array_fn))
        return array_fn

    def write_job_array(self, fns):
        # write a job array (LSF or PBS)
        if self.cluster == 'broad':
            array_fn = self.write_LSF_array(fns)
        elif self.cluster == 'coyote':
            array_fn = self.write_PBS_array(fns)
        return array_fn

    def submit(self, commands, out = False):
        # submit a job array to the cluster
        if out == False:
            fns = self.write_jobs(commands)
            array_fn = self.write_job_array(fns)
            job_ids = self.submit_jobs([array_fn])
            return job_ids
        elif out == True:
            print '\n'.join(commands)
            return []

    def wait(self, job_ids, out = False):
        # wait for jobs to finish
        if out == False:
            while True:
                time.sleep(5)
                if self.jobs_finished(job_ids):
                    break

    def submit_and_wait(self, commands, out = False):
        # submit job array and wait for it to finish
        job_ids = self.submit(commands, out = out)
        self.wait(job_ids, out = out)

    def submit_pipeline(self, pipeline, out = False):
        # a pipeline is a list of lists of commands
        for commands in pipeline:
            self.submit_and_wait(commands, out = out)

    def validate_output(self, fns, out = False):
        # make sure files exist
        if out == False:
            test = [os.path.exists(fn) for fn in fns]
            if False in test:
                error('file %s does not exist' %(fns[test.index(False)]))

    def remove_files(self, fns, out = False, to_queue = False):
        # remove list of files
        cmds = ['rm %s' %(fn) for fn in fns]
        if out == False:
            if to_queue == False:
                for cmd in cmds:
                    os.system(cmd)
            elif to_queue == True:
                self.submit_and_wait(cmds, out = out)
        elif out == True:
            print '\n'.join(cmds)

    def move_files(self, x, y, out = False, to_queue = False):
        # move files from x to y
        if len(x) != len(y):
            error('move_files: len(x) != len(y)')
        cmds = ['mv %s %s' %(a, b) for a, b in zip(x, y)]
        if out == False:
            if to_queue == False:
                for cmd in cmds:
                    os.system(cmd)
            elif to_queue == True:
                self.submit_and_wait(cmds, out = out)
        else:
            print '\n'.join(cmds)

    def gzip_and_validate(self, fns, tgz, out = False, to_queue = False):
        # compress files and validate .gz file
        cmd = 'tar -cvzf %s %s' %(tgz, ' '.join(fns))
        if out == False:
            if to_queue == False:
                os.system(cmd)
            else:
                self.submit_and_wait(args, [cmd])
            try:
                test = subprocess.check_output(['gunzip', '-t', tgz])
                if test != '':
                    error('gunzip -t %s failed' %(tgz))
            except:
                error('gunzip -t %s failed' %(tgz))
        elif out == True:
            print cmd

    def run_local(self, commands, out = False):
        # Run commands locally
        if out == False:
            for command in commands:
                os.system(command)
        elif out == True:
            print '\n'.join(commands)

def initialize():
    # initialize global variables for ssub

    # parse command line args
    ssub = Ssub()

    # get list of commands
    commands = []
    if ssub.commands != '':
        commands += [command.strip() for command in ssub.commands.split(';')]
    if select.select([sys.stdin], [], [], 0)[0]:
        commands += [line.rstrip() for line in sys.stdin.readlines()]

    # calculate number of cpus
    if ssub.n < 0:
        ssub.n = len(commands)

    return ssub, commands

ssub, commands = initialize()

if __name__ == '__main__':
    ssub.submit(commands)
<|endoftext|>