<commit_before>#!/usr/bin/env python
import os
import random
from datetime import datetime, timedelta
import twilio.twiml
from bson import ObjectId
from flask import Flask, request, jsonify, abort
from flask.ext.pymongo import PyMongo, ASCENDING, DESCENDING
from utils import crossdomain, tznow
from bson import ObjectId

app = Flask(__name__)

CONFIGS = (
    'WEBAPP_URL',
    'AUTH_TOKEN',
    'CELL_NUM',
    'MONGOLAB_URI',
    'TWILIO_NUM',
)

# Set up the app
app = Flask(__name__)
app.config.update(DEBUG=(os.environ.get('DEBUG') == 'TRUE'))
app.config.update(**{v: os.environ[v] for v in CONFIGS})
app.config['MONGO_URI'] = app.config['MONGOLAB_URI']  # for flask-pymongo

# Initialize extensions
pymongo = PyMongo(app)

# Some constants
SMS_CODE_RESET = timedelta(minutes=10)
SMS_CODE_GRACE = timedelta(minutes=10)
USER_CHECKIN_EXPIRE = timedelta(minutes=15)
USER_POST_THROTTLE = timedelta(seconds=0)

"""Collection schemas

users: {
    phone_number: string  # not present if it's a qr signup
    qr_code: strong  # not present if sms signup
    created: datetime
    last_checkin: datetime
}

smscodes: {
    code: string
    created: datetime
}

qrcodes: {
    code: string
    taken: bool
}

posts: {
    message: string
    poster_id: user_id
    submitted: datetime
    showtime: datetime  # not present if it hasn't been shown yet
    extender_ids: [user_ids]
}
"""

class InvalidCodeException(Exception):
    """When user uses code that doesn't exist"""

class NotCheckedInException(Exception):
    """When user tries to vote or post before checking in"""

class NoSuchUserException(Exception):
    """When user tries to vote or post before checking in"""

class ChillOut(Exception):
    """When users get too excited (try to re-up-vote or re-post too soon)."""

def create_sms_code():
    """Create a new code. More races woo!"""
    while True:
        code = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz1234567890')
                       for _ in range(6))
        existing_with_code = pymongo.db.smscodes.find_one({'code': code})
        if existing_with_code is None:
            break

    new_sms = {
        'code': code,
        'created': tznow()
    }
    pymongo.db.smscodes.insert(new_sms)
    return new_sms

def refresh_qr_code():
    """Create a new one"""
    while True:
        code = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz1234567890')
                       for _ in range(9))
        existing_with_code = pymongo.db.qrcodes.find_one({'code': code})
        if existing_with_code is None:
            break

    new_qr = {
        'code': code,
        'created': tznow()
    }
    pymongo.db.qrcodes.insert(new_qr)
    return new_qr

def get_qr_code():
    """Fetch the current qr code for display"""
    code = pymongo.db.qrcodes.find_one(sort=[('created', DESCENDING)])
    if code is None:
        code = refresh_qr_code()
    return code

def get_sms_code():
    """Fetches the most up-to-date SMS code for the billboard.

    This may trigger a new code if one is due.
    """
    codes = pymongo.db.smscodes.find(sort=[('created', DESCENDING)])
    try:
        current = next(codes)
    except StopIteration:  # empty database
        current = create_sms_code()
    if current['created'] + SMS_CODE_RESET < tznow():
        # yo, WARNING: off to the races!
        current = create_sms_code()
    return current

def get_queue():
    """Fetch all posts currently queued."""
    unshown = pymongo.db.posts.find({'showtime': {'$exists': False}})
    queue_in_order = unshown.sort('submitted')
    return queue_in_order

def get_user_from_phone(phone_number):
    """Get a user given their phone number, or None if they don't exist"""
    return pymongo.db.users.find_one({'phone_number': phone_number})

def get_user_from_user_id(userid):
    return  pymongo.db.users.find_one({'_id': userid})

def is_checked_in(user):
    """Test whether a user is checked in or not."""
    if user is None:
        return False
    a_ok = user['last_checkin'] + USER_CHECKIN_EXPIRE > tznow()
    return a_ok

def check_sms_code(test_code):
    """Checks whether the SMS code is currently valid."""
    codes = pymongo.db.smscodes.find(sort=[('created', DESCENDING)])
    current = next(codes)
    if test_code == current['code']:
        return True
    else:
        previous = next(codes)
        if (test_code == previous['code'] and
            tznow() - current['created'] < SMS_CODE_GRACE):
            return True
        else:
            return False

def check_in_with_sms_code(phone_number, code):
    """Check in (and possibly create) a user, verified by the active code.

    Returns the user's data, or raises InvalidCodeException if the code is wrong or expired.
    """
    if not check_sms_code(code):
        raise InvalidCodeException("You fucked up")
    user = pymongo.db.users.find_one({'phone_number': phone_number})
    if user is None:  # so racey
        user = {
            'phone_number': phone_number,
            'created': tznow(),
        }
    user['last_checkin'] = tznow()
    pymongo.db.users.save(user)
    return user

def check_qr_code(test_code):
    """Validate a qr code for realz.

    QRs are immediately expired after one use.
    """
    current = pymongo.db.qrcodes.find_one(sort=[('created', DESCENDING)])
    if test_code == current['code']:
        refresh_qr_code()
        return True
    return False

def check_in_with_qr_code(user_id, code):
    """Check in an existing user with a QR code."""
    user = pymongo.db.users.find_one({'_id': ObjectId(userId)})
    if user is None:
        raise NoSuchUserException('no user exists with id {}'.format(user_id))
    if not check_qr_code(code):
        raise InvalidCodeException('You fucked up -- wrong qr code yoyo')

def create_account_with_qr_code(code):
    """Creates a new user with a QR code."""
    if not check_qr_code(code):
        raise InvalidCodeException('You fucked up -- wrong qr code yooy')
    now = tznow()
    user = {
        'qr_code': code,
        'created': now,
        'last_checkin': now,
    }
    return user

def get_current_post():
    showing = pymongo.db.posts.find_one({'showtime': {'$exists': True}},
                                        sort=[('showtime', DESCENDING)])
    return showing

def update_showing():
    next_ = pymongo.db.posts.find_one({'showtime': {'$exists': False}},
                                      sort=[('showtime', ASCENDING)])
    if next_ is not None:
        print('changing...')
        pymongo.db.posts.update({'_id': next_['_id']},
                                {'$set': {'showtime': tznow()}})
    else:
        print('nothing in the queue')

def post_message(user, message):
    """Try to queue a message for a user.

    Returns the message's position in the queue.

    Raises ChillOut if the user has posted too many messages recently.
    """
    user_id = user['_id']
    prev = pymongo.db.posts.find_one({'poster_id': user_id},
                                     sort=[('submitted', DESCENDING)])
    if (prev is not None and
        prev['submitted'] + USER_POST_THROTTLE > tznow()):
        raise ChillOut('Whoa. Chill out, hey. So many messages.')

    post = {
        'message': message,
        'poster_id': user_id,
        'submitted': tznow(),
        'extender_ids': [user_id],
    }
    pymongo.db.posts.insert(post)
    return get_queue().count()

def save_vote(user):
    """Register a vote for a user.

    Returns 1 if the vote was counted.

    Currently it is hard-coded to always succeed
    """

    current_post = get_current_post();
    pymongo.db.posts.update({"_id": current_post["_id"]},
                            {"$addToSet":{"extender_ids":user["_id"]}});
    return 1

@app.route('/sms', methods=['GET','POST'])
def handle_sms():

    #Get number and response
    from_number = request.values['From']
    from_response = request.values['Body']
    first_word = from_response.lower().strip().split(' ', 1)[0];
    try:
        user_post = from_response.lower().strip().split(' ', 1)[1]
    except IndexError:
        user_post = None
    resp = twilio.twiml.Response()

    user = get_user_from_phone(from_number)

    #Checks if user already checked in

    if is_checked_in(user):
         #Check if user response is vote
        if "vote" in first_word:
            if save_vote(user):
                message="Vote successful"
            else:
                message="Vote unsuccessful"

        #Check if user response is a post
        elif "post" in first_word:
            if user_post is None:
                message = "whoops, no post provided, nothing to do."
                resp.message(message)
                return str(resp)
            try:
                queue_num = post_message(user, user_post)
            except ChillOut:
                message = "chill out dude. wait a bit, then post again."
                resp.message(message)
                return str(resp)
            message = "Your message is queued in position {}".format(queue_num)

        else:
            #check if code is correct
            try:
                check = check_in_with_sms_code(from_number, first_word);

            except InvalidCodeException:
                #error handling
                message="fucked up, though you are already checked in..."
                resp.message(message)
                return str(resp)

            message = ("Thanks for checking in! To vote, Please text 'vote', "
                       "otherwise text 'post' and type in your message")

    #User hasn't checked in but is checking in now
    elif "post" not in first_word and "vote" not in first_word:
          #check if code is correct
            try:
                check = check_in_with_sms_code(from_number, first_word);
            except InvalidCodeException:
                #error handling
                message="fucked up, and you are not even checked in :("
                resp.message(message)
                return str(resp)

            message = ("Thanks for checking in! To vote, Please text 'vote', "
                       "otherwise text 'post' and type in your message")
    else:
        #error handling
        message="Not checked in"
        resp.message(message)
        return str(resp)

    resp.message(message)

    return str(resp)

@app.route('/')
def home():
    resp = 'sms code: {}<br/>'.format(get_sms_code().get('code'))
    update_showing()
    message = get_current_post()
    if message is not None:
        resp += '{} ({} votes)'.format(message['message'],
                                       len(message['extender_ids']))
    else:
        resp += 'No post yet :('
    return resp

@app.route('/webapp/get-id')
@crossdomain(origin='*')
def webapp_id():
    code = request.values.get('code')
    if code is None:
        resp = jsonify(status='bad', message='missing code')
        resp.status_code = 400
        return resp
    try:
        user = create_account_with_qr_code(code)
    except InvalidCodeException:
        resp = jsonify(status='bad', message='invalid code')
        resp.status_code = 400
        return resp
    return jsonify(status='cool', userId=str(user['_id']))

@app.route('/webapp/check-in')
@crossdomain(origin='*')
def webapp_checkin():
    code = request.values.get('code')
    if code is None:
        resp = jsonify(status='bad', message='missing code')
        resp.status_code = 400
        return resp
    user_id = request.values.get('userId')
    if user_id is None:
        resp = jsonify(status='bad', message='missing userId')
        resp.status_code = 400
        return resp
    try:
        check_in_with_qr_code(user_id, code)
    except NoSuchUserException:
        resp = jsonify(status='bad', message='no such user')
        resp.status_code = 400
        return resp
    except InvalidCodeException:
        resp = jsonify(status='bad', message='invalid code')
        resp.status_code = 400
        return resp
    return jsonify(status='cool')

#Endpoint to get all messages and ids from queue
@app.route('/webapp/cards')
@crossdomain(origin='*')
def webapp_cards():
    cards = list(get_queue())
    card_messages=[]
    for card in cards:
        card_messages.append({
            "message": card['message'],
            "id": str(card['_id'])
        })
    return jsonify(status='cool', content=card_messages)

@app.route('/webapp/vote', methods=['POST'])
@crossdomain(origin='*')
def webapp_vote():

    user = get_user_from_user_id(ObjectId(request.values['userId']))
    if is_checked_in(user):
        save_vote(user)
        return jsonify(status='cool')
    else:
        resp = jsonify(response="error")
        resp.status_code = 400
        return resp

@app.route('/display')
@crossdomain(origin='*')
def display_data():
    display_stuff = {
        'smsCode': get_sms_code().get('code'),
        'qrCode': get_qr_code().get('code'),
        'message': get_current_post().get('message'),
    }
    return jsonify(status='cool', **display_stuff)

# dev stuff

def push():
    """Push a test request context"""
    ctx = app.test_request_context()
    ctx.push()
    return ctx

if __name__ == '__main__':
    app.run(debug=True)
<commit_msg>do not pop the queue<commit_after>#!/usr/bin/env python
import os
import random
from datetime import datetime, timedelta
import twilio.twiml
from bson import ObjectId
from flask import Flask, request, jsonify, abort
from flask.ext.pymongo import PyMongo, ASCENDING, DESCENDING
from utils import crossdomain, tznow
from bson import ObjectId

app = Flask(__name__)

CONFIGS = (
    'WEBAPP_URL',
    'AUTH_TOKEN',
    'CELL_NUM',
    'MONGOLAB_URI',
    'TWILIO_NUM',
)

# Set up the app
app = Flask(__name__)
app.config.update(DEBUG=(os.environ.get('DEBUG') == 'TRUE'))
app.config.update(**{v: os.environ[v] for v in CONFIGS})
app.config['MONGO_URI'] = app.config['MONGOLAB_URI']  # for flask-pymongo

# Initialize extensions
pymongo = PyMongo(app)

# Some constants
SMS_CODE_RESET = timedelta(minutes=10)
SMS_CODE_GRACE = timedelta(minutes=10)
USER_CHECKIN_EXPIRE = timedelta(minutes=15)
USER_POST_THROTTLE = timedelta(seconds=0)

"""Collection schemas

users: {
    phone_number: string  # not present if it's a qr signup
    qr_code: strong  # not present if sms signup
    created: datetime
    last_checkin: datetime
}

smscodes: {
    code: string
    created: datetime
}

qrcodes: {
    code: string
    taken: bool
}

posts: {
    message: string
    poster_id: user_id
    submitted: datetime
    showtime: datetime  # not present if it hasn't been shown yet
    extender_ids: [user_ids]
}
"""

class InvalidCodeException(Exception):
    """When user uses code that doesn't exist"""

class NotCheckedInException(Exception):
    """When user tries to vote or post before checking in"""

class NoSuchUserException(Exception):
    """When user tries to vote or post before checking in"""

class ChillOut(Exception):
    """When users get too excited (try to re-up-vote or re-post too soon)."""

def create_sms_code():
    """Create a new code. More races woo!"""
    while True:
        code = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz1234567890')
                       for _ in range(6))
        existing_with_code = pymongo.db.smscodes.find_one({'code': code})
        if existing_with_code is None:
            break

    new_sms = {
        'code': code,
        'created': tznow()
    }
    pymongo.db.smscodes.insert(new_sms)
    return new_sms

def refresh_qr_code():
    """Create a new one"""
    while True:
        code = ''.join(random.choice('abcdefghijklmnopqrstuvwxyz1234567890')
                       for _ in range(9))
        existing_with_code = pymongo.db.qrcodes.find_one({'code': code})
        if existing_with_code is None:
            break

    new_qr = {
        'code': code,
        'created': tznow()
    }
    pymongo.db.qrcodes.insert(new_qr)
    return new_qr

def get_qr_code():
    """Fetch the current qr code for display"""
    code = pymongo.db.qrcodes.find_one(sort=[('created', DESCENDING)])
    if code is None:
        code = refresh_qr_code()
    return code

def get_sms_code():
    """Fetches the most up-to-date SMS code for the billboard.

    This may trigger a new code if one is due.
    """
    codes = pymongo.db.smscodes.find(sort=[('created', DESCENDING)])
    try:
        current = next(codes)
    except StopIteration:  # empty database
        current = create_sms_code()
    if current['created'] + SMS_CODE_RESET < tznow():
        # yo, WARNING: off to the races!
        current = create_sms_code()
    return current

def get_queue():
    """Fetch all posts currently queued."""
    unshown = pymongo.db.posts.find({'showtime': {'$exists': False}})
    queue_in_order = unshown.sort('submitted')
    return queue_in_order

def get_user_from_phone(phone_number):
    """Get a user given their phone number, or None if they don't exist"""
    return pymongo.db.users.find_one({'phone_number': phone_number})

def get_user_from_user_id(userid):
    return  pymongo.db.users.find_one({'_id': userid})

def is_checked_in(user):
    """Test whether a user is checked in or not."""
    if user is None:
        return False
    a_ok = user['last_checkin'] + USER_CHECKIN_EXPIRE > tznow()
    return a_ok

def check_sms_code(test_code):
    """Checks whether the SMS code is currently valid."""
    codes = pymongo.db.smscodes.find(sort=[('created', DESCENDING)])
    current = next(codes)
    if test_code == current['code']:
        return True
    else:
        previous = next(codes)
        if (test_code == previous['code'] and
            tznow() - current['created'] < SMS_CODE_GRACE):
            return True
        else:
            return False

def check_in_with_sms_code(phone_number, code):
    """Check in (and possibly create) a user, verified by the active code.

    Returns the user's data, or raises InvalidCodeException if the code is wrong or expired.
    """
    if not check_sms_code(code):
        raise InvalidCodeException("You fucked up")
    user = pymongo.db.users.find_one({'phone_number': phone_number})
    if user is None:  # so racey
        user = {
            'phone_number': phone_number,
            'created': tznow(),
        }
    user['last_checkin'] = tznow()
    pymongo.db.users.save(user)
    return user

def check_qr_code(test_code):
    """Validate a qr code for realz.

    QRs are immediately expired after one use.
    """
    current = pymongo.db.qrcodes.find_one(sort=[('created', DESCENDING)])
    if test_code == current['code']:
        refresh_qr_code()
        return True
    return False

def check_in_with_qr_code(user_id, code):
    """Check in an existing user with a QR code."""
    user = pymongo.db.users.find_one({'_id': ObjectId(userId)})
    if user is None:
        raise NoSuchUserException('no user exists with id {}'.format(user_id))
    if not check_qr_code(code):
        raise InvalidCodeException('You fucked up -- wrong qr code yoyo')

def create_account_with_qr_code(code):
    """Creates a new user with a QR code."""
    if not check_qr_code(code):
        raise InvalidCodeException('You fucked up -- wrong qr code yooy')
    now = tznow()
    user = {
        'qr_code': code,
        'created': now,
        'last_checkin': now,
    }
    return user

def get_current_post():
    showing = pymongo.db.posts.find_one({'showtime': {'$exists': True}},
                                        sort=[('showtime', DESCENDING)])
    return showing

def update_showing():
    next_ = pymongo.db.posts.find_one({'showtime': {'$exists': False}},
                                      sort=[('showtime', ASCENDING)])
    # if next_ is not None:
    #     print('changing...')
    #     pymongo.db.posts.update({'_id': next_['_id']},
    #                             {'$set': {'showtime': tznow()}})
    # else:
    #     print('nothing in the queue')

def post_message(user, message):
    """Try to queue a message for a user.

    Returns the message's position in the queue.

    Raises ChillOut if the user has posted too many messages recently.
    """
    user_id = user['_id']
    prev = pymongo.db.posts.find_one({'poster_id': user_id},
                                     sort=[('submitted', DESCENDING)])
    if (prev is not None and
        prev['submitted'] + USER_POST_THROTTLE > tznow()):
        raise ChillOut('Whoa. Chill out, hey. So many messages.')

    post = {
        'message': message,
        'poster_id': user_id,
        'submitted': tznow(),
        'extender_ids': [user_id],
    }
    pymongo.db.posts.insert(post)
    return get_queue().count()

def save_vote(user):
    """Register a vote for a user.

    Returns 1 if the vote was counted.

    Currently it is hard-coded to always succeed
    """

    current_post = get_current_post();
    pymongo.db.posts.update({"_id": current_post["_id"]},
                            {"$addToSet":{"extender_ids":user["_id"]}});
    return 1

@app.route('/sms', methods=['GET','POST'])
def handle_sms():

    #Get number and response
    from_number = request.values['From']
    from_response = request.values['Body']
    first_word = from_response.lower().strip().split(' ', 1)[0];
    try:
        user_post = from_response.lower().strip().split(' ', 1)[1]
    except IndexError:
        user_post = None
    resp = twilio.twiml.Response()

    user = get_user_from_phone(from_number)

    #Checks if user already checked in

    if is_checked_in(user):
         #Check if user response is vote
        if "vote" in first_word:
            if save_vote(user):
                message="Vote successful"
            else:
                message="Vote unsuccessful"

        #Check if user response is a post
        elif "post" in first_word:
            if user_post is None:
                message = "whoops, no post provided, nothing to do."
                resp.message(message)
                return str(resp)
            try:
                queue_num = post_message(user, user_post)
            except ChillOut:
                message = "chill out dude. wait a bit, then post again."
                resp.message(message)
                return str(resp)
            message = "Your message is queued in position {}".format(queue_num)

        else:
            #check if code is correct
            try:
                check = check_in_with_sms_code(from_number, first_word);

            except InvalidCodeException:
                #error handling
                message="fucked up, though you are already checked in..."
                resp.message(message)
                return str(resp)

            message = ("Thanks for checking in! To vote, Please text 'vote', "
                       "otherwise text 'post' and type in your message")

    #User hasn't checked in but is checking in now
    elif "post" not in first_word and "vote" not in first_word:
          #check if code is correct
            try:
                check = check_in_with_sms_code(from_number, first_word);
            except InvalidCodeException:
                #error handling
                message="fucked up, and you are not even checked in :("
                resp.message(message)
                return str(resp)

            message = ("Thanks for checking in! To vote, Please text 'vote', "
                       "otherwise text 'post' and type in your message")
    else:
        #error handling
        message="Not checked in"
        resp.message(message)
        return str(resp)

    resp.message(message)

    return str(resp)

@app.route('/')
def home():
    resp = 'sms code: {}<br/>'.format(get_sms_code().get('code'))
    update_showing()
    message = get_current_post()
    if message is not None:
        resp += '{} ({} votes)'.format(message['message'],
                                       len(message['extender_ids']))
    else:
        resp += 'No post yet :('
    return resp

@app.route('/webapp/get-id')
@crossdomain(origin='*')
def webapp_id():
    code = request.values.get('code')
    if code is None:
        resp = jsonify(status='bad', message='missing code')
        resp.status_code = 400
        return resp
    try:
        user = create_account_with_qr_code(code)
    except InvalidCodeException:
        resp = jsonify(status='bad', message='invalid code')
        resp.status_code = 400
        return resp
    return jsonify(status='cool', userId=str(user['_id']))

@app.route('/webapp/check-in')
@crossdomain(origin='*')
def webapp_checkin():
    code = request.values.get('code')
    if code is None:
        resp = jsonify(status='bad', message='missing code')
        resp.status_code = 400
        return resp
    user_id = request.values.get('userId')
    if user_id is None:
        resp = jsonify(status='bad', message='missing userId')
        resp.status_code = 400
        return resp
    try:
        check_in_with_qr_code(user_id, code)
    except NoSuchUserException:
        resp = jsonify(status='bad', message='no such user')
        resp.status_code = 400
        return resp
    except InvalidCodeException:
        resp = jsonify(status='bad', message='invalid code')
        resp.status_code = 400
        return resp
    return jsonify(status='cool')

#Endpoint to get all messages and ids from queue
@app.route('/webapp/cards')
@crossdomain(origin='*')
def webapp_cards():
    cards = list(get_queue())
    card_messages=[]
    for card in cards:
        card_messages.append({
            "message": card['message'],
            "id": str(card['_id'])
        })
    return jsonify(status='cool', content=card_messages)

@app.route('/webapp/vote', methods=['POST'])
@crossdomain(origin='*')
def webapp_vote():

    user = get_user_from_user_id(ObjectId(request.values['userId']))
    if is_checked_in(user):
        save_vote(user)
        return jsonify(status='cool')
    else:
        resp = jsonify(response="error")
        resp.status_code = 400
        return resp

@app.route('/display')
@crossdomain(origin='*')
def display_data():
    display_stuff = {
        'smsCode': get_sms_code().get('code'),
        'qrCode': get_qr_code().get('code'),
        'message': get_current_post().get('message'),
    }
    return jsonify(status='cool', **display_stuff)

# dev stuff

def push():
    """Push a test request context"""
    ctx = app.test_request_context()
    ctx.push()
    return ctx

if __name__ == '__main__':
    app.run(debug=True)
<|endoftext|>


<commit_before>import pytest
import numpy as np
from numpy.testing import assert_allclose
from keras.utils.test_utils import keras_test
from keras.layers import wrappers, Input
from keras.layers import core, convolutional, recurrent
from keras.models import Sequential, Model, model_from_json

@keras_test
def test_TimeDistributed():
    # first, test with Dense layer
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)), nb_epoch=1, batch_size=10)

    # test config
    model.get_config()

    # compare to TimeDistributedDense
    test_input = np.random.random((1, 3, 4))
    test_output = model.predict(test_input)
    weights = model.layers[0].get_weights()

    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, input_shape=(3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test when specifying a batch_input_shape
    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, batch_input_shape=(1, 3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test with Convolution2D
    model = Sequential()
    model.add(wrappers.TimeDistributed(convolutional.Convolution2D(5, 2, 2, border_mode='same'), input_shape=(2, 3, 4, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.train_on_batch(np.random.random((1, 2, 3, 4, 4)), np.random.random((1, 2, 5, 4, 4)))

    model = model_from_json(model.to_json())
    model.summary()

    # test stacked layers
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(wrappers.TimeDistributed(core.Dense(3)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')

    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test wrapping Sequential model
    model = Sequential()
    model.add(core.Dense(3, input_dim=2))
    outer_model = Sequential()
    outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test with functional API
    x = Input(shape=(3, 2))
    y = wrappers.TimeDistributed(model)(x)
    outer_model = Model(x, y)
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

@keras_test
def test_Bidirectional():
    for rnn in [recurrent.SimpleRNN, recurrent.LSTM]:
        for mode in ['sum', 'concat']:
            x = np.random.random((5, 3, 2))
            output_dim = 6 if mode == 'concat' else 3
            y = np.random.random((5, output_dim))

            # test with Sequential model
            model = Sequential()
            model.add(wrappers.Bidirectional(rnn(3), merge_mode=mode, input_shape=(3, 2)))
            model.add(core.Activation('sigmoid'))
            model.compile(loss='mse', optimizer='sgd')
            model.fit(x, y, nb_epoch=1, batch_size=5)

            # test stacked bidirectional layers
            model = Sequential()
            model.add(wrappers.Bidirectional(rnn(2, return_sequences=True), merge_mode=mode, input_shape=(3, 2)))
            model.add(wrappers.Bidirectional(rnn(3), merge_mode=mode))
            model.add(core.Activation('sigmoid'))
            model.compile(loss='mse', optimizer='sgd')
            model.fit(x, y, nb_epoch=1, batch_size=5)

            # test with functional API
            input = Input((3, 2))
            output = wrappers.Bidirectional(rnn(3), merge_mode=mode)(input)
            model = Model(input, output)
            model.compile(loss='mse', optimizer='sgd')
            model.fit(x, y, nb_epoch=1, batch_size=5)

if __name__ == '__main__':
    pytest.main([__file__])
<commit_msg>Speed up bidirectional wrapper tests<commit_after>import pytest
import numpy as np
from numpy.testing import assert_allclose
from keras.utils.test_utils import keras_test
from keras.layers import wrappers, Input
from keras.layers import core, convolutional, recurrent
from keras.models import Sequential, Model, model_from_json

@keras_test
def test_TimeDistributed():
    # first, test with Dense layer
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)), nb_epoch=1, batch_size=10)

    # test config
    model.get_config()

    # compare to TimeDistributedDense
    test_input = np.random.random((1, 3, 4))
    test_output = model.predict(test_input)
    weights = model.layers[0].get_weights()

    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, input_shape=(3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test when specifying a batch_input_shape
    reference = Sequential()
    reference.add(core.TimeDistributedDense(2, batch_input_shape=(1, 3, 4), weights=weights))
    reference.add(core.Activation('relu'))
    reference.compile(optimizer='rmsprop', loss='mse')

    reference_output = reference.predict(test_input)
    assert_allclose(test_output, reference_output, atol=1e-05)

    # test with Convolution2D
    model = Sequential()
    model.add(wrappers.TimeDistributed(convolutional.Convolution2D(5, 2, 2, border_mode='same'), input_shape=(2, 3, 4, 4)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')
    model.train_on_batch(np.random.random((1, 2, 3, 4, 4)), np.random.random((1, 2, 5, 4, 4)))

    model = model_from_json(model.to_json())
    model.summary()

    # test stacked layers
    model = Sequential()
    model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
    model.add(wrappers.TimeDistributed(core.Dense(3)))
    model.add(core.Activation('relu'))
    model.compile(optimizer='rmsprop', loss='mse')

    model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test wrapping Sequential model
    model = Sequential()
    model.add(core.Dense(3, input_dim=2))
    outer_model = Sequential()
    outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

    # test with functional API
    x = Input(shape=(3, 2))
    y = wrappers.TimeDistributed(model)(x)
    outer_model = Model(x, y)
    outer_model.compile(optimizer='rmsprop', loss='mse')
    outer_model.fit(np.random.random((10, 3, 2)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)

@keras_test
def test_Bidirectional():
    rnn = recurrent.SimpleRNN
    nb_sample = 1
    dim = 2
    timesteps = 2
    for mode in ['sum', 'concat']:
        x = np.random.random((nb_sample, timesteps, dim))
        output_dim = 2 if mode == 'concat' else 1
        y = np.random.random((nb_sample, output_dim))

        # test with Sequential model
        model = Sequential()
        model.add(wrappers.Bidirectional(rnn(1),
                                         merge_mode=mode, input_shape=(timesteps, dim)))
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)

        # test config
        model.get_config()
        model = model_from_json(model.to_json())
        model.summary()

        # test stacked bidirectional layers
        model = Sequential()
        model.add(wrappers.Bidirectional(rnn(1, return_sequences=True),
                                         merge_mode=mode, input_shape=(timesteps, dim)))
        model.add(wrappers.Bidirectional(rnn(1), merge_mode=mode))
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)

        # test with functional API
        input = Input((timesteps, dim))
        output = wrappers.Bidirectional(rnn(1), merge_mode=mode)(input)
        model = Model(input, output)
        model.compile(loss='mse', optimizer='sgd')
        model.fit(x, y, nb_epoch=1, batch_size=1)

if __name__ == '__main__':
    pytest.main([__file__])
<|endoftext|>


<commit_before>from datetime import datetime
from operator import methodcaller
from typing import Dict, Iterable, List, Optional

from channels.backends.twitter import TwitterChannel
import click
import dateutil.parser
import pytz
import requests

class Event:
    # flake8 does not support variable annotations...
    # start: datetime
    # end: datetime
    # title: str
    # url: Optional[str]

    def __init__(self, *, start, end, title, url):
        self.start = start
        self.end = end
        self.title = title
        self.url = url

    @classmethod
    def from_json(cls, data: Dict[str, Optional[str]]) -> "Event":
        start = dateutil.parser.parse(data["start"])
        end = dateutil.parser.parse(data["end"])
        return cls(start=start, end=end, title=data["title"], url=data["url"])

    def generate_message(self, tz) -> Optional[str]:
        # TODO: Support other kinds of events
        if self.title.lower() == "open":
            start = self.start.astimezone(tz).time().strftime("%H:%M")
            end = self.end.astimezone(tz).time().strftime("%H:%M")
            return """本日の CAMPHOR- HOUSE の開館時間は{}〜{}です。
みなさんのお越しをお待ちしています!!""".format(start, end)
        else:
            return None

@click.command(help="CAMPHOR- Schedule Notifier")
@click.option("--url", default="https://cal.camph.net/public/schedule.json",
              envvar="CSN_URL", help="URL of a schedule file.")
@click.option("--api-key", type=click.STRING,
              envvar="CSN_API_KEY", help="Twitter API Key.")
@click.option("--api-secret", type=click.STRING,
              envvar="CSN_API_SECRET", help="Twitter API Secret.")
@click.option("--access-token", type=click.STRING,
              envvar="CSN_ACCESS_TOKEN", help="Twitter Access Token.")
@click.option("--access-token-secret", type=click.STRING,
              envvar="CSN_ACCESS_TOKEN_SECRET",
              help="Twitter Access Token Secret.")
@click.option("--dry-run", "-n", default=False, is_flag=True,
              help="Write messages to stdout.")
@click.option("--timezone", default="Asia/Tokyo",
              help="Time zone used to show time. (default: Asia/Tokyo)")
def main(url: str, api_key: str, api_secret: str, access_token: str,
         access_token_secret: str, dry_run: bool, timezone: str):
    tz = pytz.timezone(timezone)
    now = datetime.now(tz=tz)

    events = download_events(url)
    if events is None:
        return
    messages = generate_messages(events, now, tz)
    if dry_run:
        for i, message in enumerate(messages):
            print("#{}\n{}".format(i + 1, message))
        return
    channel = TwitterChannel(api_key=api_key, api_secret=api_secret,
                             access_token=access_token,
                             access_token_secret=access_token_secret)
    for message in messages:
        channel.send(message)

def download_events(url: str) -> Optional[List[Event]]:
    response = requests.get(url)
    if response.status_code != requests.codes.ok:
        return None
    return [Event.from_json(e) for e in response.json()]

def generate_messages(events: Iterable[Event], now: datetime, tz) -> List[str]:
    now = now.astimezone(tz)
    events = filter(lambda e: e.start.astimezone(tz).date() == now.date(),
                    events)
    messages = [m for m in map(methodcaller("generate_message", tz), events)
                if m is not None]

    if len(messages) == 0:
        messages.append("本日の CAMPHOR- HOUSE は閉館です。")

    return messages

if __name__ == "__main__":
    main()
<commit_msg>Pass only datetime object to functions<commit_after>from datetime import datetime
from operator import methodcaller
from typing import Dict, Iterable, List, Optional

from channels.backends.twitter import TwitterChannel
import click
import dateutil.parser
import pytz
import requests

class Event:
    # flake8 does not support variable annotations...
    # start: datetime
    # end: datetime
    # title: str
    # url: Optional[str]

    def __init__(self, *, start, end, title, url):
        self.start = start
        self.end = end
        self.title = title
        self.url = url

    @classmethod
    def from_json(cls, data: Dict[str, Optional[str]]) -> "Event":
        start = dateutil.parser.parse(data["start"])
        end = dateutil.parser.parse(data["end"])
        return cls(start=start, end=end, title=data["title"], url=data["url"])

    def generate_message(self, now: datetime) -> Optional[str]:
        tz = now.tzinfo
        if tz is None:
            raise ValueError("'now' must be timezone aware datetime")
        # TODO: Support other kinds of events
        if self.title.lower() == "open":
            start = self.start.astimezone(tz).time().strftime("%H:%M")
            end = self.end.astimezone(tz).time().strftime("%H:%M")
            return """本日の CAMPHOR- HOUSE の開館時間は{}〜{}です。
みなさんのお越しをお待ちしています!!""".format(start, end)
        else:
            return None

@click.command(help="CAMPHOR- Schedule Notifier")
@click.option("--url", default="https://cal.camph.net/public/schedule.json",
              envvar="CSN_URL", help="URL of a schedule file.")
@click.option("--api-key", type=click.STRING,
              envvar="CSN_API_KEY", help="Twitter API Key.")
@click.option("--api-secret", type=click.STRING,
              envvar="CSN_API_SECRET", help="Twitter API Secret.")
@click.option("--access-token", type=click.STRING,
              envvar="CSN_ACCESS_TOKEN", help="Twitter Access Token.")
@click.option("--access-token-secret", type=click.STRING,
              envvar="CSN_ACCESS_TOKEN_SECRET",
              help="Twitter Access Token Secret.")
@click.option("--dry-run", "-n", default=False, is_flag=True,
              help="Write messages to stdout.")
@click.option("--timezone", default="Asia/Tokyo",
              help="Time zone used to show time. (default: Asia/Tokyo)")
def main(url: str, api_key: str, api_secret: str, access_token: str,
         access_token_secret: str, dry_run: bool, timezone: str):
    now = datetime.now(tz=pytz.timezone(timezone))

    events = download_events(url)
    if events is None:
        return
    messages = generate_messages(events, now)
    if dry_run:
        for i, message in enumerate(messages):
            print("#{}\n{}".format(i + 1, message))
        return
    channel = TwitterChannel(api_key=api_key, api_secret=api_secret,
                             access_token=access_token,
                             access_token_secret=access_token_secret)
    for message in messages:
        channel.send(message)

def download_events(url: str) -> Optional[List[Event]]:
    response = requests.get(url)
    if response.status_code != requests.codes.ok:
        return None
    return [Event.from_json(e) for e in response.json()]

def generate_messages(events: Iterable[Event], now: datetime) -> List[str]:
    tz = now.tzinfo
    if tz is None:
        raise ValueError("'now' must be timezone aware datetime")
    now = now.astimezone(tz)
    events = filter(lambda e: e.start.astimezone(tz).date() == now.date(),
                    events)
    messages = [m for m in map(methodcaller("generate_message", now), events)
                if m is not None]

    if len(messages) == 0:
        messages.append("本日の CAMPHOR- HOUSE は閉館です。")

    return messages

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests, sys, time, os, json, shutil, argparse, datetime

# Parse arguments
parser = argparse.ArgumentParser(description='Twitch Chat Downloader')
parser.add_argument('-v', '--video', help='Video id')
parser.add_argument('-i', '--client-id', help='Twitch client id')
parser.add_argument('-p', '--print', dest='p', help='Print messages', action='store_true')
parser.add_argument('-o', '--output', help='Output folder')
parser.add_argument('-f', '--format', help='Message format', choices=['timestamp', 'relative', 'srt', 'ssa', 'ass', 'raw'])
parser.add_argument('--cooldown', type=float, help='Cooldown time between API requests in seconds')
parser.add_argument('--start', type=int, help='Start time in seconds from video start')
parser.add_argument('--stop', type=int, help='Stop time in seconds from video start')

arguments = parser.parse_args()

# Get video ID
if arguments.video:
    videoId = 'v' + arguments.video.replace('v', '')
else:
    videoId = 'v' + raw_input('Video ID: ').replace('v', '')

# Program requires at least example.setting.json or settings.json to run properly
if not os.path.isfile('example.settings.json') and not os.path.isfile('settings.json'):
    print 'Error: Missing settings file.'
    sys.exit(1)

# Copy settings example file if settings.json doesn't exist
if not os.path.isfile('settings.json'):
    shutil.copyfile('example.settings.json', 'settings.json')

# Load settings
with open('settings.json', 'r') as settings_file:
    settings = json.load(settings_file)

# Check settings version
if os.path.isfile('example.settings.json'):
    with open('example.settings.json', 'r') as example_settings_file:
        exampleSettings = json.load(example_settings_file)
        if 'version' not in settings:
            print '[Warning]\nYour settings.json file does not contain a version number. Compare settings.json to example.settings.json to make sure it\'s up to date.\n'

        elif 'version' in settings and settings['version'] != exampleSettings['version']:
            print '[Warning]\nYour settings.json file is outdated. Compare settings.json to example.settings.json.\nYour version: ' + settings['version'] + '\nNewest version: ' + exampleSettings['version'] + '\n'

# Check if a client_id was provided as an argument
if arguments.client_id:
    if not arguments.client_id == settings['client_id']:
        settings['client_id'] = arguments.client_id
        answer = raw_input('Save client ID? (Y/n): ')
        if (not answer.lower() == "n"):
            with open('settings.json', 'w') as settings_file:
                json.dump(settings, settings_file)

# Check if client_id is required
if settings['require_client_id'] and not settings['client_id']:
    print "Twitch requires a client ID to use their API.\nRegister an application on https://www.twitch.tv/settings/connections to get yours."
    settings['client_id'] = raw_input('Client ID: ')
    answer = raw_input('Save client ID? (Y/n): ')
    if (not answer.lower() == "n"):
        with open('settings.json', 'w') as settings_file:
            json.dump(settings, settings_file)

# Get client id parameter for URL
def getClientIdParameter():
    if (settings['client_id']):
        return 'client_id='+settings['client_id'] + '&'
    else:
        return ''

# Overwrite remaining settings with arguments
if arguments.p:
    settings['print'] = arguments.p

if arguments.format:
    settings['format'] = arguments.format

if arguments.cooldown:
    settings['cooldown'] = arguments.cooldown

# API URL
apiUrl = 'https://rechat.twitch.tv/rechat-messages'

# Get start and stop time by looking at the 'detail' message from Twitch
#
# If you query this API with invalid an invalid timestamp (none or out of range),
# it will tell you the start and stop timestamp, however, in text format.
response = requests.get(apiUrl + '?' + str(getClientIdParameter()) + 'start=0&video_id=' + videoId).json()

# Parse response for start and stop
#
# The response will look something like this
# {
#   "errors": [
#     {
#       "status": 400,
#       "detail": "0 is not between 1469108651 and 1469133795"
#     }
#   ]
# }
#
# As the start and stop timestamp is (for some weird reason)
# in text format, we have to parse the response.
detail = response['errors'][0]['detail'].split(' ') # We split the detail string into an array

# Check if valid video ID
# If the length is 8, it's (most likely) invalid
# If the length is 7, it's (most likely) valid
if len(detail) != 7:
    if settings['require_client_id']:
        print 'Error: Invalid video or client ID'
    else:
        print 'Error: Invalid video ID'
    sys.exit(1)

# Start and stop points
start = int(detail[4])    # The start timestamp is on index 4
stop = int(detail[6])     # while stop has the index 6

# Original start and stop
fullStart = start         # Keep original full-length start point
fullStop = stop           # Keep original full-length stop potin

# Used message ids
#
# Every message has an unique ID, which can be used for checking if we've already stored it.
messageIds = []

# Open output file
#
# This is where we save the messages.

# Output directory

if arguments.output:
    directory = arguments.output
else:
    directory = settings['output']

if not os.path.exists(directory):
    os.makedirs(directory)

# Open file (different file extension for subtitle formats)
if settings['format'] == 'srt' or settings['format'] == 'ssa' or settings['format'] == 'ass':
    file = open(directory + '/' + videoId + '.' + settings['format'], 'w')
elif settings['format'] == 'raw':
    file = open(directory + '/' + videoId + '.json', 'w')
else:
    file = open(directory + '/' + videoId + '.txt', 'w')

# Add format line if SSA/ASS subtitle format
if settings['format'] == 'ssa' or settings['format'] == 'ass':
    file.write('[Script Info]\n')
    file.write('Timer: 100,0000\n')

    file.write('\n[V4 Styles]\n')
    file.write(settings['ssa_style_format'] + '\n')
    file.write(settings['ssa_style_default']  + '\n')

    file.write('\n[Events]\n')
    file.write(settings['ssa_events_format'] + '\n')

# When saving as raw format (json), messages will be added to this
# object array and written to file after fetching all the messages
# to avoid opening, reading, writing to and closing the file for every
# message.
rawData = []

# Stop time argument
if arguments.stop and start + arguments.stop <= stop:
    stop = start + arguments.stop

# Start time argument
if arguments.start:
    start += arguments.start

# Download messages from timestamps between start and stop.
timestamp = start

while timestamp <= stop:

    # Wait for cooldown timer and request new messages from Twitch
    # The API returns the next 30 seconds of messages
    time.sleep(settings['cooldown'])
    response = requests.get(apiUrl + '?start=' + str(timestamp) + '&video_id=' + videoId).json()
    data = response['data'];

    # Increase timestamp to get the next 30 seconds of messages in the next loop
    timestamp += 30

    for message in data:

        # Timestamp for message (seconds)
        messageTimestampInSeconds = message['attributes']['timestamp']/1000.

        # Check the unique message ID to make sure it's not already saved.
        if not any(message['id'] in s for s in messageIds) and messageTimestampInSeconds <= stop:

            # If this is a new message, save the unique ID to prevent duplication later.
            messageIds.append(message['id'])
            date    = time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(messageTimestampInSeconds))
            sender  = message['attributes']['from'].encode('utf-8')
            color   = message['attributes']['color']
            text    = message['attributes']['message'].encode('utf-8')

            if color is None:
                color = '000000'
            else:
                color = color.replace('#', '')

            # Timestamp format
            if settings['format'] == 'timestamp':
                line = date + ' ' + sender + ': ' + text + '\n'
                printLine = '\033[94m' + date + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            # Relative timestamp format
            if settings['format'] == 'relative':
                line = str(datetime.timedelta(seconds=messageTimestampInSeconds - start)) + ' ' + sender + ': ' + text + '\n'
                printLine = '\033[94m' + str(datetime.timedelta(seconds=messageTimestampInSeconds-start)) + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            # Subtitle formats
            if ['srt', 'ass', 'ssa'].index(settings['format']) != -1:
                subtitleStart = str(datetime.timedelta(seconds=messageTimestampInSeconds - start))
                subtitleStop = str(datetime.timedelta(seconds=messageTimestampInSeconds - start + 2))

                # Bugfix - add milliseconds if missing
                # https://github.com/PetterKraabol/Twitch-Chat-Downloader/issues/3
                if len(subtitleStart) == 7:
                    subtitleStart += '.000000'

                if len(subtitleStop) == 7:
                    subtitleStop += '.000000'

                # srt format
                if settings['format'] == 'srt':
                    line = str(len(messageIds)) + '\n' + subtitleStart[:-3] + ' --> ' + subtitleStop[:-3] + '\n' + sender + ': ' + text + '\n\n'
                    printLine = printLine = '\033[94m' + subtitleStart + ' \033[92m'+ sender + '\033[0m' + ': ' + text

                # SSA/ASS format
                if settings['format'] == 'ssa' or settings['format'] == 'ass':
                    line = 'Dialogue: Marked=0, ' + subtitleStart[:-4] + ', ' + subtitleStop[:-4] + ', Default, ' + sender + ', 0000, 0000, 0000 , , {\c&H' + color + '&}' + sender + '{\c&H000000&}: ' + text + '\n'
                    printLine = printLine = '\033[94m' + str(datetime.timedelta(seconds=messageTimestampInSeconds - start)) + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            if settings['format'] == 'raw':
                rawData.append(message)

            # Save messages to file unless saving raw data.
            # This is done after download all messages
            if settings['format'] != 'raw':
                file.write(line)

            # Print messages, if not, show progress
            if settings['print']:
                print printLine
            else:

                #Show progress %
                progress = round((timestamp - start)*100 / float(stop - start), 2)

                # Bugfix: progress can go slightly above 100% on the last loop
                if progress > 100.0:
                    progress = 100.0

                sys.stdout.write('Downloading ' + str(int(messageTimestampInSeconds - start)) + '/' + str(stop - start) + 's (' + str(progress) + '%) \r')
                sys.stdout.flush()

# If format is set to raw, save raw data
if settings['format'] == 'raw':
    file.write(json.dumps(rawData))

# Close file
file.close()
sys.stdout.write('Finished downloading ' + videoId + '\r')
<commit_msg>Fixed issue 5: correctly checking if format is srt, ass or ssa<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import requests, sys, time, os, json, shutil, argparse, datetime

# Parse arguments
parser = argparse.ArgumentParser(description='Twitch Chat Downloader')
parser.add_argument('-v', '--video', help='Video id')
parser.add_argument('-i', '--client-id', help='Twitch client id')
parser.add_argument('-p', '--print', dest='p', help='Print messages', action='store_true')
parser.add_argument('-o', '--output', help='Output folder')
parser.add_argument('-f', '--format', help='Message format', choices=['timestamp', 'relative', 'srt', 'ssa', 'ass', 'raw'])
parser.add_argument('--cooldown', type=float, help='Cooldown time between API requests in seconds')
parser.add_argument('--start', type=int, help='Start time in seconds from video start')
parser.add_argument('--stop', type=int, help='Stop time in seconds from video start')

arguments = parser.parse_args()

# Get video ID
if arguments.video:
    videoId = 'v' + arguments.video.replace('v', '')
else:
    videoId = 'v' + raw_input('Video ID: ').replace('v', '')

# Program requires at least example.setting.json or settings.json to run properly
if not os.path.isfile('example.settings.json') and not os.path.isfile('settings.json'):
    print 'Error: Missing settings file.'
    sys.exit(1)

# Copy settings example file if settings.json doesn't exist
if not os.path.isfile('settings.json'):
    shutil.copyfile('example.settings.json', 'settings.json')

# Load settings
with open('settings.json', 'r') as settings_file:
    settings = json.load(settings_file)

# Check settings version
if os.path.isfile('example.settings.json'):
    with open('example.settings.json', 'r') as example_settings_file:
        exampleSettings = json.load(example_settings_file)
        if 'version' not in settings:
            print '[Warning]\nYour settings.json file does not contain a version number. Compare settings.json to example.settings.json to make sure it\'s up to date.\n'

        elif 'version' in settings and settings['version'] != exampleSettings['version']:
            print '[Warning]\nYour settings.json file is outdated. Compare settings.json to example.settings.json.\nYour version: ' + settings['version'] + '\nNewest version: ' + exampleSettings['version'] + '\n'

# Check if a client_id was provided as an argument
if arguments.client_id:
    if not arguments.client_id == settings['client_id']:
        settings['client_id'] = arguments.client_id
        answer = raw_input('Save client ID? (Y/n): ')
        if (not answer.lower() == "n"):
            with open('settings.json', 'w') as settings_file:
                json.dump(settings, settings_file)

# Check if client_id is required
if settings['require_client_id'] and not settings['client_id']:
    print "Twitch requires a client ID to use their API.\nRegister an application on https://www.twitch.tv/settings/connections to get yours."
    settings['client_id'] = raw_input('Client ID: ')
    answer = raw_input('Save client ID? (Y/n): ')
    if (not answer.lower() == "n"):
        with open('settings.json', 'w') as settings_file:
            json.dump(settings, settings_file)

# Get client id parameter for URL
def getClientIdParameter():
    if (settings['client_id']):
        return 'client_id='+settings['client_id'] + '&'
    else:
        return ''

# Overwrite remaining settings with arguments
if arguments.p:
    settings['print'] = arguments.p

if arguments.format:
    settings['format'] = arguments.format

if arguments.cooldown:
    settings['cooldown'] = arguments.cooldown

# API URL
apiUrl = 'https://rechat.twitch.tv/rechat-messages'

# Get start and stop time by looking at the 'detail' message from Twitch
#
# If you query this API with invalid an invalid timestamp (none or out of range),
# it will tell you the start and stop timestamp, however, in text format.
response = requests.get(apiUrl + '?' + str(getClientIdParameter()) + 'start=0&video_id=' + videoId).json()

# Parse response for start and stop
#
# The response will look something like this
# {
#   "errors": [
#     {
#       "status": 400,
#       "detail": "0 is not between 1469108651 and 1469133795"
#     }
#   ]
# }
#
# As the start and stop timestamp is (for some weird reason)
# in text format, we have to parse the response.
detail = response['errors'][0]['detail'].split(' ') # We split the detail string into an array

# Check if valid video ID
# If the length is 8, it's (most likely) invalid
# If the length is 7, it's (most likely) valid
if len(detail) != 7:
    if settings['require_client_id']:
        print 'Error: Invalid video or client ID'
    else:
        print 'Error: Invalid video ID'
    sys.exit(1)

# Start and stop points
start = int(detail[4])    # The start timestamp is on index 4
stop = int(detail[6])     # while stop has the index 6

# Original start and stop
fullStart = start         # Keep original full-length start point
fullStop = stop           # Keep original full-length stop potin

# Used message ids
#
# Every message has an unique ID, which can be used for checking if we've already stored it.
messageIds = []

# Open output file
#
# This is where we save the messages.

# Output directory

if arguments.output:
    directory = arguments.output
else:
    directory = settings['output']

if not os.path.exists(directory):
    os.makedirs(directory)

# Open file (different file extension for subtitle formats)
if settings['format'] == 'srt' or settings['format'] == 'ssa' or settings['format'] == 'ass':
    file = open(directory + '/' + videoId + '.' + settings['format'], 'w')
elif settings['format'] == 'raw':
    file = open(directory + '/' + videoId + '.json', 'w')
else:
    file = open(directory + '/' + videoId + '.txt', 'w')

# Add format line if SSA/ASS subtitle format
if settings['format'] == 'ssa' or settings['format'] == 'ass':
    file.write('[Script Info]\n')
    file.write('Timer: 100,0000\n')

    file.write('\n[V4 Styles]\n')
    file.write(settings['ssa_style_format'] + '\n')
    file.write(settings['ssa_style_default']  + '\n')

    file.write('\n[Events]\n')
    file.write(settings['ssa_events_format'] + '\n')

# When saving as raw format (json), messages will be added to this
# object array and written to file after fetching all the messages
# to avoid opening, reading, writing to and closing the file for every
# message.
rawData = []

# Stop time argument
if arguments.stop and start + arguments.stop <= stop:
    stop = start + arguments.stop

# Start time argument
if arguments.start:
    start += arguments.start

# Download messages from timestamps between start and stop.
timestamp = start

while timestamp <= stop:

    # Wait for cooldown timer and request new messages from Twitch
    # The API returns the next 30 seconds of messages
    time.sleep(settings['cooldown'])
    response = requests.get(apiUrl + '?start=' + str(timestamp) + '&video_id=' + videoId).json()
    data = response['data'];

    # Increase timestamp to get the next 30 seconds of messages in the next loop
    timestamp += 30

    for message in data:

        # Timestamp for message (seconds)
        messageTimestampInSeconds = message['attributes']['timestamp']/1000.

        # Check the unique message ID to make sure it's not already saved.
        if not any(message['id'] in s for s in messageIds) and messageTimestampInSeconds <= stop:

            # If this is a new message, save the unique ID to prevent duplication later.
            messageIds.append(message['id'])
            date    = time.strftime('%Y-%m-%d %H:%M:%S %Z', time.gmtime(messageTimestampInSeconds))
            sender  = message['attributes']['from'].encode('utf-8')
            color   = message['attributes']['color']
            text    = message['attributes']['message'].encode('utf-8')

            if color is None:
                color = '000000'
            else:
                color = color.replace('#', '')

            # Timestamp format
            if settings['format'] == 'timestamp':
                line = date + ' ' + sender + ': ' + text + '\n'
                printLine = '\033[94m' + date + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            # Relative timestamp format
            if settings['format'] == 'relative':
                line = str(datetime.timedelta(seconds=messageTimestampInSeconds - start)) + ' ' + sender + ': ' + text + '\n'
                printLine = '\033[94m' + str(datetime.timedelta(seconds=messageTimestampInSeconds-start)) + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            # Subtitle formats
            if settings['format'] in {'srt', 'ass', 'ssa'}:
                subtitleStart = str(datetime.timedelta(seconds=messageTimestampInSeconds - start))
                subtitleStop = str(datetime.timedelta(seconds=messageTimestampInSeconds - start + 2))

                # Bugfix - add milliseconds if missing
                # https://github.com/PetterKraabol/Twitch-Chat-Downloader/issues/3
                if len(subtitleStart) == 7:
                    subtitleStart += '.000000'

                if len(subtitleStop) == 7:
                    subtitleStop += '.000000'

                # srt format
                if settings['format'] == 'srt':
                    line = str(len(messageIds)) + '\n' + subtitleStart[:-3] + ' --> ' + subtitleStop[:-3] + '\n' + sender + ': ' + text + '\n\n'
                    printLine = printLine = '\033[94m' + subtitleStart + ' \033[92m'+ sender + '\033[0m' + ': ' + text

                # SSA/ASS format
                if settings['format'] == 'ssa' or settings['format'] == 'ass':
                    line = 'Dialogue: Marked=0, ' + subtitleStart[:-4] + ', ' + subtitleStop[:-4] + ', Default, ' + sender + ', 0000, 0000, 0000 , , {\c&H' + color + '&}' + sender + '{\c&H000000&}: ' + text + '\n'
                    printLine = printLine = '\033[94m' + str(datetime.timedelta(seconds=messageTimestampInSeconds - start)) + ' \033[92m'+ sender + '\033[0m' + ': ' + text

            if settings['format'] == 'raw':
                rawData.append(message)

            # Save messages to file unless saving raw data.
            # This is done after download all messages
            if settings['format'] != 'raw':
                file.write(line)

            # Print messages, if not, show progress
            if settings['print']:
                print printLine
            else:

                #Show progress %
                progress = round((timestamp - start)*100 / float(stop - start), 2)

                # Bugfix: progress can go slightly above 100% on the last loop
                if progress > 100.0:
                    progress = 100.0

                sys.stdout.write('Downloading ' + str(int(messageTimestampInSeconds - start)) + '/' + str(stop - start) + 's (' + str(progress) + '%) \r')
                sys.stdout.flush()

# If format is set to raw, save raw data
if settings['format'] == 'raw':
    file.write(json.dumps(rawData))

# Close file
file.close()
sys.stdout.write('Finished downloading ' + videoId + '\r')
<|endoftext|>


<commit_before>#!/usr/bin/python2
from gevent import monkey
monkey.patch_all()

from threading import Thread
import time
import logging
from flask import Flask, send_from_directory
from flask.ext.socketio import SocketIO, emit
import fermenter

logging.basicConfig()

###############################################################################
# PARAMETERS
###############################################################################
STATS_INTERVAL = 10 # (sec): time to wait between updating statas

###############################################################################
# GLOBALS
###############################################################################
app = Flask(__name__)
socketio = SocketIO(app)
threads = {}

###############################################################################
# EVENTS
###############################################################################
@socketio.on("socket event", namespace="/socket")
def handle_socket_event(message):
    print(message.data)

###############################################################################
# THREADS
###############################################################################
def update_stats(records, locks):
    while True:
        stats = {
            "start": records["start"],
            "stop": records["stop"],
            "temp": records["temp"][-1],
            "heater": records["heater"][-1],
            "impeller": records["impeller"][-1],
            "optics": {
                "calibration": {
                    "red": records["optics"]["calibration"]["red"],
                    "green": records["optics"]["calibration"]["green"],
                },
                "calibrations": records["optics"]["calibrations"],
                "ambient": records["optics"]["ambient"][-1],
                "red": records["optics"]["red"][-1],
                "green": records["optics"]["green"][-1],
            },
        }
        socketio.emit("stats update", stats, namespace="/socket")
        time.sleep(STATS_INTERVAL)

###############################################################################
# ROUTES
###############################################################################
@app.route("/")
def index():
    """Deliver the dashboard"""
    global threads
    if "stats" not in threads.keys():
        threads["stats"] = Thread(target=update_stats, name="stats",
                                  args=(records, locks))
        threads["stats"].start()
    return send_from_directory("static", "dashboard.html")

###############################################################################
# MAIN
###############################################################################
if __name__ == "__main__":
    (records, locks, events, threads) = fermenter.run_fermenter()
    socketio.run(app, host='0.0.0.0', port=80)
<commit_msg>Fixed issue with the connection event handler<commit_after>#!/usr/bin/python2
from gevent import monkey
monkey.patch_all()

from threading import Thread
import time
import logging
from flask import Flask, send_from_directory
from flask.ext.socketio import SocketIO, emit
import fermenter

logging.basicConfig()

###############################################################################
# PARAMETERS
###############################################################################
STATS_INTERVAL = 10 # (sec): time to wait between updating statas

###############################################################################
# GLOBALS
###############################################################################
app = Flask(__name__)
socketio = SocketIO(app)
threads = {}

###############################################################################
# EVENTS
###############################################################################
@socketio.on("socket event", namespace="/socket")
def handle_socket_event(message):
    print(message)

###############################################################################
# THREADS
###############################################################################
def update_stats(records, locks):
    while True:
        stats = {
            "start": records["start"],
            "stop": records["stop"],
            "temp": records["temp"][-1],
            "heater": records["heater"][-1],
            "impeller": records["impeller"][-1],
            "optics": {
                "calibration": {
                    "red": records["optics"]["calibration"]["red"],
                    "green": records["optics"]["calibration"]["green"],
                },
                "calibrations": records["optics"]["calibrations"],
                "ambient": records["optics"]["ambient"][-1],
                "red": records["optics"]["red"][-1],
                "green": records["optics"]["green"][-1],
            },
        }
        socketio.emit("stats update", stats, namespace="/socket")
        time.sleep(STATS_INTERVAL)

###############################################################################
# ROUTES
###############################################################################
@app.route("/")
def index():
    """Deliver the dashboard"""
    global threads
    if "stats" not in threads.keys():
        threads["stats"] = Thread(target=update_stats, name="stats",
                                  args=(records, locks))
        threads["stats"].start()
    return send_from_directory("static", "dashboard.html")

###############################################################################
# MAIN
###############################################################################
if __name__ == "__main__":
    (records, locks, events, threads) = fermenter.run_fermenter()
    socketio.run(app, host='0.0.0.0', port=80)
<|endoftext|>