<commit_before>"""
This module defines functions which create global parser
and parsers for each of subcommands.
"""

import argparse

problems = [
    'quadratic',
    'rosenbrock',
    'mixed',
    'logreg',
    'stoch_logreg',
    'stoch_linear',
    'mixed_stoch',
    'mixed_nonstoch'
]

def make_train_parser(parser):
    """
    This function creates subparser for 'train' command.
    """
    parser.add_argument('--eid', type=int, default=0, help='epoch id from which start training')

    parser.add_argument('--optimizer', type=str,
                        default='adam', choices=['adam', 'momentum'],
                        help='optimizer to train LSTM')

    parser.add_argument('--train_lr', type=float, default=1e-4, help='learning rate')
    parser.add_argument('--momentum', type=float, default=0.9, help='momentum')

    parser.add_argument('--loss_type', type=str,
                        choices=['log', 'sum', 'last'], default='log',
                        help='loss function to use')

    parser.add_argument('--lambd', type=float, default=0)

    parser.add_argument('--n_steps', type=int, default=100, help='number of steps')
    parser.add_argument('--n_bptt_steps', type=int, default=20, help='number of bptt steps')

    parser.add_argument('--n_epochs', type=int, default=100, help='number of epochs')
    parser.add_argument('--n_batches', type=int, default=100, help='number of batches per epoch')
    parser.add_argument('--batch_size', type=int, default=100, help='batch size')

    parser.add_argument('-opt', '--optimizee', type=str, nargs='+',
                        choices=problems, default='all',
                        help='space separated list of optimizees or all')

    parser.add_argument('--enable_random_scaling', action='store_true',
                        help='enable random scaling of problems')

    parser.add_argument('--noisy_grad', action='store_true',
                        help='add normal noise to gradients of non-stochastic problems')

    parser.add_argument('-f', '--force', action='store_true', help='force overwrite of checkpoint')

    return parser

def make_test_parser(parser):
    """
    This function creates subparser for 'test' command.
    """
    parser.add_argument('eid', type=int, help='epoch id from which test optimizer')
    parser.add_argument('problem', choices=problems, help='problem to run test on')
    parser.add_argument('mode', type=str, choices=['many', 'cv'], help='which mode to run')
    parser.add_argument('--enable_random_scaling', action='store_true',
                        help='enable random scaling of problems')
    parser.add_argument('--noisy_grad', action='store_true',
                        help='add normal noise to gradients of non-stochastic problems')

    parser.add_argument('--n_steps', type=int, default=1000, help='number of steps')
    parser.add_argument('--n_batches', type=int, default=100, help='number of batches per epoch')

    parser.add_argument('--compare_with', type=str, default='momentum',
                        choices=['sgd', 'momentum'], help='baseline for optimizer')

    parser.add_argument('--start_eid', type=int, default=100,
                        help='epoch from which start to run cv')
    parser.add_argument('--step', type=int, default=100, help='step in number of epochs for cv')
    parser.add_argument('--snapshot_path', type=str)

    return parser

def make_plot_parser(parser):
    """
    This function creates subparser for 'plot' command.
    """
    parser.add_argument('name', type=str, help='name of the model')
    parser.add_argument('phase', type=str, choices=['train', 'test', 'cv'],
                        help='train or test phase')
    parser.add_argument('--plot_lr', action='store_true',
                        help='plot learning rate')
    parser.add_argument('--plot_moving', action='store_true',
                        help='plot moving loss')
    parser.add_argument('-p', '--problem', type=str,
                        help='optimizee name')
    parser.add_argument('-m', '--mode', type=str, choices=['many', 'cv'],
                        help='mode of testing')
    parser.add_argument('-f', '--frac', type=float, default=1.0,
                        help='fraction of data to plot')
    parser.add_argument('-t', '--tag', type=str)
    parser.add_argument('-s', '--stochastic', action='store_true',
                        help='whether problem is stochastic')

    return parser

def make_cv_parser(parser):
    """
    This function creates subparser for 'cv' command.
    """
    methods = ['grid', 'random', 'bayesian']

    parser.add_argument('config', type=str,
                        help='path to parameter grid')
    parser.add_argument('--method', type=str, choices=methods, default='grid',
                        help='type of tuning')
    parser.add_argument('--num_tries', type=int, default=5,
                        help='number of tries for random cv')

    return parser

def make_new_parser(parser):
    """
    This function creates subparser for 'new' command.
    """
    parser.add_argument('name', type=str, help='name of the model')
    parser.add_argument('num_units', type=int, help='number of units in LSTM')
    parser.add_argument('num_layers', type=int, help='number of lstm layers')
    parser.add_argument('--layer_norm', action='store_true',
                        help='enable layer normalization')
    parser.add_argument('--add_skip', action='store_true',
                        help='add adam output to LSTM output')
    parser.add_argument('--no_stop_grad', action='store_false', dest='stop_grad',
                        help='whether to compute second derivatives')
    parser.add_argument('--rnn_type', type=str, choices=['lstm', 'gru'], default='lstm',
                        help='cell to use: LSTM or GRU')
    parser.add_argument('-f', '--force', action='store_true')

    return parser

def make_parser():
    """
    This function assemble global parser from parsers for subcommands.
    """
    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
    subparsers = parser.add_subparsers(help='command to run', dest='command_name')

    run_parser = argparse.ArgumentParser(add_help=False)
    run_parser.add_argument('name', type=str, help='name of the model')

    group = run_parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--cpu', action='store_true', help='run model on CPU')
    group.add_argument('--gpu', type=int, nargs='+', help='gpu id')

    run_parser.add_argument('--verbose', type=int, choices=[0, 1, 2], default=1)
    run_parser.add_argument('--tag', type=str, help='tag denoting run purpose/parameters')

    parser_new = subparsers.add_parser('new', help='add new model')
    parser_train = subparsers.add_parser('train', parents=[run_parser],
                                         help='train optimizer on a set of functions')
    parser_test = subparsers.add_parser('test', parents=[run_parser],
                                        help='run trained optimizer on some problem')
    parser_cv = subparsers.add_parser('cv', parents=[parser_train], add_help=False,
                                      help='tune hyperparameters by validation')
    parser_plot = subparsers.add_parser('plot', help='plot dumped results')

    parser_new = make_new_parser(parser_new)
    parser_train = make_train_parser(parser_train)
    parser_test = make_test_parser(parser_test)
    parser_cv = make_cv_parser(parser_cv)
    parser_plot = make_plot_parser(parser_plot)

    return parser
<commit_msg>updated CLI<commit_after>"""
This module defines functions which create global parser
and parsers for each of subcommands.
"""

import argparse

problems = [
    'quadratic',
    'rosenbrock',
    'mixed',
    'logreg',
    'stoch_logreg',
    'stoch_linear',
    'mixed_stoch',
    'mixed_nonstoch',
    'digits_classifier',
    'beale', 'booth', 'matyas'
]

def make_train_parser(parser):
    """
    This function creates subparser for 'train' command.
    """
    parser.add_argument('--eid', type=int, default=0, help='epoch id from which start training')

    parser.add_argument('--optimizer', type=str,
                        default='adam', choices=['adam', 'momentum'],
                        help='optimizer to train LSTM')

    parser.add_argument('--train_lr', type=float, default=1e-4, help='learning rate')
    parser.add_argument('--momentum', type=float, default=0.9, help='momentum')

    parser.add_argument('--loss_type', type=str,
                        choices=['log', 'sum', 'last'], default='log',
                        help='loss function to use')

    parser.add_argument('--lambd', type=float, default=0)

    parser.add_argument('--n_steps', type=int, default=100, help='number of steps')
    parser.add_argument('--n_bptt_steps', type=int, default=20, help='number of bptt steps')

    parser.add_argument('--n_epochs', type=int, default=100, help='number of epochs')
    parser.add_argument('--n_batches', type=int, default=100, help='number of batches per epoch')
    parser.add_argument('--batch_size', type=int, default=100, help='batch size')

    parser.add_argument('-opt', '--optimizee', type=str, nargs='+',
                        choices=problems, default='all',
                        help='space separated list of optimizees or all')

    parser.add_argument('--enable_random_scaling', action='store_true',
                        help='enable random scaling of problems')

    parser.add_argument('--noisy_grad', action='store_true',
                        help='add normal noise to gradients of non-stochastic problems')

    parser.add_argument('-f', '--force', action='store_true', help='force overwrite of checkpoint')

    return parser

def make_test_parser(parser):
    """
    This function creates subparser for 'test' command.
    """
    parser.add_argument('eid', type=int, help='epoch id from which test optimizer')
    parser.add_argument('problem', choices=problems, help='problem to run test on')
    parser.add_argument('mode', type=str, choices=['many', 'cv'], help='which mode to run')
    parser.add_argument('--enable_random_scaling', action='store_true',
                        help='enable random scaling of problems')
    parser.add_argument('--noisy_grad', action='store_true',
                        help='add normal noise to gradients of non-stochastic problems')

    parser.add_argument('--n_steps', type=int, default=1000, help='number of steps')
    parser.add_argument('--n_batches', type=int, default=100, help='number of batches per epoch')

    parser.add_argument('--compare_with', type=str, default='momentum',
                        choices=['sgd', 'momentum', 'adam'], help='baseline for optimizer')

    parser.add_argument('--start_eid', type=int, default=100,
                        help='epoch from which start to run cv')
    parser.add_argument('--step', type=int, default=100, help='step in number of epochs for cv')
    parser.add_argument('--snapshot_path', type=str)

    return parser

def make_plot_parser(parser):
    """This function creates subparser for 'plot' command."""
    parser.add_argument('name', type=str, help='name of the model')
    parser.add_argument('phase', type=str, choices=['train', 'test', 'cv'],
                        help='train or test phase')
    parser.add_argument('--plot_lr', action='store_true',
                        help='plot learning rate')
    parser.add_argument('--plot_moving', action='store_true',
                        help='plot moving loss')
    parser.add_argument('-p', '--problem', type=str,
                        help='optimizee name')
    parser.add_argument('-m', '--mode', type=str, choices=['many', 'cv'],
                        help='mode of testing')
    parser.add_argument('-f', '--frac', type=float, default=1.0,
                        help='fraction of data to plot')
    parser.add_argument('-t', '--tag', type=str)
    parser.add_argument('-s', '--stochastic', action='store_true',
                        help='whether problem is stochastic')
    parser.add_argument('--compare_with', type=str)

    return parser

def make_cv_parser(parser):
    """
    This function creates subparser for 'cv' command.
    """
    methods = ['grid', 'random', 'bayesian']

    parser.add_argument('config', type=str,
                        help='path to parameter grid')
    parser.add_argument('--method', type=str, choices=methods, default='grid',
                        help='type of tuning')
    parser.add_argument('--num_tries', type=int, default=5,
                        help='number of tries for random cv')

    return parser

def make_new_parser(parser):
    """This function creates subparser for 'new' command."""
    parser.add_argument('name', type=str, help='name of the model')
    parser.add_argument('num_units', type=int, help='number of units in LSTM')
    parser.add_argument('num_layers', type=int, help='number of lstm layers')
    parser.add_argument('--layer_norm', action='store_true',
                        help='enable layer normalization')
    parser.add_argument('--add_skip', action='store_true',
                        help='add adam output to LSTM output')
    parser.add_argument('--no_stop_grad', action='store_false', dest='stop_grad',
                        help='whether to compute second derivatives')
    parser.add_argument('--rnn_type', type=str, choices=['lstm', 'gru'], default='lstm',
                        help='cell to use: LSTM or GRU')
    parser.add_argument('--residual', action='store_true',
                        help='add residual connections in lstm')
    parser.add_argument('-f', '--force', action='store_true')

    return parser

def make_parser():
    """
    This function assemble global parser from parsers for subcommands.
    """
    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
    subparsers = parser.add_subparsers(help='command to run', dest='command_name')

    run_parser = argparse.ArgumentParser(add_help=False)
    run_parser.add_argument('name', type=str, help='name of the model')

    group = run_parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--cpu', action='store_true', help='run model on CPU')
    group.add_argument('--gpu', type=int, nargs='+', help='gpu id')

    run_parser.add_argument('--verbose', type=int, choices=[0, 1, 2], default=1)
    run_parser.add_argument('--tag', type=str, help='tag denoting run purpose/parameters')

    parser_new = subparsers.add_parser('new', help='add new model')
    parser_train = subparsers.add_parser('train', parents=[run_parser],
                                         help='train optimizer on a set of functions')
    parser_test = subparsers.add_parser('test', parents=[run_parser],
                                        help='run trained optimizer on some problem')
    parser_cv = subparsers.add_parser('cv', parents=[parser_train], add_help=False,
                                      help='tune hyperparameters by validation')
    parser_plot = subparsers.add_parser('plot', help='plot dumped results')

    parser_new = make_new_parser(parser_new)
    parser_train = make_train_parser(parser_train)
    parser_test = make_test_parser(parser_test)
    parser_cv = make_cv_parser(parser_cv)
    parser_plot = make_plot_parser(parser_plot)

    return parser
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

import itchat
from itchat.content import *
from random import choice
import time
import os
import re
import shutil
import spider, pymath # 我的一些功能

GREETINGS = ('嘿','哟','你好啊','你好呀',':)')
DONT_UNDERSTAND = ('对不起我不明白诶……','抱歉我没听懂:(','不好意思没明白','Sorry没有听懂')
HELP = '我现在虽然聊天不太行，只能打打招呼，但是我可以报时，帮你搜集新闻和热点，甚至还可以讲笑话。我还会求数列的公式！'
WEEKDAYS = {'0':'天','1':'一','2':'二','3':'三','4':'四','5':'五','6':'六'}

@itchat.msg_register(['Picture', 'Recording', 'Attachment', 'Video'])
def download_files(msg):
    # msg['Text'](msg['FileName'])
    itchat.send('@%s@%s'%('img' if msg['Type'] == 'Picture' else 'fil', 'test.png'), msg['FromUserName'])
    return '%s received'%msg['Type']

@itchat.msg_register(TEXT, isGroupChat=True)
def text_reply(msg):
    if msg.get('isAt') or '@饺子机器人\u2005' in msg['Content']: # 如果@了我
        cont = msg['Content'].replace('@熏鱼饺子\u2005','').replace('@饺子机器人\u2005','')
        print(cont)
        if '帮助' in cont or ('你' in cont and '除了' in cont and '还' in cont) or ('你' in cont and ('会' in cont or '能' in cont) and ('干' in cont or '做' in cont)):
            itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], HELP), msg['FromUserName'])
        elif '几点' in cont or ('现在' in cont and '时间' in cont):
            itchat.send('@{}\u2005现在是{}'.format(msg['ActualNickName'], time.strftime('%Y{}%m{}%d{}{}{} %H:%M:%S').format('年','月','日','星期',WEEKDAYS[time.strftime('%w')])), msg['FromUserName'])
        elif '笑话' in cont and ('讲' in cont or '说' in cont):
            itchat.send('@{}\u2005你等下哈，我编一个……'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], spider.get_joke()), msg['FromUserName'])
        elif '新闻' in cont:
            itchat.send('@{}\u2005好，稍等一下'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005\n{}'.format(msg['ActualNickName'], spider.get_news()), msg['FromUserName'])
        elif '热点' in cont:
            itchat.send('@{}\u2005好，稍等一下'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005\n{}'.format(msg['ActualNickName'], spider.get_hot()), msg['FromUserName'])
        elif '说' in cont:
            words = '说'.join(cont.split('说')[1:]).strip()
            print(words)
            itchat.send('{}'.format(words), msg['FromUserName'])
        elif cont[:3] == '求数列':
            inp = cont[3:]
            if cont[3] in (' ','：',':'):
                inp = cont[4:]
            inp = inp.replace('，',',')
            try:
                seq = pymath.Sequence(inp)
                itchat.send('@{}\u2005公式是：An = {}'.format(msg['ActualNickName'], seq.generateFormula()), msg['FromUserName'])
            except Exception:
                itchat.send('@{}\u2005这个……你的数列好像出了点毛病？'.format(msg['ActualNickName']), msg['FromUserName'])
        elif cont[:4] == '贴吧图片':
            itchat.send('@{}\u2005稍等一下哈，我收集一下图片！'.format(msg['ActualNickName']), msg['FromUserName'])
            inp = cont[5:]
            urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
            if urlRe.findall(inp):
                if inp[:7] != 'http://':
                    inp = 'http://' + inp
                if '?' in inp:
                    inp = urlRe.search(inp).group()
                dirName, imgCount = spider.get_tieba_img(inp+'/')
                for i in range(imgCount):
                    itchat.send('@img@{}'.format('{}.jpg'.format(i)), msg['FromUserName'])
            itchat.send('@{}\u2005你只能爬取第一页楼主发的图片，要爬取更多图片，请联系@熏鱼饺子\u2005进行账户升级'.format(msg['ActualNickName']), msg['FromUserName'])
            os.chdir('..')
            shutil.rmtree(dirName)
        else:
            for i in ('嘿','你好','hi','hello','hey','bonjour','bonsoir'):
                if i in cont.lower():
                    itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], choice(GREETINGS)), msg['FromUserName'])
                    break
            else:
                itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], choice(DONT_UNDERSTAND)), msg['FromUserName'])

itchat.auto_login(True, enableCmdQR=2)
itchat.run(debug=True)
<commit_msg>Update Dumpling-Bot to Alpha 1.0.11<commit_after>#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

import itchat
from itchat.content import *
from random import choice
import time
import os
import re
import shutil
import spider, pymath # 我的一些功能

GREETINGS = ('嘿','哟','你好啊','你好呀',':)')
DONT_UNDERSTAND = ('对不起我不明白诶……','抱歉我没听懂:(','不好意思没明白','Sorry没有听懂')
HELP = '我现在虽然聊天不太行，只能打打招呼，但是我可以报时，帮你搜集新闻和热点，甚至还可以讲笑话。我还会求数列的公式！'
WEEKDAYS = {'0':'天','1':'一','2':'二','3':'三','4':'四','5':'五','6':'六'}

@itchat.msg_register(['Picture', 'Recording', 'Attachment', 'Video'])
def download_files(msg):
    # msg['Text'](msg['FileName'])
    itchat.send('@%s@%s'%('img' if msg['Type'] == 'Picture' else 'fil', 'test.png'), msg['FromUserName'])
    return '%s received'%msg['Type']

@itchat.msg_register(TEXT, isGroupChat=True)
def text_reply(msg):
    if msg.get('isAt') or '@饺子机器人\u2005' in msg['Content']: # 如果@了我
        cont = msg['Content'].replace('@熏鱼饺子\u2005','').replace('@饺子机器人\u2005','')
        print(cont)
        if '帮助' in cont or ('你' in cont and '除了' in cont and '还' in cont) or ('你' in cont and ('会' in cont or '能' in cont) and ('干' in cont or '做' in cont)):
            itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], HELP), msg['FromUserName'])
        elif '几点' in cont or ('现在' in cont and '时间' in cont):
            itchat.send('@{}\u2005现在是{}'.format(msg['ActualNickName'], time.strftime('%Y{}%m{}%d{}{}{} %H:%M:%S').format('年','月','日','星期',WEEKDAYS[time.strftime('%w')])), msg['FromUserName'])
        elif '笑话' in cont and ('讲' in cont or '说' in cont):
            itchat.send('@{}\u2005你等下哈，我编一个……'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], spider.get_joke()), msg['FromUserName'])
        elif '新闻' in cont:
            itchat.send('@{}\u2005好，稍等一下'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005\n{}'.format(msg['ActualNickName'], spider.get_news()), msg['FromUserName'])
        elif '热点' in cont:
            itchat.send('@{}\u2005好，稍等一下'.format(msg['ActualNickName']), msg['FromUserName'])
            itchat.send('@{}\u2005\n{}'.format(msg['ActualNickName'], spider.get_hot()), msg['FromUserName'])
        elif '说' in cont:
            words = '说'.join(cont.split('说')[1:]).strip()
            print(words)
            itchat.send('{}'.format(words), msg['FromUserName'])
        elif cont[:3] == '求数列':
            inp = cont[3:]
            if cont[3] in (' ','：',':'):
                inp = cont[4:]
            inp = inp.replace('，',',')
            try:
                seq = pymath.Sequence(inp)
                itchat.send('@{}\u2005公式是：An = {}'.format(msg['ActualNickName'], seq.generateFormula()), msg['FromUserName'])
            except Exception:
                itchat.send('@{}\u2005这个……你的数列好像出了点毛病？'.format(msg['ActualNickName']), msg['FromUserName'])
        elif cont[:4] == '贴吧图片':
            itchat.send('@{}\u2005稍等一下哈，我收集一下图片！'.format(msg['ActualNickName']), msg['FromUserName'])
            inp = cont[5:]
            urlRe = re.compile(r'(http://)?tieba\.baidu\.com/p/[0-9]+')
            if urlRe.findall(inp):
                if inp[:7] != 'http://':
                    inp = 'http://' + inp
                if '?' in inp:
                    inp = urlRe.search(inp).group()
##                dirName, imgCount = spider.get_tieba_img(inp+'/')
##                for i in range(imgCount):
##                    itchat.send('@img@{}'.format('{}.jpg'.format(i)), msg['FromUserName'])
                title = spider.get_tieba_img(inp+'/')
                itchat.send('@fil@{}.zip'.format(title), msg['FromUserName'])
            itchat.send('@{}\u2005你只能爬取第一页楼主发的图片，要爬取更多图片，请联系@熏鱼饺子\u2005进行账户升级'.format(msg['ActualNickName']), msg['FromUserName'])
            os.remove(title+'.zip')
            shutil.rmtree(title)
        else:
            for i in ('嘿','你好','hi','hello','hey','bonjour','bonsoir'):
                if i in cont.lower():
                    itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], choice(GREETINGS)), msg['FromUserName'])
                    break
            else:
                itchat.send('@{}\u2005{}'.format(msg['ActualNickName'], choice(DONT_UNDERSTAND)), msg['FromUserName'])

itchat.auto_login(True, enableCmdQR=2)
itchat.run(debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import optparse
import inspect

#import pybossa.model as model
from pybossa.core import db, create_app
from pybossa.model.app import App
from pybossa.model.user import User
from pybossa.model.category import Category

from alembic.config import Config
from alembic import command
from html2text import html2text
from sqlalchemy.sql import text

app = create_app()

def setup_alembic_config():
    if "DATABASE_URL" not in os.environ:
        alembic_cfg = Config("alembic.ini")
    else:
        dynamic_filename = "alembic-heroku.ini"
        with file("alembic.ini.template") as f:
            with file(dynamic_filename, "w") as conf:
                for line in f.readlines():
                    if line.startswith("sqlalchemy.url"):
                        conf.write("sqlalchemy.url = %s\n" %
                                   os.environ['DATABASE_URL'])
                    else:
                        conf.write(line)
        alembic_cfg = Config(dynamic_filename)

    command.stamp(alembic_cfg, "head")

def db_create():
    '''Create the db'''
    with app.app_context():
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()
        # finally, add a minimum set of categories: Volunteer Thinking, Volunteer Sensing, Published and Draft
        categories = []
        categories.append(Category(name="Thinking",
                          short_name='thinking',
                          description='Volunteer Thinking projects'))
        categories.append(Category(name="Volunteer Sensing",
                          short_name='sensing',
                          description='Volunteer Sensing projects'))
        db.session.add_all(categories)
        db.session.commit()

def db_rebuild():
    '''Rebuild the db'''
    with app.app_context():
        db.drop_all()
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()

def fixtures():
    '''Create some fixtures!'''
    with app.app_context():
        user = User(
            name=u'tester',
            email_addr=u'tester@tester.org',
            api_key='tester'
            )
        user.set_password(u'tester')
        db.session.add(user)
        db.session.commit()

def markdown_db_migrate():
    '''Perform a migration of the app long descriptions from HTML to
    Markdown for existing database records'''
    with app.app_context():
        query = 'SELECT id, long_description FROM "app";'
        query_result = db.engine.execute(query)
        old_descriptions = query_result.fetchall()
        for old_desc in old_descriptions:
            if old_desc.long_description:
                new_description = html2text(old_desc.long_description)
                query = text('''
                           UPDATE app SET long_description=:long_description
                           WHERE id=:id''')
                db.engine.execute(query, long_description = new_description, id = old_desc.id)

def bootstrap_avatars():
    """Download current links from user avatar and projects to real images hosted in the
    PyBossa server."""
    import requests
    import os
    import time
    from urlparse import urlparse
    from PIL import Image

    def get_gravatar_url(email, size):
        # import code for encoding urls and generating md5 hashes
        import urllib, hashlib

        # construct the url
        gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
        gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
        return gravatar_url

    with app.app_context():
        if app.config['UPLOAD_METHOD'] == 'local':
            users = User.query.order_by('id').all()
            print "Downloading avatars for %s users" % len(users)
            for u in users[0:10]:
                print "Downloading avatar for %s ..." % u.name
                container = "user_%s" % u.id
                path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                try:
                    print get_gravatar_url(u.email_addr, 100)
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        if not os.path.isdir(path):
                            os.makedirs(path)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(path, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    raise
                    print "No gravatar, this user will use the placehoder."

            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps[0:1]:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                if not os.path.isdir(path):
                                    os.makedirs(path)
                                with open(os.path.join(path, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."
        if app.config['UPLOAD_METHOD'] == 'rackspace':
            import pyrax
            import tempfile
            pyrax.set_setting("identity_type", "rackspace")
            pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
                                  api_key=app.config['RACKSPACE_API_KEY'],
                                  region=app.config['RACKSPACE_REGION'])

            cf = pyrax.cloudfiles
            users = User.query.all()
            print "Downloading avatars for %s users" % len(users)
            dirpath = tempfile.mkdtemp()
            for u in users:
                try:
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        print "Downloading avatar for %s ..." % u.name
                        container = "user_%s" % u.id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(dirpath, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                       filename))
                        cf.upload_file(container,
                                       os.path.join(dirpath, filename),
                                       obj_name=filename,
                                       etag=chksum)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    print "No gravatar, this user will use the placehoder."

            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)

                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                with open(os.path.join(dirpath, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                               filename))
                                cf.upload_file(container,
                                               os.path.join(dirpath, filename),
                                               obj_name=filename,
                                               etag=chksum)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."

def resize_avatars():
    """Resize avatars to 512px."""
    if app.config['UPLOAD_METHOD'] == 'rackspace':
        import pyrax
        import tempfile
        import requests
        from PIL import Image
        import time
        pyrax.set_setting("identity_type", "rackspace")
        pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
                              api_key=app.config['RACKSPACE_API_KEY'],
                              region=app.config['RACKSPACE_REGION'])

        cf = pyrax.cloudfiles
        #users = User.query.all()
        #users = [User.query.get(5305)]
        #users = [User.query.get(5662)]
        users = [User.query.get(3)]
        print "Downloading avatars for %s users" % len(users)
        dirpath = tempfile.mkdtemp()
        for u in users:
            try:
                cont = cf.get_container(u.info['container'])
                avatar_url = "%s/%s" % (cont.cdn_uri, u.info['avatar'])
                r = requests.get(avatar_url, stream=True)
                if r.status_code == 200:
                    print "Downloading avatar for %s ..." % u.name
                    #container = "user_%s" % u.id
                    #try:
                    #    cf.get_container(container)
                    #except pyrax.exceptions.NoSuchContainer:
                    #    cf.create_container(container)
                    #    cf.make_container_public(container)
                    prefix = time.time()
                    filename = "%s_avatar.png" % prefix
                    with open(os.path.join(dirpath, filename), 'wb') as f:
                        for chunk in r.iter_content(1024):
                            f.write(chunk)
                    # Resize image
                    im = Image.open(os.path.join(dirpath, filename))
                    size = 512, 512
                    tmp = im.resize(size, Image.ANTIALIAS)
                    scale_down_img = tmp.convert('P', colors=255, palette=Image.ADAPTIVE)
                    scale_down_img.save(os.path.join(dirpath, filename), format='png')

                    print "New scaled down image created!"
                    print "%s" % (os.path.join(dirpath, filename))
                    print "---"

                    chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                   filename))
                    cf.upload_file(container,
                                   os.path.join(dirpath, filename),
                                   obj_name=filename,
                                   etag=chksum)
                    # delete old avatar
                    obj = cont.get_object(u.info['avatar'])
                    obj.delete()
                    # Update new values
                    u.info['avatar'] = filename
                    u.info['container'] = container
                    db.session.commit()
                    print "Done!"
                else:
                    print "No Avatar found."
            except:
                raise
                print "No Avatar, this user will use the placehoder."

        #apps = App.query.all()
        #print "Downloading avatars for %s projects" % len(apps)
        #for a in apps:
        #    if a.info.get('thumbnail') and not a.info.get('container'):
        #        print "Working on project: %s ..." % a.short_name
        #        print "Saving avatar: %s ..." % a.info.get('thumbnail')
        #        url = urlparse(a.info.get('thumbnail'))
        #        if url.scheme and url.netloc:
        #            container = "user_%s" % a.owner_id
        #            try:
        #                cf.get_container(container)
        #            except pyrax.exceptions.NoSuchContainer:
        #                cf.create_container(container)
        #                cf.make_container_public(container)

        #            try:
        #                r = requests.get(a.info.get('thumbnail'), stream=True)
        #                if r.status_code == 200:
        #                    prefix = time.time()
        #                    filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
        #                    with open(os.path.join(dirpath, filename), 'wb') as f:
        #                        for chunk in r.iter_content(1024):
        #                            f.write(chunk)
        #                    chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
        #                                                                   filename))
        #                    cf.upload_file(container,
        #                                   os.path.join(dirpath, filename),
        #                                   obj_name=filename,
        #                                   etag=chksum)
        #                    a.info['thumbnail'] = filename
        #                    a.info['container'] = container
        #                    db.session.commit()
        #                    print "Done!"
        #            except:
        #                print "Something failed, this project will use the placehoder."

## ==================================================
## Misc stuff for setting up a command line interface

def _module_functions(functions):
    local_functions = dict(functions)
    for k,v in local_functions.items():
        if not inspect.isfunction(v) or k.startswith('_'):
            del local_functions[k]
    return local_functions

def _main(functions_or_object):
    isobject = inspect.isclass(functions_or_object)
    if isobject:
        _methods = _object_methods(functions_or_object)
    else:
        _methods = _module_functions(functions_or_object)

    usage = '''%prog {action}

Actions:
    '''
    usage += '\n    '.join(
        [ '%s: %s' % (name, m.__doc__.split('\n')[0] if m.__doc__ else '') for (name,m)
        in sorted(_methods.items()) ])
    parser = optparse.OptionParser(usage)
    # Optional: for a config file
    # parser.add_option('-c', '--config', dest='config',
    #         help='Config file to use.')
    options, args = parser.parse_args()

    if not args or not args[0] in _methods:
        parser.print_help()
        sys.exit(1)

    method = args[0]
    if isobject:
        getattr(functions_or_object(), method)(*args[1:])
    else:
        _methods[method](*args[1:])

__all__ = [ '_main' ]

if __name__ == '__main__':
    _main(locals())

<commit_msg>Fix var name<commit_after>#!/usr/bin/env python
import os
import sys
import optparse
import inspect

#import pybossa.model as model
from pybossa.core import db, create_app
from pybossa.model.app import App
from pybossa.model.user import User
from pybossa.model.category import Category

from alembic.config import Config
from alembic import command
from html2text import html2text
from sqlalchemy.sql import text

app = create_app()

def setup_alembic_config():
    if "DATABASE_URL" not in os.environ:
        alembic_cfg = Config("alembic.ini")
    else:
        dynamic_filename = "alembic-heroku.ini"
        with file("alembic.ini.template") as f:
            with file(dynamic_filename, "w") as conf:
                for line in f.readlines():
                    if line.startswith("sqlalchemy.url"):
                        conf.write("sqlalchemy.url = %s\n" %
                                   os.environ['DATABASE_URL'])
                    else:
                        conf.write(line)
        alembic_cfg = Config(dynamic_filename)

    command.stamp(alembic_cfg, "head")

def db_create():
    '''Create the db'''
    with app.app_context():
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()
        # finally, add a minimum set of categories: Volunteer Thinking, Volunteer Sensing, Published and Draft
        categories = []
        categories.append(Category(name="Thinking",
                          short_name='thinking',
                          description='Volunteer Thinking projects'))
        categories.append(Category(name="Volunteer Sensing",
                          short_name='sensing',
                          description='Volunteer Sensing projects'))
        db.session.add_all(categories)
        db.session.commit()

def db_rebuild():
    '''Rebuild the db'''
    with app.app_context():
        db.drop_all()
        db.create_all()
        # then, load the Alembic configuration and generate the
        # version table, "stamping" it with the most recent rev:
        setup_alembic_config()

def fixtures():
    '''Create some fixtures!'''
    with app.app_context():
        user = User(
            name=u'tester',
            email_addr=u'tester@tester.org',
            api_key='tester'
            )
        user.set_password(u'tester')
        db.session.add(user)
        db.session.commit()

def markdown_db_migrate():
    '''Perform a migration of the app long descriptions from HTML to
    Markdown for existing database records'''
    with app.app_context():
        query = 'SELECT id, long_description FROM "app";'
        query_result = db.engine.execute(query)
        old_descriptions = query_result.fetchall()
        for old_desc in old_descriptions:
            if old_desc.long_description:
                new_description = html2text(old_desc.long_description)
                query = text('''
                           UPDATE app SET long_description=:long_description
                           WHERE id=:id''')
                db.engine.execute(query, long_description = new_description, id = old_desc.id)

def bootstrap_avatars():
    """Download current links from user avatar and projects to real images hosted in the
    PyBossa server."""
    import requests
    import os
    import time
    from urlparse import urlparse
    from PIL import Image

    def get_gravatar_url(email, size):
        # import code for encoding urls and generating md5 hashes
        import urllib, hashlib

        # construct the url
        gravatar_url = "http://www.gravatar.com/avatar/" + hashlib.md5(email.lower()).hexdigest() + "?"
        gravatar_url += urllib.urlencode({'d':404, 's':str(size)})
        return gravatar_url

    with app.app_context():
        if app.config['UPLOAD_METHOD'] == 'local':
            users = User.query.order_by('id').all()
            print "Downloading avatars for %s users" % len(users)
            for u in users[0:10]:
                print "Downloading avatar for %s ..." % u.name
                container = "user_%s" % u.id
                path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                try:
                    print get_gravatar_url(u.email_addr, 100)
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        if not os.path.isdir(path):
                            os.makedirs(path)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(path, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    raise
                    print "No gravatar, this user will use the placehoder."

            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps[0:1]:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        path = os.path.join(app.config.get('UPLOAD_FOLDER'), container)
                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                if not os.path.isdir(path):
                                    os.makedirs(path)
                                with open(os.path.join(path, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."
        if app.config['UPLOAD_METHOD'] == 'rackspace':
            import pyrax
            import tempfile
            pyrax.set_setting("identity_type", "rackspace")
            pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
                                  api_key=app.config['RACKSPACE_API_KEY'],
                                  region=app.config['RACKSPACE_REGION'])

            cf = pyrax.cloudfiles
            users = User.query.all()
            print "Downloading avatars for %s users" % len(users)
            dirpath = tempfile.mkdtemp()
            for u in users:
                try:
                    r = requests.get(get_gravatar_url(u.email_addr, 100), stream=True)
                    if r.status_code == 200:
                        print "Downloading avatar for %s ..." % u.name
                        container = "user_%s" % u.id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)
                        prefix = time.time()
                        filename = "%s_avatar.png" % prefix
                        with open(os.path.join(dirpath, filename), 'wb') as f:
                            for chunk in r.iter_content(1024):
                                f.write(chunk)
                        chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                       filename))
                        cf.upload_file(container,
                                       os.path.join(dirpath, filename),
                                       obj_name=filename,
                                       etag=chksum)
                        u.info['avatar'] = filename
                        u.info['container'] = container
                        db.session.commit()
                        print "Done!"
                    else:
                        print "No Gravatar, this user will use the placeholder."
                except:
                    print "No gravatar, this user will use the placehoder."

            apps = App.query.all()
            print "Downloading avatars for %s projects" % len(apps)
            for a in apps:
                if a.info.get('thumbnail') and not a.info.get('container'):
                    print "Working on project: %s ..." % a.short_name
                    print "Saving avatar: %s ..." % a.info.get('thumbnail')
                    url = urlparse(a.info.get('thumbnail'))
                    if url.scheme and url.netloc:
                        container = "user_%s" % a.owner_id
                        try:
                            cf.get_container(container)
                        except pyrax.exceptions.NoSuchContainer:
                            cf.create_container(container)
                            cf.make_container_public(container)

                        try:
                            r = requests.get(a.info.get('thumbnail'), stream=True)
                            if r.status_code == 200:
                                prefix = time.time()
                                filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
                                with open(os.path.join(dirpath, filename), 'wb') as f:
                                    for chunk in r.iter_content(1024):
                                        f.write(chunk)
                                chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                               filename))
                                cf.upload_file(container,
                                               os.path.join(dirpath, filename),
                                               obj_name=filename,
                                               etag=chksum)
                                a.info['thumbnail'] = filename
                                a.info['container'] = container
                                db.session.commit()
                                print "Done!"
                        except:
                            print "Something failed, this project will use the placehoder."

def resize_avatars():
    """Resize avatars to 512px."""
    if app.config['UPLOAD_METHOD'] == 'rackspace':
        import pyrax
        import tempfile
        import requests
        from PIL import Image
        import time
        pyrax.set_setting("identity_type", "rackspace")
        pyrax.set_credentials(username=app.config['RACKSPACE_USERNAME'],
                              api_key=app.config['RACKSPACE_API_KEY'],
                              region=app.config['RACKSPACE_REGION'])

        cf = pyrax.cloudfiles
        #users = User.query.all()
        #users = [User.query.get(5305)]
        #users = [User.query.get(5662)]
        users = [User.query.get(3)]
        print "Downloading avatars for %s users" % len(users)
        dirpath = tempfile.mkdtemp()
        for u in users:
            try:
                cont = cf.get_container(u.info['container'])
                avatar_url = "%s/%s" % (cont.cdn_uri, u.info['avatar'])
                r = requests.get(avatar_url, stream=True)
                if r.status_code == 200:
                    print "Downloading avatar for %s ..." % u.name
                    #container = "user_%s" % u.id
                    #try:
                    #    cf.get_container(container)
                    #except pyrax.exceptions.NoSuchContainer:
                    #    cf.create_container(container)
                    #    cf.make_container_public(container)
                    prefix = time.time()
                    filename = "%s_avatar.png" % prefix
                    with open(os.path.join(dirpath, filename), 'wb') as f:
                        for chunk in r.iter_content(1024):
                            f.write(chunk)
                    # Resize image
                    im = Image.open(os.path.join(dirpath, filename))
                    size = 512, 512
                    tmp = im.resize(size, Image.ANTIALIAS)
                    scale_down_img = tmp.convert('P', colors=255, palette=Image.ADAPTIVE)
                    scale_down_img.save(os.path.join(dirpath, filename), format='png')

                    print "New scaled down image created!"
                    print "%s" % (os.path.join(dirpath, filename))
                    print "---"

                    chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
                                                                   filename))
                    cf.upload_file(cont,
                                   os.path.join(dirpath, filename),
                                   obj_name=filename,
                                   etag=chksum)
                    # delete old avatar
                    obj = cont.get_object(u.info['avatar'])
                    obj.delete()
                    # Update new values
                    u.info['avatar'] = filename
                    u.info['container'] = container
                    db.session.commit()
                    print "Done!"
                else:
                    print "No Avatar found."
            except:
                raise
                print "No Avatar, this user will use the placehoder."

        #apps = App.query.all()
        #print "Downloading avatars for %s projects" % len(apps)
        #for a in apps:
        #    if a.info.get('thumbnail') and not a.info.get('container'):
        #        print "Working on project: %s ..." % a.short_name
        #        print "Saving avatar: %s ..." % a.info.get('thumbnail')
        #        url = urlparse(a.info.get('thumbnail'))
        #        if url.scheme and url.netloc:
        #            container = "user_%s" % a.owner_id
        #            try:
        #                cf.get_container(container)
        #            except pyrax.exceptions.NoSuchContainer:
        #                cf.create_container(container)
        #                cf.make_container_public(container)

        #            try:
        #                r = requests.get(a.info.get('thumbnail'), stream=True)
        #                if r.status_code == 200:
        #                    prefix = time.time()
        #                    filename = "app_%s_thumbnail_%i.png" % (a.id, prefix)
        #                    with open(os.path.join(dirpath, filename), 'wb') as f:
        #                        for chunk in r.iter_content(1024):
        #                            f.write(chunk)
        #                    chksum = pyrax.utils.get_checksum(os.path.join(dirpath,
        #                                                                   filename))
        #                    cf.upload_file(container,
        #                                   os.path.join(dirpath, filename),
        #                                   obj_name=filename,
        #                                   etag=chksum)
        #                    a.info['thumbnail'] = filename
        #                    a.info['container'] = container
        #                    db.session.commit()
        #                    print "Done!"
        #            except:
        #                print "Something failed, this project will use the placehoder."

## ==================================================
## Misc stuff for setting up a command line interface

def _module_functions(functions):
    local_functions = dict(functions)
    for k,v in local_functions.items():
        if not inspect.isfunction(v) or k.startswith('_'):
            del local_functions[k]
    return local_functions

def _main(functions_or_object):
    isobject = inspect.isclass(functions_or_object)
    if isobject:
        _methods = _object_methods(functions_or_object)
    else:
        _methods = _module_functions(functions_or_object)

    usage = '''%prog {action}

Actions:
    '''
    usage += '\n    '.join(
        [ '%s: %s' % (name, m.__doc__.split('\n')[0] if m.__doc__ else '') for (name,m)
        in sorted(_methods.items()) ])
    parser = optparse.OptionParser(usage)
    # Optional: for a config file
    # parser.add_option('-c', '--config', dest='config',
    #         help='Config file to use.')
    options, args = parser.parse_args()

    if not args or not args[0] in _methods:
        parser.print_help()
        sys.exit(1)

    method = args[0]
    if isobject:
        getattr(functions_or_object(), method)(*args[1:])
    else:
        _methods[method](*args[1:])

__all__ = [ '_main' ]

if __name__ == '__main__':
    _main(locals())

<|endoftext|>


<commit_before>import os
import atexit
import logging
import socket
import traceback
import sys

from .preflight import preflight_check
from .log import configure_logging
from .notifier import notify
from .constants import SOCKET_PATH, SOCKET_TERMINATOR
from .commands import process_command

def _clean_up_existing_socket(socket_path):
    try:
        os.unlink(socket_path)
    except OSError:
        if os.path.exists(socket_path):
            raise

def _listen_on_socket(socket_path):
    _clean_up_existing_socket(socket_path)

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.bind(socket_path)
    sock.listen(1)
    logging.info('Listening on socket at {}'.format(socket_path))

    notify('Dusty is listening for commands')
    atexit.register(notify, 'Dusty daemon has terminated')

    while True:
        try:
            connection, client_address = sock.accept()
            try:
                while True:
                    data = connection.recv(1024)
                    if not data:
                        break
                    logging.info('Received command: {}'.format(data))
                    try:
                        for line in process_command(data):
                            connection.sendall('{}\n'.format(line.encode('utf-8')))
                    except Exception as e:
                        exc_type, exc_value, exc_traceback = sys.exc_info()
                        traceback.print_exception(exc_type, exc_value, exc_traceback,
                              limit=100, file=sys.stdout)
                        connection.sendall('ERROR: {}\n'.format(e.message).encode('utf-8'))
                    connection.sendall(SOCKET_TERMINATOR)
            finally:
                connection.close()
        except KeyboardInterrupt:
            break
        except:
            logging.exception('Exception on socket listen')

def main():
    notify('Dusty initializing...')
    configure_logging()
    preflight_check()
    _listen_on_socket(SOCKET_PATH)

if __name__ == '__main__':
    main()
<commit_msg>remove debug logging<commit_after>import os
import atexit
import logging
import socket

from .preflight import preflight_check
from .log import configure_logging
from .notifier import notify
from .constants import SOCKET_PATH, SOCKET_TERMINATOR
from .commands import process_command

def _clean_up_existing_socket(socket_path):
    try:
        os.unlink(socket_path)
    except OSError:
        if os.path.exists(socket_path):
            raise

def _listen_on_socket(socket_path):
    _clean_up_existing_socket(socket_path)

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.bind(socket_path)
    sock.listen(1)
    logging.info('Listening on socket at {}'.format(socket_path))

    notify('Dusty is listening for commands')
    atexit.register(notify, 'Dusty daemon has terminated')

    while True:
        try:
            connection, client_address = sock.accept()
            try:
                while True:
                    data = connection.recv(1024)
                    if not data:
                        break
                    logging.info('Received command: {}'.format(data))
                    try:
                        for line in process_command(data):
                            connection.sendall('{}\n'.format(line.encode('utf-8')))
                    except Exception as e:
                        connection.sendall('ERROR: {}\n'.format(e.message).encode('utf-8'))
                    connection.sendall(SOCKET_TERMINATOR)
            finally:
                connection.close()
        except KeyboardInterrupt:
            break
        except:
            logging.exception('Exception on socket listen')

def main():
    notify('Dusty initializing...')
    configure_logging()
    preflight_check()
    _listen_on_socket(SOCKET_PATH)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import sys
import math
import argparse
import config
import shutil
from turkic.cli import handler, importparser, Command, LoadCommand
from turkic import database
from vision import ffmpeg
import vision.visualize
import vision.track.interpolation
import turkic.models
from models import *

@handler("Decompresses an entire video into frames")
class extract(Command):
    def setup(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("video")
        parser.add_argument("output")
        parser.add_argument("--width", default=720, type=int)
        parser.add_argument("--height", default=480, type=int)
        parser.add_argument("--no-cleanup", action="store_true", default=False)
        return parser

    def __call__(self, args):
        try:
            os.makedirs(args.output)
        except:
            pass
        sequence = ffmpeg.extract(args.video)
        try:
            for frame, image in enumerate(sequence):
                if frame % 100 == 0:
                    print "Decoding frames {0} to {1}".format(frame, frame + 100)
                image.thumbnail((args.width, args.height), Image.BILINEAR)
                path = Video.getframepath(frame, args.output)
                try:
                    image.save(path)
                except IOError:
                    os.makedirs(os.path.dirname(path))
                    image.save(path)
        except:
            if not args.no_cleanup:
                print "Aborted. Cleaning up..."
                shutil.rmtree(args.output)
            raise

@handler("Imports a set of video frames")
class load(LoadCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [importparser])
        parser.add_argument("slug")
        parser.add_argument("location")
        parser.add_argument("labels", nargs="+")
        parser.add_argument("--length", type=int, default = 300)
        parser.add_argument("--overlap", type=int, default = 20)
        return parser

    def title(self, args):
        return "Video annotation"

    def description(self, args):
        return "Draw boxes around objects moving around in a video."

    def cost(self, args):
        return 1.00

    def duration(self, args):
        return 7200 * 3

    def keywords(self, args):
        return "video, annotation, computer, vision"

    def __call__(self, args, group):
        print "Checking integrity..."

        # read first frame to get sizes
        path = Video.getframepath(0, args.location)
        try:
            im = Image.open(path)
        except IOError:
            print "Cannot read {0}".format(path)
            return
        width, height = im.size

        print "Searching for last frame..."

        # search for last frame
        toplevel = max(int(x)
            for x in os.listdir(args.location))
        secondlevel = max(int(x)
            for x in os.listdir("{0}/{1}".format(args.location, toplevel)))
        maxframes = max(int(os.path.splitext(x)[0])
            for x in os.listdir("{0}/{1}/{2}"
            .format(args.location, toplevel, secondlevel)))

        print "Found {0} frames.".format(maxframes)

        # can we read the last frame?
        path = Video.getframepath(maxframes, args.location)
        try:
            im = Image.open(path)
        except IOError:
            print "Cannot read {0}".format(path)
            return

        # check last frame sizes
        if im.size[0] != width and im.size[1] != height:
            print "First frame dimensions differs from last frame"
            return

        session = database.connect()
        try:
            # create video
            video = Video(slug = args.slug,
                          location = args.location,
                          width = width,
                          height = height,
                          totalframes = maxframes)
            session.add(video)

            print "Binding labels..."

            # create labels
            for labeltext in args.labels:
                label = Label(text = labeltext)
                session.add(label)
                video.labels.append(label)

            print "Creating symbolic link..."
            symlink = "public/frames/{0}".format(video.slug)
            try:
                os.symlink(video.location, symlink)
            except:
                print "Cannot create symbolic link!"

            print "Creating segments..."

            # create shots and jobs
            for start in range(0, video.totalframes, args.length):
                stop = min(start + args.length + args.overlap + 1, video.totalframes)
                segment = Segment(start = start, stop = stop, video = video)
                job = Job(segment = segment)
                session.add(segment)
                session.add(job)
                session.commit()

                hit = turkic.models.HIT(group = group,
                                        page = "?id={0}".format(job.id))
                job.hit = hit
                session.add(hit)
                session.add(job)

            session.add(group)
            session.commit()
            print "Video imported and ready for publication."
        finally:
            session.close()

class DumpCommand(Command):
    parent = argparse.ArgumentParser(add_help=False)
    parent.add_argument("slug")
    parent.add_argument("--interpolate", "-i", action="store_true", default=False)
    parent.add_argument("--merge", "-m", action="store_true", default=False)

    class Tracklet(object):
        def __init__(self, label, boxes):
            self.label = label
            self.boxes = sorted(boxes, key = lambda x: x.frame)

    def getdata(self, args):
        response = []
        session = database.connect()
        try:
            video = session.query(Video).filter(Video.slug == args.slug).one()
            for segment in video.segments:
                for job in segment.jobs:
                    for path in job.paths:
                        tracklet = DumpCommand.Tracklet(path.label.text,
                                                        path.getboxes())
                        response.append(tracklet)

            if args.interpolate:
                interpolated = []
                for track in response:
                    path = vision.track.interpolation.LinearFill(track.boxes)
                    tracklet = DumpCommand.Tracklet(track.label, path)
                    interpolated.append(tracklet)
                response = interpolated

            return video, response
        finally:
            session.close()

@handler("Highlights a video sequence")
class visualize(DumpCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [self.parent])
        parser.add_argument("output")
        return parser

    def __call__(self, args):
        print "Fetching data..."
        video, data = self.getdata(args)

        print "Processing {0} tracks...".format(len(data))
        paths = [x.boxes for x in data]

        print "Highlighting frames..."
        it = vision.visualize.highlight_paths(video, paths)
        vision.visualize.save(it, lambda x: "{0}/{1}.jpg".format(args.output, x))

@handler("Dumps the tracking data")
class dump(DumpCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [self.parent])
        parser.add_argument("--output", "-o")
        parser.add_argument("--xml", "-x", action="store_true", default=False)
        parser.add_argument("--json", "-j", action="store_true", default=False)
        return parser

    def __call__(self, args):
        file = sys.stdout
        if args.output:
            file = open(args.output, 'w')

        video, data = self.getdata(args)

        if args.xml:
            self.dumpxml(file, data)
        elif args.json:
            self.dumpjson(file, data)
        else:
            self.dumptext(file, data)

        if args.output:
            file.close()

    def dumpxml(self, file, data):
        file.write("<annotations count=\"{0}\">\n".format(len(data)))
        for id, track in enumerate(data):
            file.write("\t<track id=\"{0}\" label=\"{1}\">\n".format(id, track.label))
            for box in track.boxes:
                file.write("\t\t<box frame=\"{0}\"".format(box.frame))
                file.write(" xtl=\"{0}\"".format(box.xtl))
                file.write(" ytl=\"{0}\"".format(box.ytl))
                file.write(" xbr=\"{0}\"".format(box.xbr))
                file.write(" ybr=\"{0}\"".format(box.ybr))
                file.write(" outside=\"{0}\"".format(box.lost))
                file.write(" occluded=\"{0}\" />\n".format(box.occluded))
            file.write("\t</track>\n")
        file.write("</annotations>\n")

    def dumpjson(self, file, data):
        file.write("annotations: {\n")
        for id, track in enumerate(data):
            file.write("\t{0}: {{\n".format(id))
            file.write("\t\tlabel: \"{0}\",\n".format(track.label))
            file.write("\t\tboxes: {\n")
            for box in track.boxes:
                file.write("\t\t\t{0}: {{\n".format(box.frame))
                file.write("\t\t\t\txtl: {0},\n".format(box.xtl))
                file.write("\t\t\t\tytl: {0},\n".format(box.ytl))
                file.write("\t\t\t\txbr: {0},\n".format(box.xbr))
                file.write("\t\t\t\tybr: {0},\n".format(box.ybr))
                file.write("\t\t\t\toutside: {0},\n".format(box.lost))
                file.write("\t\t\t\toccluded: {0}\n".format(box.occluded))
                file.write("\t\t\t},\n")
            file.write("\t\t},\n")
            file.write("\t},\n")
        file.write("}\n");

    def dumptext(self, file, data):
        for id, track in enumerate(data):
            for box in track.boxes:
                file.write(str(id))
                file.write(" ")
                file.write(str(box.xtl))
                file.write(" ")
                file.write(str(box.ytl))
                file.write(" ")
                file.write(str(box.xbr))
                file.write(" ")
                file.write(str(box.ybr))
                file.write(" ")
                file.write(str(box.frame))
                file.write(" ")
                file.write(str(box.lost))
                file.write(" ")
                file.write(str(box.occluded))
                file.write(" \"")
                file.write(track.label)
                file.write("\"\n")
<commit_msg>allow saving without resize<commit_after>import os
import sys
import math
import argparse
import config
import shutil
from turkic.cli import handler, importparser, Command, LoadCommand
from turkic import database
from vision import ffmpeg
import vision.visualize
import vision.track.interpolation
import turkic.models
from models import *

@handler("Decompresses an entire video into frames")
class extract(Command):
    def setup(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("video")
        parser.add_argument("output")
        parser.add_argument("--width", default=720, type=int)
        parser.add_argument("--height", default=480, type=int)
        parser.add_argument("--no-resize", action="store_true", default = False)
        parser.add_argument("--no-cleanup", action="store_true", default=False)
        return parser

    def __call__(self, args):
        try:
            os.makedirs(args.output)
        except:
            pass
        sequence = ffmpeg.extract(args.video)
        try:
            for frame, image in enumerate(sequence):
                if frame % 100 == 0:
                    print "Decoding frames {0} to {1}".format(frame, frame + 100)
                if not args.no_resize:
                    image.thumbnail((args.width, args.height), Image.BILINEAR)
                path = Video.getframepath(frame, args.output)
                try:
                    image.save(path)
                except IOError:
                    os.makedirs(os.path.dirname(path))
                    image.save(path)
        except:
            if not args.no_cleanup:
                print "Aborted. Cleaning up..."
                shutil.rmtree(args.output)
            raise

@handler("Imports a set of video frames")
class load(LoadCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [importparser])
        parser.add_argument("slug")
        parser.add_argument("location")
        parser.add_argument("labels", nargs="+")
        parser.add_argument("--length", type=int, default = 300)
        parser.add_argument("--overlap", type=int, default = 20)
        return parser

    def title(self, args):
        return "Video annotation"

    def description(self, args):
        return "Draw boxes around objects moving around in a video."

    def cost(self, args):
        return 1.00

    def duration(self, args):
        return 7200 * 3

    def keywords(self, args):
        return "video, annotation, computer, vision"

    def __call__(self, args, group):
        print "Checking integrity..."

        # read first frame to get sizes
        path = Video.getframepath(0, args.location)
        try:
            im = Image.open(path)
        except IOError:
            print "Cannot read {0}".format(path)
            return
        width, height = im.size

        print "Searching for last frame..."

        # search for last frame
        toplevel = max(int(x)
            for x in os.listdir(args.location))
        secondlevel = max(int(x)
            for x in os.listdir("{0}/{1}".format(args.location, toplevel)))
        maxframes = max(int(os.path.splitext(x)[0])
            for x in os.listdir("{0}/{1}/{2}"
            .format(args.location, toplevel, secondlevel)))

        print "Found {0} frames.".format(maxframes)

        # can we read the last frame?
        path = Video.getframepath(maxframes, args.location)
        try:
            im = Image.open(path)
        except IOError:
            print "Cannot read {0}".format(path)
            return

        # check last frame sizes
        if im.size[0] != width and im.size[1] != height:
            print "First frame dimensions differs from last frame"
            return

        session = database.connect()
        try:
            # create video
            video = Video(slug = args.slug,
                          location = args.location,
                          width = width,
                          height = height,
                          totalframes = maxframes)
            session.add(video)

            print "Binding labels..."

            # create labels
            for labeltext in args.labels:
                label = Label(text = labeltext)
                session.add(label)
                video.labels.append(label)

            print "Creating symbolic link..."
            symlink = "public/frames/{0}".format(video.slug)
            try:
                os.symlink(video.location, symlink)
            except:
                print "Cannot create symbolic link!"

            print "Creating segments..."

            # create shots and jobs
            for start in range(0, video.totalframes, args.length):
                stop = min(start + args.length + args.overlap + 1, video.totalframes)
                segment = Segment(start = start, stop = stop, video = video)
                job = Job(segment = segment)
                session.add(segment)
                session.add(job)
                session.commit()

                hit = turkic.models.HIT(group = group,
                                        page = "?id={0}".format(job.id))
                job.hit = hit
                session.add(hit)
                session.add(job)

            session.add(group)
            session.commit()
            print "Video imported and ready for publication."
        finally:
            session.close()

class DumpCommand(Command):
    parent = argparse.ArgumentParser(add_help=False)
    parent.add_argument("slug")
    parent.add_argument("--interpolate", "-i", action="store_true", default=False)
    parent.add_argument("--merge", "-m", action="store_true", default=False)

    class Tracklet(object):
        def __init__(self, label, boxes):
            self.label = label
            self.boxes = sorted(boxes, key = lambda x: x.frame)

    def getdata(self, args):
        response = []
        session = database.connect()
        try:
            video = session.query(Video).filter(Video.slug == args.slug).one()
            for segment in video.segments:
                for job in segment.jobs:
                    for path in job.paths:
                        tracklet = DumpCommand.Tracklet(path.label.text,
                                                        path.getboxes())
                        response.append(tracklet)

            if args.interpolate:
                interpolated = []
                for track in response:
                    path = vision.track.interpolation.LinearFill(track.boxes)
                    tracklet = DumpCommand.Tracklet(track.label, path)
                    interpolated.append(tracklet)
                response = interpolated

            return video, response
        finally:
            session.close()

@handler("Highlights a video sequence")
class visualize(DumpCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [self.parent])
        parser.add_argument("output")
        return parser

    def __call__(self, args):
        print "Fetching data..."
        video, data = self.getdata(args)

        print "Processing {0} tracks...".format(len(data))
        paths = [x.boxes for x in data]

        print "Highlighting frames..."
        it = vision.visualize.highlight_paths(video, paths)
        vision.visualize.save(it, lambda x: "{0}/{1}.jpg".format(args.output, x))

@handler("Dumps the tracking data")
class dump(DumpCommand):
    def setup(self):
        parser = argparse.ArgumentParser(parents = [self.parent])
        parser.add_argument("--output", "-o")
        parser.add_argument("--xml", "-x", action="store_true", default=False)
        parser.add_argument("--json", "-j", action="store_true", default=False)
        return parser

    def __call__(self, args):
        file = sys.stdout
        if args.output:
            file = open(args.output, 'w')

        video, data = self.getdata(args)

        if args.xml:
            self.dumpxml(file, data)
        elif args.json:
            self.dumpjson(file, data)
        else:
            self.dumptext(file, data)

        if args.output:
            file.close()

    def dumpxml(self, file, data):
        file.write("<annotations count=\"{0}\">\n".format(len(data)))
        for id, track in enumerate(data):
            file.write("\t<track id=\"{0}\" label=\"{1}\">\n".format(id, track.label))
            for box in track.boxes:
                file.write("\t\t<box frame=\"{0}\"".format(box.frame))
                file.write(" xtl=\"{0}\"".format(box.xtl))
                file.write(" ytl=\"{0}\"".format(box.ytl))
                file.write(" xbr=\"{0}\"".format(box.xbr))
                file.write(" ybr=\"{0}\"".format(box.ybr))
                file.write(" outside=\"{0}\"".format(box.lost))
                file.write(" occluded=\"{0}\" />\n".format(box.occluded))
            file.write("\t</track>\n")
        file.write("</annotations>\n")

    def dumpjson(self, file, data):
        file.write("annotations: {\n")
        for id, track in enumerate(data):
            file.write("\t{0}: {{\n".format(id))
            file.write("\t\tlabel: \"{0}\",\n".format(track.label))
            file.write("\t\tboxes: {\n")
            for box in track.boxes:
                file.write("\t\t\t{0}: {{\n".format(box.frame))
                file.write("\t\t\t\txtl: {0},\n".format(box.xtl))
                file.write("\t\t\t\tytl: {0},\n".format(box.ytl))
                file.write("\t\t\t\txbr: {0},\n".format(box.xbr))
                file.write("\t\t\t\tybr: {0},\n".format(box.ybr))
                file.write("\t\t\t\toutside: {0},\n".format(box.lost))
                file.write("\t\t\t\toccluded: {0}\n".format(box.occluded))
                file.write("\t\t\t},\n")
            file.write("\t\t},\n")
            file.write("\t},\n")
        file.write("}\n");

    def dumptext(self, file, data):
        for id, track in enumerate(data):
            for box in track.boxes:
                file.write(str(id))
                file.write(" ")
                file.write(str(box.xtl))
                file.write(" ")
                file.write(str(box.ytl))
                file.write(" ")
                file.write(str(box.xbr))
                file.write(" ")
                file.write(str(box.ybr))
                file.write(" ")
                file.write(str(box.frame))
                file.write(" ")
                file.write(str(box.lost))
                file.write(" ")
                file.write(str(box.occluded))
                file.write(" \"")
                file.write(track.label)
                file.write("\"\n")
<|endoftext|>