<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.secex.services import Product as ProductService
from dataviva.api.secex.services import ProductTradePartners as ProductTradePartnersService
from dataviva.api.secex.services import ProductMunicipalities as ProductMunicipalitiesService
from dataviva.api.secex.services import ProductLocations as ProductLocationsService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    header = {}
    body = {}

    header['product_id'] = product_id
    bra_id = request.args.get('bra_id')

    #None database fields must be treated (/05?bra_id=2ce020008),
    #templates with no data shall be omitted and verify usage of R$ and US$

    #Vars to do tests:
    #section (depth == 2)
    #positon (depth == 6)
    #brazil (bra_id == None)
    #product_id = #05 #052601
    #bra_id = #None #4mg #4mg01 #4mg0000 #4mg010206 #2ce020008

    context = {
        'background_image':'static/img/bg-profile-location.jpg',
        'portrait':'static/img/mineric_product.jpg'
    }

    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)
    context['depth'] = len(product_id)

    trade_partners_service = ProductTradePartnersService(product_id=product_id, bra_id=bra_id)
    municipalities_service = ProductMunicipalitiesService(product_id=product_id, bra_id=bra_id)

    if len(product_id) == 6:
            product_service = ProductService(product_id=product_id)
            header['pci'] = product_service.product_complexity()

    if bra_id:
        product_service = ProductLocationsService(product_id=product_id, bra_id=bra_id)

        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()
        body['export_value_growth_in_five_years'] = product_service.export_value_growth_in_five_years()

        if len(product_id) == 6:
            header['rca_wld'] = product_service.rca_wld()
            header['distance_wld'] = product_service.distance_wld()
            header['opportunity_gain_wld'] = product_service.opp_gain_wld()

        if len(bra_id) != 9:
            body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
            body['municipality_export_value'] = municipalities_service.highest_export_value()
            body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
            body['municipality_import_value'] = municipalities_service.highest_import_value()

    else:
        product_service = ProductService(product_id=product_id)

        body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
        body['municipality_export_value'] = municipalities_service.highest_export_value()
        body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
        body['municipality_import_value'] = municipalities_service.highest_import_value()
        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()
        body['export_value_growth_in_five_years'] = product_service.export_value_growth_in_five_years()

    header['name'] = product_service.product_name()
    header['year'] = product_service.year()
    header['trade_balance'] = product_service.trade_balance()
    header['export_value'] = product_service.total_exported()
    header['export_net_weight'] = product_service.unity_weight_export_price()
    header['import_value'] = product_service.total_imported()
    header['import_net_weight'] = product_service.unity_weight_import_price()

    from dataviva.api.secex.models import Ymp
    from dataviva import db
    from sqlalchemy.sql.expression import func, desc

    max_year_query = db.session.query(func.max(Ymp.year)).filter(Ymp.hs_id == product_id)

    secex_query = Ymp.query.filter(
        Ymp.year == max_year_query,
        Ymp.month == 0).order_by(Ymp.export_val.desc())
    secex = secex_query.all()

    for ranking, product in enumerate(secex):
        if secex[ranking].hs_id == product_id:
            header['export_value_ranking'] = ranking + 1
            break

    secex_query = Ymp.query.filter(
        Ymp.year == max_year_query,
        Ymp.month == 0).order_by(Ymp.import_val.desc())
    secex = secex_query.all()

    for ranking, product in enumerate(secex):
        if secex[ranking].hs_id == product_id:
            header['import_value_ranking'] = ranking + 1
            break

    return render_template('product/index.html', body_class='perfil-estado', header=header, body=body, context=context)
<commit_msg>Removing mock variables and reorganizing code.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.secex.services import Product as ProductService
from dataviva.api.secex.services import ProductTradePartners as ProductTradePartnersService
from dataviva.api.secex.services import ProductMunicipalities as ProductMunicipalitiesService
from dataviva.api.secex.services import ProductLocations as ProductLocationsService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    header = {}
    context = {}
    body = {}

    header['product_id'] = product_id
    context['depth'] = len(product_id)

    bra_id = request.args.get('bra_id')
    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)

    trade_partners_service = ProductTradePartnersService(product_id=product_id, bra_id=bra_id)
    municipalities_service = ProductMunicipalitiesService(product_id=product_id, bra_id=bra_id)

    if len(product_id) == 6:
            product_service = ProductService(product_id=product_id)
            header['pci'] = product_service.product_complexity()

    if bra_id:
        product_service = ProductLocationsService(product_id=product_id, bra_id=bra_id)

        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()
        body['export_value_growth_in_five_years'] = product_service.export_value_growth_in_five_years()

        if len(product_id) == 6:
            header['rca_wld'] = product_service.rca_wld()
            header['distance_wld'] = product_service.distance_wld()
            header['opportunity_gain_wld'] = product_service.opp_gain_wld()

        if len(bra_id) != 9:
            body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
            body['municipality_export_value'] = municipalities_service.highest_export_value()
            body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
            body['municipality_import_value'] = municipalities_service.highest_import_value()

    else:
        product_service = ProductService(product_id=product_id)

        body['municipality_name_export'] = municipalities_service.municipality_with_more_exports()
        body['municipality_export_value'] = municipalities_service.highest_export_value()
        body['municipality_name_import'] = municipalities_service.municipality_with_more_imports()
        body['municipality_import_value'] = municipalities_service.highest_import_value()
        body['destination_name_export'] = trade_partners_service.destination_with_more_exports()
        body['destination_export_value'] = trade_partners_service.highest_export_value()
        body['origin_name_import'] = trade_partners_service.origin_with_more_imports()
        body['origin_import_value'] = trade_partners_service.highest_import_value()
        body['export_value_growth_in_five_years'] = product_service.export_value_growth_in_five_years()

    header['name'] = product_service.product_name()
    header['year'] = product_service.year()
    header['trade_balance'] = product_service.trade_balance()
    header['export_value'] = product_service.total_exported()
    header['export_net_weight'] = product_service.unity_weight_export_price()
    header['import_value'] = product_service.total_imported()
    header['import_net_weight'] = product_service.unity_weight_import_price()

    #Get rankings vars, code should be refactored
    from dataviva.api.secex.models import Ymp
    from dataviva import db
    from sqlalchemy.sql.expression import func, desc

    max_year_query = db.session.query(func.max(Ymp.year)).filter(Ymp.hs_id == product_id)

    secex_query = Ymp.query.filter(
        Ymp.year == max_year_query,
        Ymp.month == 0).order_by(Ymp.export_val.desc())
    secex = secex_query.all()

    for ranking, product in enumerate(secex):
        if secex[ranking].hs_id == product_id:
            header['export_value_ranking'] = ranking + 1
            break

    secex_query = Ymp.query.filter(
        Ymp.year == max_year_query,
        Ymp.month == 0).order_by(Ymp.import_val.desc())
    secex = secex_query.all()

    for ranking, product in enumerate(secex):
        if secex[ranking].hs_id == product_id:
            header['import_value_ranking'] = ranking + 1
            break

    return render_template('product/index.html', body_class='perfil-estado', header=header, body=body, context=context)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, redirect, url_for, flash, jsonify, request, send_from_directory
from dataviva.apps.general.views import get_locale
from dataviva.translations.dictionary import dictionary
from dataviva import app, db, admin_email
from dataviva.utils import upload_helper
from models import Article, AuthorScholar, KeyWord
from forms import RegistrationForm
from sqlalchemy import desc
from datetime import datetime
from flask.ext.login import login_required
from dataviva.apps.admin.views import required_roles
from dataviva.utils.send_mail import send_mail
import os
import shutil
import fnmatch

mod = Blueprint('scholar', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/scholar')

@mod.before_request
def before_request():
    g.page_type = mod.name

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/', methods=['GET'])
def index():
    articles = Article.query.filter_by(approval_status=True).order_by(desc(Article.postage_date)).all()
    return render_template('scholar/index.html', articles=articles)

@mod.route('/article/<id>', methods=['GET'])
def show(id):
    article = Article.query.filter_by(id=id).first_or_404()
    return render_template('scholar/show.html', article=article)

@mod.route('/admin', methods=['GET'])
@login_required
@required_roles(1)
def admin():
    articles = Article.query.all()
    return render_template('scholar/admin.html', articles=articles)

@mod.route('/admin', methods=['POST'])
@login_required
@required_roles(1)
def admin_update():
    for id, approval_status in request.form.iteritems():
        article = Article.query.filter_by(id=id).first_or_404()
        article.approval_status = approval_status == u'true'
        db.session.commit()
    message = u"Estudo(s) atualizados com sucesso!"
    return message

@mod.route('/admin/article/<status>/<status_value>', methods=['POST'])
@login_required
@required_roles(1)
def admin_activate(status, status_value):
    for id in request.form.getlist('ids[]'):
        article = Article.query.filter_by(id=id).first_or_404()
        setattr(article, status, status_value == u'true')
        db.session.commit()

    message = u"Artigo(s) alterada(s) com sucesso!"
    return message, 200

def new_article_advise(article):
    advise_message = render_template('scholar/mail/new_article_advise.html', article=article)
    send_mail("New Article", [admin_email], advise_message)

@mod.route('/admin/article/new', methods=['GET'])
@login_required
def new():
    form = RegistrationForm()
    return render_template('scholar/new.html', form=form, action=url_for('scholar.create'))

@mod.route('/admin/article/new', methods=['POST'])
@login_required
def create():
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if not os.path.exists(upload_folder):
        flash(u'Selecione o arquivo do artigo para enviá-lo.', 'danger')
        return render_template('scholar/new.html', form=form)

    if form.validate() is False:
        return render_template('scholar/new.html', form=form)
    else:
        article = Article()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.approval_status = 0

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        db.session.add(article)
        db.session.flush()

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        new_article_advise(article)

        message = dictionary()["article_submission"]
        flash(message, 'success')

        return redirect(url_for('scholar.index'))

@mod.route('/admin/article/<id>/edit', methods=['GET'])
@login_required
@required_roles(1)
def edit(id):
    form = RegistrationForm()
    article = Article.query.filter_by(id=id).first_or_404()
    form.title.data = article.title
    form.theme.data = article.theme
    form.authors.data = article.authors_str()
    form.keywords.data = article.keywords_str()
    form.abstract.data = article.abstract
    article_url = article.url

    return render_template('scholar/edit.html', form=form, action=url_for('scholar.update', id=id), article_url=article_url)

@mod.route('/admin/article/<id>/edit', methods=['POST'])
@login_required
@required_roles(1)
def update(id):
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if form.validate() is False:
        return render_template('scholar/edit.html', form=form)
    else:
        article = Article.query.filter_by(id=id).first_or_404()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.authors = []
        article.keywords = []

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        message = u'Estudo editado com sucesso!'
        flash(message, 'success')
        return redirect(url_for('scholar.admin'))

@mod.route('/admin/article/delete', methods=['POST'])
@login_required
@required_roles(1)
def admin_delete():
    ids = request.form.getlist('ids[]')
    if ids:
        articles = Article.query.filter(Article.id.in_(ids)).all()
        for article in articles:
            upload_helper.delete_s3_folder(os.path.join(mod.name, str(article.id)))
            db.session.delete(article)

        db.session.commit()
        return u"Artigo(s) excluído(s) com sucesso!", 200
    else:
        return u'Selecione algum artigo para excluí-lo.', 205

@mod.route('/admin/articles/all', methods=['GET'])
@login_required
@required_roles(1)
def all():
    result = Article.query.all()
    articles = []
    for row in result:
        articles += [(row.id, row.title, row.authors_str(),
                      row.postage_date.strftime('%d/%m/%Y'), row.approval_status)]
    return jsonify(articles=articles)

@mod.route('/admin/article/upload', methods=['POST'])
@login_required
def upload():

    csrf_token = request.values['csrf_token']

    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        if [file for file in os.listdir(upload_folder)][0]:
            file_name = [file for file in os.listdir(upload_folder)][0]
            os.remove(os.path.join(upload_folder, file_name))
    else:
        os.makedirs(upload_folder)

    if request.files:
        file_name = request.files.keys()[0]
        file = request.files[file_name]
        file.save(os.path.join(upload_folder, file_name))
        return 'Arquivo salvo com sucesso!', 200
    else:
        return 'Não foi possível salvar o arquivo devido a um erro no servidor.', 400

@mod.route('/admin/article/delete', methods=['DELETE'])
@login_required
def delete():
    csrf_token = request.data
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        try:
            shutil.rmtree(os.path.split(upload_folder)[0])
            return 'Arquivo removido com sucesso!', 200
        except:
            return 'Não foi possível remover o arquivo devido a um erro no servidor.', 400

# serve static files on server
@mod.route('/admin/file/<string:csrf_token1>/<string:csrf_token2>', methods=['GET'])
@login_required
def get_file(csrf_token1, csrf_token2):
    import pdb; pdb.set_trace()
    csrf_token = csrf_token1 + '##' + csrf_token2
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')
    file_name = [file for file in os.listdir(upload_folder)][0]
    return send_from_directory(upload_folder, file_name)

# show static files on server before send to s3
@mod.route('/admin/data', methods=['GET'])
@login_required
@required_roles(1)
def data():
    matches = []
    for root, dirnames, filenames in os.walk(os.path.join(app.config['UPLOAD_FOLDER'], mod.name)):
        for filename in fnmatch.filter(filenames, '*.pdf'):
            matches.append(filename)
    return jsonify(file_names=matches)
<commit_msg>In scholar views, trying to fix problem in email send.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, redirect, url_for, flash, jsonify, request, send_from_directory
from dataviva.apps.general.views import get_locale
from dataviva.translations.dictionary import dictionary
from dataviva import app, db, admin_email
from dataviva.utils import upload_helper
from models import Article, AuthorScholar, KeyWord
from forms import RegistrationForm
from sqlalchemy import desc
from datetime import datetime
from flask.ext.login import login_required
from dataviva.apps.admin.views import required_roles
from dataviva.utils.send_mail import send_mail
import os
import shutil
import fnmatch

mod = Blueprint('scholar', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/scholar')

@mod.before_request
def before_request():
    g.page_type = mod.name

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/', methods=['GET'])
def index():
    articles = Article.query.filter_by(approval_status=True).order_by(desc(Article.postage_date)).all()
    return render_template('scholar/index.html', articles=articles)

@mod.route('/article/<id>', methods=['GET'])
def show(id):
    article = Article.query.filter_by(id=id).first_or_404()
    return render_template('scholar/show.html', article=article)

@mod.route('/admin', methods=['GET'])
@login_required
@required_roles(1)
def admin():
    articles = Article.query.all()
    return render_template('scholar/admin.html', articles=articles)

@mod.route('/admin', methods=['POST'])
@login_required
@required_roles(1)
def admin_update():
    for id, approval_status in request.form.iteritems():
        article = Article.query.filter_by(id=id).first_or_404()
        article.approval_status = approval_status == u'true'
        db.session.commit()
    message = u"Estudo(s) atualizados com sucesso!"
    return message

@mod.route('/admin/article/<status>/<status_value>', methods=['POST'])
@login_required
@required_roles(1)
def admin_activate(status, status_value):
    for id in request.form.getlist('ids[]'):
        article = Article.query.filter_by(id=id).first_or_404()
        setattr(article, status, status_value == u'true')
        db.session.commit()

    message = u"Artigo(s) alterada(s) com sucesso!"
    return message, 200

def new_article_advise(article):
    advise_message = render_template('scholar/mail/new_article_advise.html', article=article)
    send_mail("New Article", ["contato@dataviva.info","dataviva@fapemig.br"], advise_message)

@mod.route('/admin/article/new', methods=['GET'])
@login_required
def new():
    form = RegistrationForm()
    return render_template('scholar/new.html', form=form, action=url_for('scholar.create'))

@mod.route('/admin/article/new', methods=['POST'])
@login_required
def create():
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if not os.path.exists(upload_folder):
        flash(u'Selecione o arquivo do artigo para enviá-lo.', 'danger')
        return render_template('scholar/new.html', form=form)

    if form.validate() is False:
        return render_template('scholar/new.html', form=form)
    else:
        article = Article()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.approval_status = 0

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        db.session.add(article)
        db.session.flush()

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        new_article_advise(article)

        message = dictionary()["article_submission"]
        flash(message, 'success')

        return redirect(url_for('scholar.index'))

@mod.route('/admin/article/<id>/edit', methods=['GET'])
@login_required
@required_roles(1)
def edit(id):
    form = RegistrationForm()
    article = Article.query.filter_by(id=id).first_or_404()
    form.title.data = article.title
    form.theme.data = article.theme
    form.authors.data = article.authors_str()
    form.keywords.data = article.keywords_str()
    form.abstract.data = article.abstract
    article_url = article.url

    return render_template('scholar/edit.html', form=form, action=url_for('scholar.update', id=id), article_url=article_url)

@mod.route('/admin/article/<id>/edit', methods=['POST'])
@login_required
@required_roles(1)
def update(id):
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if form.validate() is False:
        return render_template('scholar/edit.html', form=form)
    else:
        article = Article.query.filter_by(id=id).first_or_404()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.authors = []
        article.keywords = []

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        message = u'Estudo editado com sucesso!'
        flash(message, 'success')
        return redirect(url_for('scholar.admin'))

@mod.route('/admin/article/delete', methods=['POST'])
@login_required
@required_roles(1)
def admin_delete():
    ids = request.form.getlist('ids[]')
    if ids:
        articles = Article.query.filter(Article.id.in_(ids)).all()
        for article in articles:
            upload_helper.delete_s3_folder(os.path.join(mod.name, str(article.id)))
            db.session.delete(article)

        db.session.commit()
        return u"Artigo(s) excluído(s) com sucesso!", 200
    else:
        return u'Selecione algum artigo para excluí-lo.', 205

@mod.route('/admin/articles/all', methods=['GET'])
@login_required
@required_roles(1)
def all():
    result = Article.query.all()
    articles = []
    for row in result:
        articles += [(row.id, row.title, row.authors_str(),
                      row.postage_date.strftime('%d/%m/%Y'), row.approval_status)]
    return jsonify(articles=articles)

@mod.route('/admin/article/upload', methods=['POST'])
@login_required
def upload():

    csrf_token = request.values['csrf_token']

    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        if [file for file in os.listdir(upload_folder)][0]:
            file_name = [file for file in os.listdir(upload_folder)][0]
            os.remove(os.path.join(upload_folder, file_name))
    else:
        os.makedirs(upload_folder)

    if request.files:
        file_name = request.files.keys()[0]
        file = request.files[file_name]
        file.save(os.path.join(upload_folder, file_name))
        return 'Arquivo salvo com sucesso!', 200
    else:
        return 'Não foi possível salvar o arquivo devido a um erro no servidor.', 400

@mod.route('/admin/article/delete', methods=['DELETE'])
@login_required
def delete():
    csrf_token = request.data
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        try:
            shutil.rmtree(os.path.split(upload_folder)[0])
            return 'Arquivo removido com sucesso!', 200
        except:
            return 'Não foi possível remover o arquivo devido a um erro no servidor.', 400

# serve static files on server
@mod.route('/admin/file/<string:csrf_token1>/<string:csrf_token2>', methods=['GET'])
@login_required
def get_file(csrf_token1, csrf_token2):
    import pdb; pdb.set_trace()
    csrf_token = csrf_token1 + '##' + csrf_token2
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')
    file_name = [file for file in os.listdir(upload_folder)][0]
    return send_from_directory(upload_folder, file_name)

# show static files on server before send to s3
@mod.route('/admin/data', methods=['GET'])
@login_required
@required_roles(1)
def data():
    matches = []
    for root, dirnames, filenames in os.walk(os.path.join(app.config['UPLOAD_FOLDER'], mod.name)):
        for filename in fnmatch.filter(filenames, '*.pdf'):
            matches.append(filename)
    return jsonify(file_names=matches)
<|endoftext|>


<commit_before># Copyright 2013-2015 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bottle Server Module."""

import copy
import logging
import os
import sys
import textwrap

import bottle

from simpl import config
from simpl.utils import cli as cli_utils

LOG = logging.getLogger(__name__)

_fill = lambda text: textwrap.fill(text, 50)

OPTIONS = [
    config.Option(
        '--app', '-a',
        help=("WSGI application to load by name.\n"
              "Ex: package.module gets the module\n"
              "    package.module:name gets the variable 'name'\n"
              "    package.module.func() calls func() and gets the result"),
        group='Server Options',
    ),
    config.Option(
        '--host',
        help='Server address to bind to.',
        default='127.0.0.1',
        group='Server Options',
    ),
    config.Option(
        '--port', '-p',
        help='Server port to bind to.',
        type=int,
        default=8080,
        group='Server Options',
    ),
    config.Option(
        '--server', '-s',
        help=('Server adapter to use. To see more, run:\n'
              '`python -c "import bottle;print('
              'bottle.server_names.keys())"`\n'),
        default='xtornado',
        group='Server Options',
    ),
    config.Option(
        '--debug-server',
        help=_fill(
            'Run bottle server with debug=True which is useful for '
            'development or troubleshooting. Warning: This may expose raw '
            'tracebacks and unmodified error messages in responses! Note: '
            'this is not an option to configure DEBUG level logging.'),
        default=False,
        action='store_true',
        group='Server Options',
    ),
    config.Option(
        '--quiet-server',
        help=_fill(
            'Suppress bottle\'s output to stdout and stderr, e.g. '
            '"Bottle v0.12.8 server starting up..." and others.'),
        default=False,
        action='store_true',
        group='Server Options',
    ),
    config.Option(
        '--no-reloader',
        default=True,
        dest='reloader',
        action='store_false',
        help=_fill(
            'Disable bottle auto-reloading server, which automatically '
            'restarts the server when file changes are detected. Note: '
            'some server adapters, such as eventlet, do not support '
            'auto-reloading.'),
        group='Server Options',
    ),
    config.Option(
        '--interval', '-i',
        help='Auto-reloader interval in seconds',
        type=int,
        default=1,
        group='Server Options',
    ),
    config.Option(
        '--adapter-options', '-o',
        help=(
            "Key-value pairs separated by '=' to be passed to \n"
            "the underlying server adapter, e.g. XEventletServer, \n"
            "and are mapped to the adapter's self.options \n"
            "instance attribute. Example usage:\n"
            "  simpl server -s xeventlet -o keyfile=~/mykey ciphers=GOST94\n"),
        nargs='*',
        type=cli_utils.kwarg,
        group='Server Options',
    ),
]

CONFIG = config.Config(
    prog='simpl_server',
    options=OPTIONS,
    argparser_class=cli_utils.HelpfulParser,
    formatter_class=cli_utils.SimplHelpFormatter
)

class EventletLogFilter(object):  # pylint: disable=R0903

    """Receives eventlet log.write() calls and routes them.

    Thread and WSGI messages only get written in debug mode. Eventlet writes
    these out starting with a thread id in parens or "wsgi" messages:
    - "(46722) wsgi starting up on http://127.0.0.1:8080/"
    - "(47001) accepted ('127.0.0.1', 53046)"
    - "wsgi exiting"

    All other calls are assumed to be apache log-style API calls and we want
    these written as INFO and to an access_log if provided.

    API calls are assumed to start with the IP address:
    - "127.0.0.1 - - [07/Jul/2015 16:16:31] "GET /version HTTP/1.1" 200 ..."

    An instance of this class can be passed in to a bottle.run command using
    the `log` keyword.
    """

    def __init__(self, log, access_log=None):
        """Initialize with config and optional access_log.

        :param log: logger instance (ex. logging.getLogger()).
        :keyword access_log: a file handle to an access log that should receive
            apache-style entries for each call and response.
        """
        self.log = log
        self.access_log = access_log

    def write(self, text):
        """Write to appropriate target."""
        if text:
            if text[0] in '(w':
                # write thread and wsgi messages to debug only
                self.log.debug(text[:-1])
                return
            if self.access_log:
                self.access_log.write(text)
            self.log.info(text[:-1])

class XEventletServer(bottle.ServerAdapter):

    r"""Eventlet Bottle Server Adapter with extensions.

    Supports SSL. Accepts additional tuning parameters:

    * `backlog` adjust the eventlet backlog parameter which is the maximum
      number of queued connections. Should be at least 1; the maximum
      value is system-dependent.
    * `family`: (default is 2) socket family, optional. See socket
      documentation for available families.
    * `**kwargs`: directly map to python's ssl.wrap_socket arguments from
      https://docs.python.org/2/library/ssl.html#ssl.wrap_socket and
      wsgi.server arguments from
      http://eventlet.net/doc/modules/wsgi.html#wsgi-wsgi-server

    To create a self-signed key and start the eventlet server using SSL::

      openssl genrsa -des3 -out server.orig.key 2048
      openssl rsa -in server.orig.key -out test.key
      openssl req -new -key test.key -out server.csr
      openssl x509 -req -days 365 -in server.csr -signkey test.key -out \
      test.crt

      bottle.run(server='eventlet', keyfile='test.key', certfile='test.crt')
    """

    def get_socket(self):
        """Create listener socket based on bottle server parameters."""
        import eventlet

        # Separate out socket.listen arguments
        socket_args = {}
        for arg in ('backlog', 'family'):
            try:
                socket_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        # Separate out wrap_ssl arguments
        ssl_args = {}
        for arg in ('keyfile', 'certfile', 'server_side', 'cert_reqs',
                    'ssl_version', 'ca_certs', 'do_handshake_on_connect',
                    'suppress_ragged_eofs', 'ciphers'):
            try:
                ssl_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        address = (self.host, self.port)
        try:
            sock = eventlet.listen(address, **socket_args)
        except TypeError:
            # Fallback, if we have old version of eventlet
            sock = eventlet.listen(address)
        if ssl_args:
            sock = eventlet.wrap_ssl(sock, **ssl_args)
        return sock

    def run(self, handler):
        """Start bottle server."""
        import eventlet.patcher
        if not eventlet.patcher.is_monkey_patched(os):
            msg = ("%s requires eventlet.monkey_patch() (before "
                   "import)" % self.__class__.__name__)
            raise RuntimeError(msg)

        # Separate out wsgi.server arguments
        wsgi_args = {}
        for arg in ('log', 'environ', 'max_size', 'max_http_version',
                    'protocol', 'server_event', 'minimum_chunk_size',
                    'log_x_forwarded_for', 'custom_pool', 'keepalive',
                    'log_output', 'log_format', 'url_length_limit', 'debug',
                    'socket_timeout', 'capitalize_response_headers'):
            try:
                wsgi_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        if 'log_output' not in wsgi_args:
            wsgi_args['log_output'] = not self.quiet

        import eventlet.wsgi
        sock = self.options.pop('shared_socket', None) or self.get_socket()
        eventlet.wsgi.server(sock, handler, **wsgi_args)

    def __repr__(self):
        """Show class name, even if subclassed."""
        return self.__class__.__name__

bottle.server_names['xeventlet'] = XEventletServer

class XTornadoServer(bottle.ServerAdapter):  # pylint: disable=R0903

    """The Tornado Server Adapter with xheaders enabled."""

    def run(self, handler):
        """Start up the server."""
        import tornado.httpserver
        import tornado.ioloop
        import tornado.wsgi
        container = tornado.wsgi.WSGIContainer(handler)
        server = tornado.httpserver.HTTPServer(container, xheaders=True)
        server.listen(port=self.port, address=self.host)
        tornado.ioloop.IOLoop.instance().start()

bottle.server_names['xtornado'] = XTornadoServer

def attach_parser(subparser):
    """Given a subparser, build and return the server parser."""
    return subparser.add_parser(
        'server',
        help='Run a bottle based server',
        parents=[
            CONFIG.build_parser(
                add_help=False,
                # might need conflict_handler
            ),
        ],
    )

def run(conf):
    """Simpl server command line interface."""
    if isinstance(conf.adapter_options, list):
        options = {key: val for _dict in conf.adapter_options
                   for key, val in _dict.items()}
    elif conf.adapter_options is None:
        options = {}
    else:
        options = copy.copy(conf.adapter_options)
    try:
        if conf.reloader and not os.getenv('BOTTLE_CHILD'):
            LOG.info("Running bottle server with reloader.")
        return bottle.run(
            app=conf.app,
            server=conf.server,
            host=conf.host,
            port=conf.port,
            interval=conf.interval,
            reloader=conf.reloader,
            quiet=conf.quiet_server,
            debug=conf.debug_server,
            **options
        )
    except KeyboardInterrupt:
        sys.exit("\nKilled simpl server.")

def main(argv=None):
    """Entry point for server, runs based on parsed CONFIG."""
    CONFIG.parse(argv=argv)
    return run(CONFIG)

if __name__ == '__main__':
    main()
<commit_msg>fix(simpl-server): add cwd to sys.path if needed<commit_after># Copyright 2013-2015 Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Bottle Server Module."""

import copy
import logging
import os
import sys
import textwrap

import bottle

from simpl import config
from simpl.utils import cli as cli_utils

LOG = logging.getLogger(__name__)

_fill = lambda text: textwrap.fill(text, 50)

OPTIONS = [
    config.Option(
        '--app', '-a',
        help=("WSGI application to load by name.\n"
              "Ex: package.module gets the module\n"
              "    package.module:name gets the variable 'name'\n"
              "    package.module.func() calls func() and gets the result"),
        group='Server Options',
    ),
    config.Option(
        '--host',
        help='Server address to bind to.',
        default='127.0.0.1',
        group='Server Options',
    ),
    config.Option(
        '--port', '-p',
        help='Server port to bind to.',
        type=int,
        default=8080,
        group='Server Options',
    ),
    config.Option(
        '--server', '-s',
        help=('Server adapter to use. To see more, run:\n'
              '`python -c "import bottle;print('
              'bottle.server_names.keys())"`\n'),
        default='xtornado',
        group='Server Options',
    ),
    config.Option(
        '--debug-server',
        help=_fill(
            'Run bottle server with debug=True which is useful for '
            'development or troubleshooting. Warning: This may expose raw '
            'tracebacks and unmodified error messages in responses! Note: '
            'this is not an option to configure DEBUG level logging.'),
        default=False,
        action='store_true',
        group='Server Options',
    ),
    config.Option(
        '--quiet-server',
        help=_fill(
            'Suppress bottle\'s output to stdout and stderr, e.g. '
            '"Bottle v0.12.8 server starting up..." and others.'),
        default=False,
        action='store_true',
        group='Server Options',
    ),
    config.Option(
        '--no-reloader',
        default=True,
        dest='reloader',
        action='store_false',
        help=_fill(
            'Disable bottle auto-reloading server, which automatically '
            'restarts the server when file changes are detected. Note: '
            'some server adapters, such as eventlet, do not support '
            'auto-reloading.'),
        group='Server Options',
    ),
    config.Option(
        '--interval', '-i',
        help='Auto-reloader interval in seconds',
        type=int,
        default=1,
        group='Server Options',
    ),
    config.Option(
        '--adapter-options', '-o',
        help=(
            "Key-value pairs separated by '=' to be passed to \n"
            "the underlying server adapter, e.g. XEventletServer, \n"
            "and are mapped to the adapter's self.options \n"
            "instance attribute. Example usage:\n"
            "  simpl server -s xeventlet -o keyfile=~/mykey ciphers=GOST94\n"),
        nargs='*',
        type=cli_utils.kwarg,
        group='Server Options',
    ),
]

CONFIG = config.Config(
    prog='simpl_server',
    options=OPTIONS,
    argparser_class=cli_utils.HelpfulParser,
    formatter_class=cli_utils.SimplHelpFormatter
)

class EventletLogFilter(object):  # pylint: disable=R0903

    """Receives eventlet log.write() calls and routes them.

    Thread and WSGI messages only get written in debug mode. Eventlet writes
    these out starting with a thread id in parens or "wsgi" messages:
    - "(46722) wsgi starting up on http://127.0.0.1:8080/"
    - "(47001) accepted ('127.0.0.1', 53046)"
    - "wsgi exiting"

    All other calls are assumed to be apache log-style API calls and we want
    these written as INFO and to an access_log if provided.

    API calls are assumed to start with the IP address:
    - "127.0.0.1 - - [07/Jul/2015 16:16:31] "GET /version HTTP/1.1" 200 ..."

    An instance of this class can be passed in to a bottle.run command using
    the `log` keyword.
    """

    def __init__(self, log, access_log=None):
        """Initialize with config and optional access_log.

        :param log: logger instance (ex. logging.getLogger()).
        :keyword access_log: a file handle to an access log that should receive
            apache-style entries for each call and response.
        """
        self.log = log
        self.access_log = access_log

    def write(self, text):
        """Write to appropriate target."""
        if text:
            if text[0] in '(w':
                # write thread and wsgi messages to debug only
                self.log.debug(text[:-1])
                return
            if self.access_log:
                self.access_log.write(text)
            self.log.info(text[:-1])

class XEventletServer(bottle.ServerAdapter):

    r"""Eventlet Bottle Server Adapter with extensions.

    Supports SSL. Accepts additional tuning parameters:

    * `backlog` adjust the eventlet backlog parameter which is the maximum
      number of queued connections. Should be at least 1; the maximum
      value is system-dependent.
    * `family`: (default is 2) socket family, optional. See socket
      documentation for available families.
    * `**kwargs`: directly map to python's ssl.wrap_socket arguments from
      https://docs.python.org/2/library/ssl.html#ssl.wrap_socket and
      wsgi.server arguments from
      http://eventlet.net/doc/modules/wsgi.html#wsgi-wsgi-server

    To create a self-signed key and start the eventlet server using SSL::

      openssl genrsa -des3 -out server.orig.key 2048
      openssl rsa -in server.orig.key -out test.key
      openssl req -new -key test.key -out server.csr
      openssl x509 -req -days 365 -in server.csr -signkey test.key -out \
      test.crt

      bottle.run(server='eventlet', keyfile='test.key', certfile='test.crt')
    """

    def get_socket(self):
        """Create listener socket based on bottle server parameters."""
        import eventlet

        # Separate out socket.listen arguments
        socket_args = {}
        for arg in ('backlog', 'family'):
            try:
                socket_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        # Separate out wrap_ssl arguments
        ssl_args = {}
        for arg in ('keyfile', 'certfile', 'server_side', 'cert_reqs',
                    'ssl_version', 'ca_certs', 'do_handshake_on_connect',
                    'suppress_ragged_eofs', 'ciphers'):
            try:
                ssl_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        address = (self.host, self.port)
        try:
            sock = eventlet.listen(address, **socket_args)
        except TypeError:
            # Fallback, if we have old version of eventlet
            sock = eventlet.listen(address)
        if ssl_args:
            sock = eventlet.wrap_ssl(sock, **ssl_args)
        return sock

    def run(self, handler):
        """Start bottle server."""
        import eventlet.patcher
        if not eventlet.patcher.is_monkey_patched(os):
            msg = ("%s requires eventlet.monkey_patch() (before "
                   "import)" % self.__class__.__name__)
            raise RuntimeError(msg)

        # Separate out wsgi.server arguments
        wsgi_args = {}
        for arg in ('log', 'environ', 'max_size', 'max_http_version',
                    'protocol', 'server_event', 'minimum_chunk_size',
                    'log_x_forwarded_for', 'custom_pool', 'keepalive',
                    'log_output', 'log_format', 'url_length_limit', 'debug',
                    'socket_timeout', 'capitalize_response_headers'):
            try:
                wsgi_args[arg] = self.options.pop(arg)
            except KeyError:
                pass
        if 'log_output' not in wsgi_args:
            wsgi_args['log_output'] = not self.quiet

        import eventlet.wsgi
        sock = self.options.pop('shared_socket', None) or self.get_socket()
        eventlet.wsgi.server(sock, handler, **wsgi_args)

    def __repr__(self):
        """Show class name, even if subclassed."""
        return self.__class__.__name__

bottle.server_names['xeventlet'] = XEventletServer

class XTornadoServer(bottle.ServerAdapter):  # pylint: disable=R0903

    """The Tornado Server Adapter with xheaders enabled."""

    def run(self, handler):
        """Start up the server."""
        import tornado.httpserver
        import tornado.ioloop
        import tornado.wsgi
        container = tornado.wsgi.WSGIContainer(handler)
        server = tornado.httpserver.HTTPServer(container, xheaders=True)
        server.listen(port=self.port, address=self.host)
        tornado.ioloop.IOLoop.instance().start()

bottle.server_names['xtornado'] = XTornadoServer

def attach_parser(subparser):
    """Given a subparser, build and return the server parser."""
    return subparser.add_parser(
        'server',
        help='Run a bottle based server',
        parents=[
            CONFIG.build_parser(
                add_help=False,
                # might need conflict_handler
            ),
        ],
    )

def run(conf):
    """Simpl server command line interface."""
    if isinstance(conf.adapter_options, list):
        options = {key: val for _dict in conf.adapter_options
                   for key, val in _dict.items()}
    elif conf.adapter_options is None:
        options = {}
    else:
        options = copy.copy(conf.adapter_options)

    # waiting for https://github.com/bottlepy/bottle/pull/783
    if conf.app and (os.getcwd() not in sys.path):
        sys.path.append(os.getcwd())

    try:
        if conf.reloader and not os.getenv('BOTTLE_CHILD'):
            LOG.info("Running bottle server with reloader.")
        return bottle.run(
            app=conf.app,
            server=conf.server,
            host=conf.host,
            port=conf.port,
            interval=conf.interval,
            reloader=conf.reloader,
            quiet=conf.quiet_server,
            debug=conf.debug_server,
            **options
        )
    except KeyboardInterrupt:
        sys.exit("\nKilled simpl server.")

def main(argv=None):
    """Entry point for server, runs based on parsed CONFIG."""
    CONFIG.parse(argv=argv)
    return run(CONFIG)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 Mick Krippendorf <m.krippendorf@freenet.de>

__version__ = '0.2.3a'
__date__ = '2014-09-27'
__author__ = 'Mick Krippendorf <m.krippendorf@freenet.de>'
__license__ = 'MIT'

from functools import partial, wraps, reduce as foldl
from itertools import chain, count, tee, zip_longest

from toolz.functoolz import flip

def noop(*a, **k):
    return None

def wrap(f):
    return wraps(f)(lambda *a, **k: f(*a, **k))

def increment(x):
    return x + 1

def decrement(x):
    return x - 1

def const(x):
    "The K combinator"
    return lambda _: x

def first_arg(x, *a, **k):
    return x

def tabulate(function, start=0):
    "Return function(0), function(1), ..."
    return map(function, count(start))

_sentinel = object()

def foldr(func, seq, start=_sentinel):
    if start is _sentinel:
        return foldl(flip(func), reversed(seq))
    return foldl(flip(func), reversed(seq), start)

def rfoldl(func, seq, start=_sentinel):
    if start is _sentinel:
        return foldl(flip(func), seq)
    return foldl(flip(func), seq, start)

def rfoldr(func, seq, start=_sentinel):
    if start is _sentinel:
        return foldl(func, reversed(seq))
    return foldl(func, reversed(seq), start)

def rpartial(f, *args, **kwargs):
    return partial(flip(f), *args, **kwargs)

def pairwise(iterable, *, rotate=False, fillvalue=_sentinel):
    if rotate and fillvalue is not _sentinel:
        raise ValueError(
            'The parameters rotate and fillvalue are mutually exclusive.')
    a, b = tee(iterable)
    last = next(b, None)
    if rotate:
        return zip(a, chain(b, [last]))
    elif fillvalue is _sentinel:
        return zip(a, b)
    else:
        return zip_longest(a, b, fillvalue=fillvalue)

def qualname(fullname):

    name = fullname.rsplit('.', 1).pop()

    def name_setter(func):
        func.__name__ = name
        func.__qualname__ = fullname
        return func

    return name_setter

def rotate(iterable):
    iterable = iter(iterable)
    tmp = next(iterable)
    yield from iterable
    yield tmp

def splitpairs(iterable):
    iterable = iter(iterable)
    for left in iterable:
        right = next(iterable)
        yield left, right

def install_symbols_module(name, factory):

    import sys

    from functools import lru_cache
    from importlib.abc import MetaPathFinder, Loader
    from importlib.machinery import ModuleSpec
    from types import ModuleType

    class SymbolsModule(ModuleType):
        __all__ = []
        __file__ = None  # needed so nose doesn't barf at us
        __getattr__ = staticmethod(lru_cache()(factory))

    class SymbolsImporter(MetaPathFinder, Loader):

        def find_spec(self, fullname, path=None, target=None):
            if fullname == name:
                return ModuleSpec(fullname, self)

        def create_module(self, spec):
            return sys.modules.setdefault(spec.name, SymbolsModule(spec.name))

        def exec_module(self, module):
            pass

    sys.meta_path.insert(0, SymbolsImporter())
<commit_msg>removed some unused functions from util.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2014 Mick Krippendorf <m.krippendorf@freenet.de>

__version__ = '0.2.3a'
__date__ = '2014-09-27'
__author__ = 'Mick Krippendorf <m.krippendorf@freenet.de>'
__license__ = 'MIT'

from functools import partial, reduce as foldl
from itertools import chain, count, tee, zip_longest

from toolz.functoolz import flip

def noop(*a, **k):
    return None

def increment(x):
    return x + 1

def decrement(x):
    return x - 1

def const(x):
    "The K combinator"
    return lambda _: x

def first_arg(x, *a, **k):
    return x

def tabulate(function, start=0):
    "Return function(0), function(1), ..."
    return map(function, count(start))

_sentinel = object()

def foldr(func, seq, start=_sentinel):
    if start is _sentinel:
        return foldl(flip(func), reversed(seq))
    return foldl(flip(func), reversed(seq), start)

def rpartial(f, *args, **kwargs):
    return partial(flip(f), *args, **kwargs)

def pairwise(iterable, *, rotate=False, fillvalue=_sentinel):
    if rotate and fillvalue is not _sentinel:
        raise ValueError(
            'The parameters rotate and fillvalue are mutually exclusive.')
    a, b = tee(iterable)
    last = next(b, None)
    if rotate:
        return zip(a, chain(b, [last]))
    elif fillvalue is _sentinel:
        return zip(a, b)
    else:
        return zip_longest(a, b, fillvalue=fillvalue)

def qualname(fullname):

    name = fullname.rsplit('.', 1).pop()

    def name_setter(func):
        func.__name__ = name
        func.__qualname__ = fullname
        return func

    return name_setter

def rotate(iterable):
    iterable = iter(iterable)
    tmp = next(iterable)
    yield from iterable
    yield tmp

def splitpairs(iterable):
    iterable = iter(iterable)
    for left in iterable:
        right = next(iterable)
        yield left, right

def install_symbols_module(name, factory):

    import sys

    from functools import lru_cache
    from importlib.abc import MetaPathFinder, Loader
    from importlib.machinery import ModuleSpec
    from types import ModuleType

    class SymbolsModule(ModuleType):
        __all__ = []
        __file__ = None  # needed so nose doesn't barf at us
        __getattr__ = staticmethod(lru_cache()(factory))

    class SymbolsImporter(MetaPathFinder, Loader):

        def find_spec(self, fullname, path=None, target=None):
            if fullname == name:
                return ModuleSpec(fullname, self)

        def create_module(self, spec):
            return sys.modules.setdefault(spec.name, SymbolsModule(spec.name))

        def exec_module(self, module):
            pass

    sys.meta_path.insert(0, SymbolsImporter())
<|endoftext|>


<commit_before>__author__ = 'willi'
<commit_msg>add basic tool for printing out the SDS index entries<commit_after>#!/bin/python

#    This file is part of INDXParse.
#
#   Copyright 2011-13 Will Ballenthin <william.ballenthin@mandiant.com>
#                    while at Mandiant <http://www.mandiant.com>
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
#   Version v.1.2
from SDS import SDS

def main():
    import sys
    import mmap
    import contextlib
    import argparse

    parser = argparse.ArgumentParser(description='Get an SDS record by index.')
    parser.add_argument('-v', action="store_true", dest="verbose",
                        help="Print debugging information")
    parser.add_argument('SDS', action="store",
                        help="Input SDS file path")
    parser.add_argument('index', action="store", type=int,
                        help="Entry index to fetch")
    results = parser.parse_args()

    with open(results.SDS, 'r') as f:
        with contextlib.closing(mmap.mmap(f.fileno(), 0,
                                          access=mmap.ACCESS_READ)) as buf:
            s = SDS(buf, 0, None)
            print "SDS"
            for e in s.sds_entries():
                print("  SDS_ENTRY")
                print(e.get_all_string(indent=2))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Functions to load data from Worldbank API.

We use its bulkdownload utilities.

Source link: `WorldBank website`_

.. _WorldBank website: http://data.worldbank.org

"""

import os.path as osp

import requests
import pandas as pd

from urllib.parse import urlsplit
from concurrent.futures import ThreadPoolExecutor, as_completed

from . common import DataFactory

class WorldBankLoader(DataFactory):
    __doc__ = """T.B.D"""
    url = 'http://api.worldbank.org/v2/datacatalog?format=json'

    def load_metadata(self):
        def load_url(url, page=None):
            if page:
                url = url + '&page=' + str(page)
            res = requests.get(url)
            return res.json()

        def handle_data(data):
            for v in data['datacatalog']:
                d = {}
                for v_ in v['metatype']:
                    d[v_['id']] = v_['value']
                yield d

        first_page = self.url
        data = load_url(first_page)

        metadata = []

        for d in handle_data(data):
            metadata.append(d)

        pages = range(2, data['pages']+1)

        executor = ThreadPoolExecutor(max_workers=3)
        res = [executor.submit(load_url, self.url, page=p) for p in pages]

        for r in as_completed(res):
            for d in handle_data(r.result()):
                # print(d)
                metadata.append(d)

        metadata = pd.DataFrame.from_records(metadata)
        self.metadata = metadata
        return metadata

    def has_newer_source(self, dataset, date):
        if self.metadata is None:
            self.load_metadata()
        metadata = self.metadata
        lastrevisiondate = metadata.loc[
            metadata.acronym == dataset,
            'lastrevisiondate'].values[0]
        if pd.to_datetime(lastrevisiondate) > pd.to_datetime(date):
            return True
        else:
            return False

    def bulk_download(self, dataset, out_dir):
        if self.metadata is None:
            self.load_metadata()
        metadata = self.metadata
        s = metadata.loc[metadata.acronym == dataset, 'bulkdownload'].values[0]
        url = s.split(';')[1].split('=')[1]
        res = requests.get(url, stream=True)

        fs_path = osp.join(out_dir, osp.basename(urlsplit(url).path))

        with open(fs_path, 'wb') as f:
            for c in res.iter_content(chunk_size=1024):
                f.write(c)
            f.close()

        return out_dir
<commit_msg>worldbank data downloader update<commit_after># -*- coding: utf-8 -*-

"""Functions to load data from Worldbank API.

We use its bulkdownload utilities.

Source link: `WorldBank website`_

.. _WorldBank website: http://data.worldbank.org

"""

import os.path as osp

import logging
import requests
import pandas as pd

from urllib.parse import urlsplit
from concurrent.futures import ThreadPoolExecutor, as_completed

from . common import DataFactory, requests_retry_session

logger = logging.getLogger("WorldBankLoader")

class WorldBankLoader(DataFactory):
    __doc__ = """T.B.D"""
    url = 'http://api.worldbank.org/v2/datacatalog?format=json'

    def load_metadata(self):
        def load_url(url, page=None):
            if page:
                url = url + '&page=' + str(page)
            res = requests.get(url)
            return res.json()

        def handle_data(data):
            for v in data['datacatalog']:
                d = {}
                for v_ in v['metatype']:
                    d[v_['id']] = v_['value']
                yield d

        first_page = self.url
        data = load_url(first_page)

        metadata = []

        for d in handle_data(data):
            metadata.append(d)

        pages = range(2, data['pages']+1)

        executor = ThreadPoolExecutor(max_workers=3)
        res = [executor.submit(load_url, self.url, page=p) for p in pages]

        for r in as_completed(res):
            for d in handle_data(r.result()):
                # print(d)
                metadata.append(d)

        metadata = pd.DataFrame.from_records(metadata)
        self.metadata = metadata
        return metadata

    def has_newer_source(self, dataset, date):
        if self.metadata is None:
            self.load_metadata()
        metadata = self.metadata
        lastrevisiondate = metadata.loc[
            metadata.acronym == dataset,
            'lastrevisiondate'].values[0]
        if pd.to_datetime(lastrevisiondate) > pd.to_datetime(date):
            return True
        else:
            return False

    def bulk_download(self, dataset, out_dir):
        if self.metadata is None:
            self.load_metadata()
        metadata = self.metadata
        s = metadata.loc[metadata.acronym == dataset, 'bulkdownload'].values[0]
        url = s.split(';')[1].split('=')[1]
        session = requests_retry_session()
        res = session.get(url, stream=True)

        fs_path = osp.join(out_dir, osp.basename(urlsplit(url).path))

        with open(fs_path, 'wb') as f:
            for c in res.iter_content(chunk_size=1024):
                f.write(c)
            f.close()

        return out_dir
<|endoftext|>


<commit_before>from rl.util import log_self

class Optimizer(object):

    '''
    The base class of Optimizer, with the core methods
    '''

    def __init__(self, **kwargs):
        '''Construct externally, and set at Agent.compile()'''
        self.agent = None
        self.keras_optimizer = None
        self.optim_param = {}
        self.update_optim_param(**kwargs)
        self.init_optimizer()
        log_self(self)

    def update_optim_param(self, **kwargs):
        o_param = {
            k: kwargs.get(k) for k in self.optim_param_keys
            if kwargs.get(k) is not None}
        self.optim_param.update(o_param)

    def init_optimizer(self):
        raise NotImplementedError()

    def change_optim_param(self, **new_param):
        self.update_optim_param(**new_param)
        self.init_optimizer()
        logger.info("Optimizer param changed")
        log_self(self)
<commit_msg>fix missing logger import<commit_after>from rl.util import log_self, logger

class Optimizer(object):

    '''
    The base class of Optimizer, with the core methods
    '''

    def __init__(self, **kwargs):
        '''Construct externally, and set at Agent.compile()'''
        self.agent = None
        self.keras_optimizer = None
        self.optim_param = {}
        self.update_optim_param(**kwargs)
        self.init_optimizer()
        log_self(self)

    def update_optim_param(self, **kwargs):
        o_param = {
            k: kwargs.get(k) for k in self.optim_param_keys
            if kwargs.get(k) is not None}
        self.optim_param.update(o_param)

    def init_optimizer(self):
        raise NotImplementedError()

    def change_optim_param(self, **new_param):
        self.update_optim_param(**new_param)
        self.init_optimizer()
        logger.info("Optimizer param changed")
        log_self(self)
<|endoftext|>


<commit_before>from math import pi

#import socket
from ..communication.network import unicast
from ..communication import grsim
from collections import defaultdict
from time import time

from ..utils.mathutils import sin, cos
from ..utils.keydefaultdict import keydefaultdict

class Commander(object):
#class Commander(Process):

    def __init__(self):
        #super(Commander, self).__init__()
        #self._recvr, self._sendr = Pipe()
        #self.conn = None
        #self._exit = Event()
        pass

    #def start(self):
    #    super(Commander, self).start()
    #    self.conn = self._sendr

    #def run(self):
    #    self.conn = self._recvr
    #    while self._exit.is_set():
    #        self.send(self.conn.recv())
    #    self.conn.close()

    #def stop(self):
    #    self.conn.close()
    #    self._exit.set()

    def send(self, actions):
        raise NotImplemented

class Tx2012Commander(Commander):
    '''
    Sends commands as strings via unicast UDP to the C# transmitter process that will dispatch commands
    via RF to the robots. Since the robots' firmware IDs are not the same as the UIDs sent by the
    VisionUpdater, a dict mapping the UIDs to the firmware IDs should be provided. If it is not provided,
    we shall use the trivial mapping: x => x.

    This commander uses a transmission protocol compatible with the RoboIME MK-2012 architecture.
    This might be deprecated soon. Or not.
    '''
    def __init__(self, team, mapping_dict=None, kicking_power_dict=None, ipaddr='127.0.0.1', port=9050, verbose=False, **kwargs):
        super(Tx2012Commander, self).__init__(**kwargs)
        self.default_map = mapping_dict is None
        self.mapping_dict = mapping_dict if mapping_dict is not None else keydefaultdict(lambda x: x)
        self.kicking_power_dict = kicking_power_dict if kicking_power_dict is not None else defaultdict(lambda: 100)
        self.team = team
        self.verbose = verbose
        self.sender = unicast.UnicastSender(address=(ipaddr, port))
        #self.sock = so
        # FIXME: These values should be on the robot prototype to allow for mixed-chassis teams. NOT HERE!

        # RoboIME 2013
        self.wheel_angles = [
            -60.,
            -135.,
            +135.,
            +60.,
        ]

        # RoboIME 2012
        #self.wheel_angles = [
        #    -57.,
        #    +57.,
        #    +135.,
        #    -135.,
        #]
        #self.wheel_radius = 28.9
        #self.wheel_distance = 80.6

        #Values in meters.
        self.wheel_radius = .0289
        self.wheel_distance = .0806

    def omniwheel_speeds(self, theta, vx, vy, va):
        speeds = []
        for j in xrange(4):
            a = self.wheel_angles[j]
            val = cos(a) * (vy * cos(theta) - vx * sin(theta)) - sin(a) * (vx * cos(theta) + vy * sin(theta)) + va * self.wheel_distance
            val /= self.wheel_radius
            speeds.append(val)
        return speeds

    def send(self, actions):
        actions_dict = defaultdict(lambda:['0','0','0','0','0','0','0'])
        dirty = False
        if len(actions) > 0:
            for a in actions:
                string_list = []
                if not a:
                    continue
                dirty = True
                vx, vy, va = a.speeds
                # Convert va to angular speed.
                va = va * pi / 180
                if self.default_map or a.uid in self.mapping_dict:
                    pass
                    #string_list.append(str(self.mapping_dict[a.uid]))
                else:
                    continue
                #string_list.extend(['10','10','10','10'])
                string_list.extend([str(i) for i in self.omniwheel_speeds(a.robot.angle, vx, vy, va)])
                string_list.append(str((a.dribble or 0.0)))
                if a.kick > 0 and self.kicking_power_dict[a.uid] > 0:
                    string_list.append(str((a.kick * 100 / self.kicking_power_dict[a.uid] or 0.0)))
                    #string_list.append(str((a.kick or 0.0)))
                    string_list.append('0')
                elif a.chipkick > 0 and self.kicking_power_dict[a.uid] > 0:
                    string_list.append('0')
                    string_list.append(str((a.chipkick * 100 /self.kicking_power_dict[a.uid] or 0.0)))
                    #string_list.append(str((a.chipkick or 0.0)))
                else:
                    string_list.append('0')
                    string_list.append('0')

                actions_dict[self.mapping_dict[a.uid]] = string_list
                a.reset()
            if dirty:
                string_list = []
                for i in xrange(6):
                    string_list.extend(actions_dict[i])

                packet = ' '.join(string_list)

                if packet:
                    if self.verbose:
                        print self.kicking_power_dict
                        print packet
                    self.sender.send(packet)

class TxCommander(Commander):
    '''
    Sends commands as strings via unicast UDP to the C# transmitter process that will dispatch commands
    via RF to the robots. Since the robots' firmware IDs are not the same as the UIDs sent by the
    VisionUpdater, a function mapping the UIDs to the firmware IDs should be provided. If it is not provided,
    we shall use the trivial mapping: x => x.

    This commander uses the transmission protocol for the RoboIME MK-2013 architecture.

    Main differences are:

    - Not passing wheel velocities directly, sending instead the triplet vx, vy, theta;
    - Chip kick.
    '''
    def __init__(self, team, mapping_dict=None, ipaddr='127.0.0.1', port=9050, verbose=False, **kwargs):
        super(TxCommander, self).__init__(**kwargs)
        self.default_map = mapping_dict is None
        self.mapping_dict = mapping_dict if mapping_dict is not None else keydefaultdict(lambda x: x)
        self.team = team
        self.verbose = verbose
        self.sender = unicast.UnicastSender(address=(ipaddr, port))

    def send(self, actions):
        string_list = []
        if len(actions) > 0:
            for a in actions:
                if not a:
                    continue
                vx, vy, va = a.speeds

                if self.default_map or a.uid in self.mapping_dict:
                    string_list.append(str(self.mapping_dict[a.uid]))
                else:
                    continue

                string_list.append(str((a.robot.angle or 0.0)))
                string_list.append(str((vx or 0.0)))
                string_list.append(str((vy or 0.0)))
                string_list.append(str((va or 0.0)))
                string_list.append(str((a.dribble or 0.0)))
                string_list.append(str((a.kick or 0.0)))
                string_list.append(str((a.chipkick or 0.0)))
                a.reset()

            packet = ' '.join(string_list)
            if packet:
                if self.verbose:
                    print packet
                self.sender.send(packet)

class SimCommander(Commander):

    def __init__(self, team, **kwargs):
        super(SimCommander, self).__init__(**kwargs)
        self.team = team
        #self.sender = grsim.grSimSender(('200.20.120.133', 20011))
        self.sender = grsim.grSimSender()

    def send(self, actions):
        packet = self.sender.new_packet()

        if len(actions) > 0:
            packet.commands.isteamyellow = self.team.is_yellow
            packet.commands.timestamp = time()
            for a in actions:
                if not a:
                    continue
                vx, vy, va = a.speeds
                c = packet.commands.robot_commands.add()
                c.id = a.uid
                chip_angle = 45
                c.kickspeedz = (a.chipkick or 0.0) * sin(chip_angle)
                if c.kickspeedz > 0:
                    # XXX FIXME this should be tested,
                    # we don't know at what angle we
                    # will be able to chipkick
                    c.kickspeedx = a.chipkick * cos(chip_angle)
                else:
                    c.kickspeedx = (a.kick or 0.0) * 5
                c.veltangent = vx
                c.velnormal = vy
                c.velangular = va * pi / 180
                c.spinner = (a.dribble or 0.0) > 0
                c.wheelsspeed = False

                # reset action values
                a.reset()

        #print packet
        self.sender.send_packet(packet)
<commit_msg>Fixing and simplifying omniwheels equation.<commit_after>from math import pi

#import socket
from ..communication.network import unicast
from ..communication import grsim
from collections import defaultdict
from time import time

from ..utils.mathutils import sin, cos
from ..utils.keydefaultdict import keydefaultdict

class Commander(object):
#class Commander(Process):

    def __init__(self):
        #super(Commander, self).__init__()
        #self._recvr, self._sendr = Pipe()
        #self.conn = None
        #self._exit = Event()
        pass

    #def start(self):
    #    super(Commander, self).start()
    #    self.conn = self._sendr

    #def run(self):
    #    self.conn = self._recvr
    #    while self._exit.is_set():
    #        self.send(self.conn.recv())
    #    self.conn.close()

    #def stop(self):
    #    self.conn.close()
    #    self._exit.set()

    def send(self, actions):
        raise NotImplemented

class Tx2012Commander(Commander):
    '''
    Sends commands as strings via unicast UDP to the C# transmitter process that will dispatch commands
    via RF to the robots. Since the robots' firmware IDs are not the same as the UIDs sent by the
    VisionUpdater, a dict mapping the UIDs to the firmware IDs should be provided. If it is not provided,
    we shall use the trivial mapping: x => x.

    This commander uses a transmission protocol compatible with the RoboIME MK-2012 architecture.
    This might be deprecated soon. Or not.
    '''
    def __init__(self, team, mapping_dict=None, kicking_power_dict=None, ipaddr='127.0.0.1', port=9050, verbose=False, **kwargs):
        super(Tx2012Commander, self).__init__(**kwargs)
        self.default_map = mapping_dict is None
        self.mapping_dict = mapping_dict if mapping_dict is not None else keydefaultdict(lambda x: x)
        self.kicking_power_dict = kicking_power_dict if kicking_power_dict is not None else defaultdict(lambda: 100)
        self.team = team
        self.verbose = verbose
        self.sender = unicast.UnicastSender(address=(ipaddr, port))
        #self.sock = so
        # FIXME: These values should be on the robot prototype to allow for mixed-chassis teams. NOT HERE!

        # RoboIME 2013
        self.wheel_angles = [
            -60.,
            -135.,
            +135.,
            +60.,
        ]

        # RoboIME 2012
        #self.wheel_angles = [
        #    -57.,
        #    +57.,
        #    +135.,
        #    -135.,
        #]
        #self.wheel_radius = 28.9
        #self.wheel_distance = 80.6

        #Values in meters.
        self.wheel_radius = .0289
        self.wheel_distance = .0806

    def omniwheel_speeds(self, vx, vy, va):
        return [(vy * cos(a) - vx * sin(a) + va * self.wheel_distance) / self.wheel_radius for a in self.wheel_angles]

    def send(self, actions):
        actions_dict = defaultdict(lambda:['0','0','0','0','0','0','0'])
        dirty = False
        if len(actions) > 0:
            for a in actions:
                string_list = []
                if not a:
                    continue
                dirty = True
                vx, vy, va = a.speeds
                # Convert va to angular speed.
                va = va * pi / 180
                if self.default_map or a.uid in self.mapping_dict:
                    pass
                    #string_list.append(str(self.mapping_dict[a.uid]))
                else:
                    continue

                string_list.extend([str(i) for i in self.omniwheel_speeds(vx, vy, -va)])
                string_list.append(str((a.dribble or 0.0)))
                if a.kick > 0 and self.kicking_power_dict[a.uid] > 0:
                    string_list.append(str((a.kick * 100 / self.kicking_power_dict[a.uid] or 0.0)))
                    #string_list.append(str((a.kick or 0.0)))
                    string_list.append('0')
                elif a.chipkick > 0 and self.kicking_power_dict[a.uid] > 0:
                    string_list.append('0')
                    string_list.append(str((a.chipkick * 100 /self.kicking_power_dict[a.uid] or 0.0)))
                    #string_list.append(str((a.chipkick or 0.0)))
                else:
                    string_list.append('0')
                    string_list.append('0')

                actions_dict[self.mapping_dict[a.uid]] = string_list
                a.reset()
            if dirty:
                string_list = []
                for i in xrange(6):
                    string_list.extend(actions_dict[i])

                packet = ' '.join(string_list)

                if packet:
                    if self.verbose:
                        print self.kicking_power_dict
                        print packet
                    self.sender.send(packet)

class TxCommander(Commander):
    '''
    Sends commands as strings via unicast UDP to the C# transmitter process that will dispatch commands
    via RF to the robots. Since the robots' firmware IDs are not the same as the UIDs sent by the
    VisionUpdater, a function mapping the UIDs to the firmware IDs should be provided. If it is not provided,
    we shall use the trivial mapping: x => x.

    This commander uses the transmission protocol for the RoboIME MK-2013 architecture.

    Main differences are:

    - Not passing wheel velocities directly, sending instead the triplet vx, vy, theta;
    - Chip kick.
    '''
    def __init__(self, team, mapping_dict=None, ipaddr='127.0.0.1', port=9050, verbose=False, **kwargs):
        super(TxCommander, self).__init__(**kwargs)
        self.default_map = mapping_dict is None
        self.mapping_dict = mapping_dict if mapping_dict is not None else keydefaultdict(lambda x: x)
        self.team = team
        self.verbose = verbose
        self.sender = unicast.UnicastSender(address=(ipaddr, port))

    def send(self, actions):
        string_list = []
        if len(actions) > 0:
            for a in actions:
                if not a:
                    continue
                vx, vy, va = a.speeds

                if self.default_map or a.uid in self.mapping_dict:
                    string_list.append(str(self.mapping_dict[a.uid]))
                else:
                    continue

                string_list.append(str((a.robot.angle or 0.0)))
                string_list.append(str((vx or 0.0)))
                string_list.append(str((vy or 0.0)))
                string_list.append(str((va or 0.0)))
                string_list.append(str((a.dribble or 0.0)))
                string_list.append(str((a.kick or 0.0)))
                string_list.append(str((a.chipkick or 0.0)))
                a.reset()

            packet = ' '.join(string_list)
            if packet:
                if self.verbose:
                    print packet
                self.sender.send(packet)

class SimCommander(Commander):

    def __init__(self, team, **kwargs):
        super(SimCommander, self).__init__(**kwargs)
        self.team = team
        #self.sender = grsim.grSimSender(('200.20.120.133', 20011))
        self.sender = grsim.grSimSender()

    def send(self, actions):
        packet = self.sender.new_packet()

        if len(actions) > 0:
            packet.commands.isteamyellow = self.team.is_yellow
            packet.commands.timestamp = time()
            for a in actions:
                if not a:
                    continue
                vx, vy, va = a.speeds
                c = packet.commands.robot_commands.add()
                c.id = a.uid
                chip_angle = 45
                c.kickspeedz = (a.chipkick or 0.0) * sin(chip_angle)
                if c.kickspeedz > 0:
                    # XXX FIXME this should be tested,
                    # we don't know at what angle we
                    # will be able to chipkick
                    c.kickspeedx = a.chipkick * cos(chip_angle)
                else:
                    c.kickspeedx = (a.kick or 0.0) * 5
                c.veltangent = vx
                c.velnormal = vy
                c.velangular = va * pi / 180
                c.spinner = (a.dribble or 0.0) > 0
                c.wheelsspeed = False

                # reset action values
                a.reset()

        #print packet
        self.sender.send_packet(packet)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import re
import attr

import databases.helpers.embl as embl
from databases.helpers.phylogeny import UnknownTaxonId

from databases.data import Reference

ONTOLOGIES = set([
    'ECO',
    'SO',
    'GO',
])

def source_qualifier_value(record, qualifier, pattern=r'^(.+)$'):
    source = embl.source_feature(record)
    return embl.qualifier_value(source, qualifier, pattern)

def chromosome(record):
    return source_qualifier_value(record, 'chromosome')

def primary_id(_):
    return ''

def sequence(record):
    return str(record.seq)

def as_reference(accession, ref):
    return Reference(
        accession=accession,
        authors=ref.authors,
        location=ref.journal,
        title=ref.title,
        pmid=int(ref.pubmed_id),
    )

def references(accession, record):
    return [as_reference(accession, ref) for ref in record.annotations['references']]

def rna_type(feature):
    if feature.type == 'ncRNA':
        return embl.qualifier_value(feature, 'ncRNA_class', r'^(.+)$')
    return feature.type

def mol_type(record):
    return source_qualifier_value(record, 'mol_type')

def product(feature):
    return embl.qualifier_string(feature, 'product', separator='; ')

def note_data(feature):
    data = {}
    text = []
    onts = set()
    for note in feature.qualifiers.get('note', []):
        for ontology in ONTOLOGIES:
            if note.startswith(ontology + ':'):
                onts.add(note)
                break
        else:
            text.append(note)

    if text:
        data['text'] = text
    if onts:
        data['ontology'] = sorted(onts)
    return data

def url(record):
    """
    Gets the standard url for this record.
    """
    return 'https://www.ebi.ac.uk/ena/data/view/Non-coding:%s' % accession(record)

def exons(record, feature):
    data = []
    chrom = chromosome(record)
    if not chrom:
        return embl.exons(feature)
    for exon in embl.exons(feature):
        data.append(attr.evolve(
            exon,
            chromosome=chrom
        ))
    return data

def accession(record):
    """
    Uses the record id as the accession for a feature.
    """
    return record.id

def is_composite(_):
    """
    Always returns 'N'
    """
    return 'N'

def function(record):
    return source_qualifier_value(record, 'function')

def allele(record):
    return source_qualifier_value(record, 'allele')

def anticodon(record, feature):
    value = embl.qualifier_value(feature, 'anticodon', r'^(.+)$')
    if not value:
        return None

    match = re.search('seq:([ACGUT]{3})', value)
    if match:
        return match.group(1).upper()

    gene = embl.gene(feature)
    if not gene:
        return value

    match = re.search(r'tRNA-\w+ \(([ACGU]{3})\)$', gene)
    if match:
        return match.group(1)

    match = re.search(r'tRNA-\w{3}-([ACGUT]{3})', gene)
    if match:
        return match.group(1)

    # match = re.search('pos:(.+),aa', value)
    # if match and match.group(1):
    #     location_string = match.group(1)
    #     consumer = _FeatureConsumer(1)
    #     consumer.feature_key('anticodon')
    #     consumer.location(location_string)
    #     feature = consumer._cur_feature

    #     # If the feature could not get parsed.
    #     if feature.location is None:
    #         return value

    #     from pprint import pprint
    #     print(feature)
    #     pprint(feature)
    #     pprint(record)

    #     # The feature may not refer to the current record, for some reason
    #         try:
    #         return str(feature.extract(record.seq))
    #     except ValueError:
    #         return value

    return value

def map(_):
    return None

def old_locus_tag(_):
    return None

def keywords(record):
    keys = [k for k in record.annotations['keywords'] if k]
    if not keys:
        return None
    return '; '.join(keys)

def parent_accession(record):
    return record.id.split('.', 1)[0]

def ordinal(_):
    return None

def organelle(record):
    return source_qualifier_value(record, 'organelle')

def operon(_):
    return None

def pseudogene(_):
    return None

def gene_synonyms(_):
    return []

def non_coding_id(_):
    return None

def taxid(record):
    try:
        return embl.taxid(record)
    except UnknownTaxonId:
        return 32644  # Unclassified sequence

def species(record):
    try:
        return embl.species(record)
    except UnknownTaxonId:
        return record.annotations.get('organism', None)

def common_name(record):
    try:
        return embl.common_name(record)
    except UnknownTaxonId:
        organism = record.annotations.get('organism', None)
        if organism and organism[-1] == ')':
            match = re.search(r'\(.\+\)$', organism)
            return match.group(1)
        return None

def lineage(record):
    try:
        return embl.lineage(record)
    except UnknownTaxonId:
        taxonomy = record.annotations.get('taxonomy', [])
        if taxonomy:
            return '; '.join(taxonomy)
        return None

def division(record):
    try:
        return embl.division(record)
    except UnknownTaxonId:
        return None
<commit_msg>Correctly match common names<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import re
import attr

import databases.helpers.embl as embl
from databases.helpers.phylogeny import UnknownTaxonId

from databases.data import Reference

ONTOLOGIES = set([
    'ECO',
    'SO',
    'GO',
])

def source_qualifier_value(record, qualifier, pattern=r'^(.+)$'):
    source = embl.source_feature(record)
    return embl.qualifier_value(source, qualifier, pattern)

def chromosome(record):
    return source_qualifier_value(record, 'chromosome')

def primary_id(_):
    return ''

def sequence(record):
    return str(record.seq)

def as_reference(accession, ref):
    return Reference(
        accession=accession,
        authors=ref.authors,
        location=ref.journal,
        title=ref.title,
        pmid=int(ref.pubmed_id),
    )

def references(accession, record):
    return [as_reference(accession, ref) for ref in record.annotations['references']]

def rna_type(feature):
    if feature.type == 'ncRNA':
        return embl.qualifier_value(feature, 'ncRNA_class', r'^(.+)$')
    return feature.type

def mol_type(record):
    return source_qualifier_value(record, 'mol_type')

def product(feature):
    return embl.qualifier_string(feature, 'product', separator='; ')

def note_data(feature):
    data = {}
    text = []
    onts = set()
    for note in feature.qualifiers.get('note', []):
        for ontology in ONTOLOGIES:
            if note.startswith(ontology + ':'):
                onts.add(note)
                break
        else:
            text.append(note)

    if text:
        data['text'] = text
    if onts:
        data['ontology'] = sorted(onts)
    return data

def url(record):
    """
    Gets the standard url for this record.
    """
    return 'https://www.ebi.ac.uk/ena/data/view/Non-coding:%s' % accession(record)

def exons(record, feature):
    data = []
    chrom = chromosome(record)
    if not chrom:
        return embl.exons(feature)
    for exon in embl.exons(feature):
        data.append(attr.evolve(
            exon,
            chromosome=chrom
        ))
    return data

def accession(record):
    """
    Uses the record id as the accession for a feature.
    """
    return record.id

def is_composite(_):
    """
    Always returns 'N'
    """
    return 'N'

def function(record):
    return source_qualifier_value(record, 'function')

def allele(record):
    return source_qualifier_value(record, 'allele')

def anticodon(record, feature):
    value = embl.qualifier_value(feature, 'anticodon', r'^(.+)$')
    if not value:
        return None

    match = re.search('seq:([ACGUT]{3})', value)
    if match:
        return match.group(1).upper()

    gene = embl.gene(feature)
    if not gene:
        return value

    match = re.search(r'tRNA-\w+ \(([ACGU]{3})\)$', gene)
    if match:
        return match.group(1)

    match = re.search(r'tRNA-\w{3}-([ACGUT]{3})', gene)
    if match:
        return match.group(1)

    # match = re.search('pos:(.+),aa', value)
    # if match and match.group(1):
    #     location_string = match.group(1)
    #     consumer = _FeatureConsumer(1)
    #     consumer.feature_key('anticodon')
    #     consumer.location(location_string)
    #     feature = consumer._cur_feature

    #     # If the feature could not get parsed.
    #     if feature.location is None:
    #         return value

    #     from pprint import pprint
    #     print(feature)
    #     pprint(feature)
    #     pprint(record)

    #     # The feature may not refer to the current record, for some reason
    #         try:
    #         return str(feature.extract(record.seq))
    #     except ValueError:
    #         return value

    return value

def map(_):
    return None

def old_locus_tag(_):
    return None

def keywords(record):
    keys = [k for k in record.annotations['keywords'] if k]
    if not keys:
        return None
    return '; '.join(keys)

def parent_accession(record):
    return record.id.split('.', 1)[0]

def ordinal(_):
    return None

def organelle(record):
    return source_qualifier_value(record, 'organelle')

def operon(_):
    return None

def pseudogene(_):
    return None

def gene_synonyms(_):
    return []

def non_coding_id(_):
    return None

def taxid(record):
    try:
        return embl.taxid(record)
    except UnknownTaxonId:
        return 32644  # Unclassified sequence

def species(record):
    try:
        return embl.species(record)
    except UnknownTaxonId:
        return record.annotations.get('organism', None)

def common_name(record):
    try:
        return embl.common_name(record)
    except UnknownTaxonId:
        organism = record.annotations.get('organism', None)
        if organism:
            match = re.search(r'\((.+)\)$', organism)
            if match:
                return match.group(1)
        return None

def lineage(record):
    try:
        return embl.lineage(record)
    except UnknownTaxonId:
        taxonomy = record.annotations.get('taxonomy', [])
        if taxonomy:
            return '; '.join(taxonomy)
        return None

def division(record):
    try:
        return embl.division(record)
    except UnknownTaxonId:
        return None
<|endoftext|>