<commit_before>import tempfile
from bs4 import BeautifulSoup as bs
from urllib2 import urlopen
import os, datetime, time, tempfile, codecs, unicodecsv, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
from flask.ext.uploads import UploadSet, configure_uploads, TEXT, patch_request_class, UploadNotAllowed
try:
    from docx import opendocx, getdocumenttext
except:
    pass
import databasic

logger = logging.getLogger(__name__)

ENCODING = 'utf-8'
TEMP_DIR = tempfile.gettempdir()

samples = []
docs = None

def init_uploads():
    global docs
    global TEMP_DIR
    logger.info("Uploads will be written to %s", TEMP_DIR)
    databasic.app.config['UPLOADED_DOCS_DEST'] = TEMP_DIR
    docs = UploadSet(name='docs', extensions=('txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls'))
    configure_uploads(databasic.app, (docs))
    patch_request_class(databasic.app, 100 * 1024 * 1024) # 100MB

def init_samples():
    global samples
    samples_config_file_path = os.path.join(databasic.get_config_dir(),'sample-data.json')
    samples = json.load(open(samples_config_file_path))
    if databasic.app.config.get(databasic.ENV_APP_MODE) == databasic.APP_MODE_DEV:
        # change the paths to absolute ones
        for sample in samples:
            sample['path'] = os.path.join(databasic.get_base_dir(),sample['source'])
        logger.info("Updated sample data with base dir: %s" % databasic.get_base_dir())
    else:
        # copy from server to local temp dir and change to abs paths (to temp dir files)
        url_base = databasic.app.config.get('SAMPLE_DATA_SERVER')
        for sample in samples:
            url = url_base+sample['source']
            text = urlopen(url).read()
            f = tempfile.NamedTemporaryFile(delete=False)
            f.write(text)
            f.close()
            sample['path'] = f.name
        logger.info("Downloaded sample data and saved to tempdir")
    for sample in samples:
        logger.debug("  Cached %s to %s", sample['source'], sample['path'])

def write_to_temp_file(text):
    file_path = _get_temp_file()
    file = codecs.open(file_path, 'w', ENCODING)
    file.write(text)
    file.close()
    return file_path

def write_to_csv(headers, rows, file_name_suffix=None, timestamp=True):
    file_path = _get_temp_file(file_name_suffix, timestamp)
    with open(file_path, 'w') as f:
        writer = unicodecsv.writer(f, encoding=ENCODING)
        writer.writerow(headers)
        for row in rows:
            writer.writerow(row)
    return file_path

def generate_csv(file_path):
    file_name = _get_file_name(file_path)
    if not os.path.isfile(file_path):
        return abort(400)
    def generate():
        with open(file_path, 'r') as f:
            reader = unicodecsv.reader(f, encoding=ENCODING)
            for row in reader:
                yield ','.join(row) + '\n'
    return Response(generate(), headers={'Content-Disposition':'attachment;filename='+file_name},mimetype='text/csv')

def convert_to_txt(file_path):
    words = None
    if not os.path.exists(file_path):
        logger.error("missing file %s", file_path)
    ext = _get_extension(file_path)
    if ext == '.txt':
        logger.debug("loading txt file")
        with codecs.open(file_path, 'r', ENCODING) as myfile:
            words = myfile.read()
    elif ext == '.docx':
        logger.debug("loading docx file")
        words = _docx_to_txt(file_path)
    elif ext == '.rtf':
        logger.debug("loading rtf file")
        doc = Rtf15Reader.read(open(file_path))
        words = PlaintextWriter.write(doc).getvalue()
    logger.debug("loaded %d chars" % len(words))
    return words

def convert_to_csv(file_path):
    ext = _get_extension(file_path)
    if ext == '.csv':
        return [file_path]
    elif ext == '.xlsx' or ext == '.xls':
        wb = xlrd.open_workbook(file_path)
        files = []
        for i in range(wb.nsheets):
            files.append(_open_sheet(wb, i))
        return files
    print ext + ' could not be converted to csv'
    return [file_path]

def open_doc(doc):
    try:
        file_name = docs.save(doc)
        file_path = os.path.join(TEMP_DIR, file_name)
        return file_path
    except UploadNotAllowed:
        print "supported filetypes: txt, docx, rtf, csv, xlsx, xls, love"

def open_docs(docs):
    file_paths = []
    for doc in docs:
        file_paths.append(open_doc(doc))
    return file_paths

def delete_files(file_paths):
    for f in file_paths:
        delete_file(f)

def delete_file(file_path):
    os.remove(file_path)

def open_workbook(book):
    file_paths = []
    for i, worksheet in enumerate(book.worksheets()):
        file_path = _get_temp_file('-' + worksheet.title + '.csv')
        with open(file_path, 'wb') as f:
            writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
            writer.writerows(worksheet.get_all_values())
        file_paths.append(file_path)
    return file_paths

def get_samples(tool_id):
    choices = []
    texts = []
    for text in samples:
        if tool_id in text['modules']:
            if(os.path.exists(text['path'])):
                texts.append((text['source'], text['title']))
            else:
                logger.error("%s: file for %s doesn't exist at %s",tool_id, text['source'], text['path'])
    choices = texts
    return choices

def get_sample_title(source):
    for text in samples:
        if source in text['source']:
            return text['title']
    return source

def get_sample_path(source):
    for text in samples:
        if source in text['source']:
            return text['path']
    return source

def get_file_names(file_paths):
    file_names = []
    for f in file_paths:
        file_names.append(_get_file_name(f))
    return file_names

def generate_filename(ext, suffix, *args):
    files = '-'.join(args) + '-' if len(args) > 0 else ''
    suffix = suffix + '-' if suffix is not None and suffix is not '' else ''
    suffix = suffix.replace(' ', '-')
    ext = ext[1:] if '.' in ext[0] else ext
    return files + suffix + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '.' + ext

def download_webpage(url):
    soup = bs(urlopen(url),"lxml")
    if soup.p is not None:
        soup.p.encode(ENCODING)
    for script in soup(['script', 'style']):
        script.extract()
    return {'title': soup.title.string, 'text': soup.get_text()}

def _open_sheet(workbook, index):
    sh = workbook.sheet_by_index(index)
    name = workbook.sheet_names()[index]
    new_file = _get_temp_file('-' + name + '.csv')
    with open(new_file, 'wb') as f:
        writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
        for row in xrange(sh.nrows):
            writer.writerow(sh.row_values(row))
    return new_file

def _get_file_name(file_path):
    return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None, timestamp=True):
    file_name = ''
    if timestamp:
        file_name = time.strftime("%Y%m%d-%H%M%S")
    if file_name_suffix is not None:
        file_name += file_name_suffix
    file_path = os.path.join(TEMP_DIR, file_name)
    logger.debug("new tempfile at %s", file_path)
    return file_path

def _get_extension(file_path):
    return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
    # http://davidmburke.com/2014/02/04/python-convert-documents-doc-docx-odt-pdf-to-plain-text-without-libreoffice/
    document = opendocx(file_path)
    paratextlist = getdocumenttext(document)
    newparatextlist = []
    for paratext in paratextlist:
        newparatextlist.append(paratext.encode(ENCODING))
    return '\n\n'.join(newparatextlist)
<commit_msg>more logging<commit_after>import tempfile
from bs4 import BeautifulSoup as bs
from urllib2 import urlopen
import os, datetime, time, tempfile, codecs, unicodecsv, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
from flask.ext.uploads import UploadSet, configure_uploads, TEXT, patch_request_class, UploadNotAllowed
try:
    from docx import opendocx, getdocumenttext
except:
    pass
import databasic

logger = logging.getLogger(__name__)

ENCODING = 'utf-8'
TEMP_DIR = tempfile.gettempdir()

samples = []
docs = None

def init_uploads():
    global docs
    global TEMP_DIR
    logger.info("Uploads will be written to %s", TEMP_DIR)
    databasic.app.config['UPLOADED_DOCS_DEST'] = TEMP_DIR
    docs = UploadSet(name='docs', extensions=('txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls'))
    configure_uploads(databasic.app, (docs))
    patch_request_class(databasic.app, 100 * 1024 * 1024) # 100MB

def init_samples():
    global samples
    samples_config_file_path = os.path.join(databasic.get_config_dir(),'sample-data.json')
    samples = json.load(open(samples_config_file_path))
    if databasic.app.config.get(databasic.ENV_APP_MODE) == databasic.APP_MODE_DEV:
        # change the paths to absolute ones
        for sample in samples:
            sample['path'] = os.path.join(databasic.get_base_dir(),sample['source'])
        logger.info("Updated sample data with base dir: %s" % databasic.get_base_dir())
    else:
        # copy from server to local temp dir and change to abs paths (to temp dir files)
        url_base = databasic.app.config.get('SAMPLE_DATA_SERVER')
        for sample in samples:
            url = url_base+sample['source']
            text = urlopen(url).read()
            f = tempfile.NamedTemporaryFile(delete=False)
            f.write(text)
            f.close()
            sample['path'] = f.name
        logger.info("Downloaded sample data and saved to tempdir")
    for sample in samples:
        file_size = os.stat(sample['path']).st_size
        logger.debug("  Cached %d bytes of %s to %s", file_size, sample['source'], sample['path'])

def write_to_temp_file(text):
    file_path = _get_temp_file()
    file = codecs.open(file_path, 'w', ENCODING)
    file.write(text)
    file.close()
    return file_path

def write_to_csv(headers, rows, file_name_suffix=None, timestamp=True):
    file_path = _get_temp_file(file_name_suffix, timestamp)
    with open(file_path, 'w') as f:
        writer = unicodecsv.writer(f, encoding=ENCODING)
        writer.writerow(headers)
        for row in rows:
            writer.writerow(row)
    return file_path

def generate_csv(file_path):
    file_name = _get_file_name(file_path)
    if not os.path.isfile(file_path):
        return abort(400)
    def generate():
        with open(file_path, 'r') as f:
            reader = unicodecsv.reader(f, encoding=ENCODING)
            for row in reader:
                yield ','.join(row) + '\n'
    return Response(generate(), headers={'Content-Disposition':'attachment;filename='+file_name},mimetype='text/csv')

def convert_to_txt(file_path):
    words = None
    if not os.path.exists(file_path):
        logger.error("missing file %s", file_path)
    file_size = os.stat(file_path).st_size
    logger.debug("convert_to_txt: %d bytes at %s",file_size, file_path)
    ext = _get_extension(file_path)
    if ext == '.txt':
        logger.debug("loading txt file")
        with codecs.open(file_path, 'r', ENCODING) as myfile:
            words = myfile.read()
    elif ext == '.docx':
        logger.debug("loading docx file")
        words = _docx_to_txt(file_path)
    elif ext == '.rtf':
        logger.debug("loading rtf file")
        doc = Rtf15Reader.read(open(file_path))
        words = PlaintextWriter.write(doc).getvalue()
    logger.debug("loaded %d chars" % len(words))
    return words

def convert_to_csv(file_path):
    ext = _get_extension(file_path)
    if ext == '.csv':
        return [file_path]
    elif ext == '.xlsx' or ext == '.xls':
        wb = xlrd.open_workbook(file_path)
        files = []
        for i in range(wb.nsheets):
            files.append(_open_sheet(wb, i))
        return files
    print ext + ' could not be converted to csv'
    return [file_path]

def open_doc(doc):
    try:
        file_name = docs.save(doc)
        file_path = os.path.join(TEMP_DIR, file_name)
        return file_path
    except UploadNotAllowed:
        print "supported filetypes: txt, docx, rtf, csv, xlsx, xls, love"

def open_docs(docs):
    file_paths = []
    for doc in docs:
        file_paths.append(open_doc(doc))
    return file_paths

def delete_files(file_paths):
    for f in file_paths:
        delete_file(f)

def delete_file(file_path):
    os.remove(file_path)

def open_workbook(book):
    file_paths = []
    for i, worksheet in enumerate(book.worksheets()):
        file_path = _get_temp_file('-' + worksheet.title + '.csv')
        with open(file_path, 'wb') as f:
            writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
            writer.writerows(worksheet.get_all_values())
        file_paths.append(file_path)
    return file_paths

def get_samples(tool_id):
    choices = []
    texts = []
    for text in samples:
        if tool_id in text['modules']:
            if(os.path.exists(text['path'])):
                texts.append((text['source'], text['title']))
            else:
                logger.error("%s: file for %s doesn't exist at %s",tool_id, text['source'], text['path'])
    choices = texts
    return choices

def get_sample_title(source):
    for text in samples:
        if source in text['source']:
            return text['title']
    return source

def get_sample_path(source):
    for text in samples:
        if source in text['source']:
            return text['path']
    return source

def get_file_names(file_paths):
    file_names = []
    for f in file_paths:
        file_names.append(_get_file_name(f))
    return file_names

def generate_filename(ext, suffix, *args):
    files = '-'.join(args) + '-' if len(args) > 0 else ''
    suffix = suffix + '-' if suffix is not None and suffix is not '' else ''
    suffix = suffix.replace(' ', '-')
    ext = ext[1:] if '.' in ext[0] else ext
    return files + suffix + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '.' + ext

def download_webpage(url):
    soup = bs(urlopen(url),"lxml")
    if soup.p is not None:
        soup.p.encode(ENCODING)
    for script in soup(['script', 'style']):
        script.extract()
    return {'title': soup.title.string, 'text': soup.get_text()}

def _open_sheet(workbook, index):
    sh = workbook.sheet_by_index(index)
    name = workbook.sheet_names()[index]
    new_file = _get_temp_file('-' + name + '.csv')
    with open(new_file, 'wb') as f:
        writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
        for row in xrange(sh.nrows):
            writer.writerow(sh.row_values(row))
    return new_file

def _get_file_name(file_path):
    return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None, timestamp=True):
    file_name = ''
    if timestamp:
        file_name = time.strftime("%Y%m%d-%H%M%S")
    if file_name_suffix is not None:
        file_name += file_name_suffix
    file_path = os.path.join(TEMP_DIR, file_name)
    logger.debug("new tempfile at %s", file_path)
    return file_path

def _get_extension(file_path):
    return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
    # http://davidmburke.com/2014/02/04/python-convert-documents-doc-docx-odt-pdf-to-plain-text-without-libreoffice/
    document = opendocx(file_path)
    paratextlist = getdocumenttext(document)
    newparatextlist = []
    for paratext in paratextlist:
        newparatextlist.append(paratext.encode(ENCODING))
    return '\n\n'.join(newparatextlist)
<|endoftext|>


<commit_before>from django import forms

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Field, Hidden, Layout, Submit
from crispy_forms.bootstrap import PrependedText

from .models import Article, Contributor

class DoiForm(forms.Form):

    doi = forms.CharField(required=False, label='')

    def __init__(self, *args, **kwargs):
        super(DoiForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_method = 'post'
        self.helper.form_action = 'doiref'
        self.helper.layout = Layout(
            PrependedText('doi', 'doi:'),
        )

class ArticleForm(forms.ModelForm):

    def __init__(self, *args, **kwargs):
        super(ArticleForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.form_id = 'id-creation'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-8'
        self.helper.attrs = {'enctype': 'multipart/form-data'}
        self.helper.add_input(Submit('submit', 'Save'))

        self.helper.layout = Layout(
            Hidden('doi', ''),
            Hidden('site_owner', '{{ user }}'),
            'title',
            'contributors',
            'code_data_abstract',
            'code_archive_file',
            'data_archive_file',
            'notes_for_staff',
            'primary_research_field',
            'secondary_research_field',
            'tags',
            Field('authorship', type='hidden'),
            Field('related_urls', type='hidden'),
            Field('paper_abstract', type='hidden'),
        )

    def save(self, **kwargs):
        kwargs.update({'commit': False})
        article = super(ArticleForm, self).save(**kwargs)
        article.save()
        for user in self.cleaned_data.get('contributors'):
            Contributor.objects.create(article=article, user=user)
        self.save_m2m()

    class Meta:
        model = Article
        # TODO I wonder if exclude is redundant since I don't list these in the layout?
        exclude = ['legacy_id', 'status_changed', ]
        widgets = {
            # TODO this is not DRY but I'm not sure how to include placeholder attr in the crispy layout
            'title': forms.TextInput(attrs={'placeholder': 'Title your compendium.'}),
            'code_data_abstract': forms.Textarea(attrs={'placeholder': 'does not need to match paper abstract'}),
        }
<commit_msg>checked in save_m2m hack for review<commit_after>from django import forms

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Field, Hidden, Layout, Submit
from crispy_forms.bootstrap import PrependedText

from .models import Article, Contributor

class DoiForm(forms.Form):

    doi = forms.CharField(required=False, label='')

    def __init__(self, *args, **kwargs):
        super(DoiForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper()
        self.helper.form_method = 'post'
        self.helper.form_action = 'doiref'
        self.helper.layout = Layout(
            PrependedText('doi', 'doi:'),
        )

class ArticleForm(forms.ModelForm):

    def __init__(self, *args, **kwargs):
        super(ArticleForm, self).__init__(*args, **kwargs)
        self.helper = FormHelper(self)
        self.helper.form_class = 'form-horizontal'
        self.helper.form_id = 'id-creation'
        self.helper.label_class = 'col-lg-2'
        self.helper.field_class = 'col-lg-8'
        self.helper.attrs = {'enctype': 'multipart/form-data'}
        self.helper.add_input(Submit('submit', 'Save'))

        self.helper.layout = Layout(
            Hidden('doi', ''),
            Hidden('site_owner', '{{ user }}'),
            'title',
            'contributors',
            'code_data_abstract',
            'code_archive_file',
            'data_archive_file',
            'notes_for_staff',
            'primary_research_field',
            'secondary_research_field',
            'tags',
            Field('authorship', type='hidden'),
            Field('related_urls', type='hidden'),
            Field('paper_abstract', type='hidden'),
        )

    def save(self, **kwargs):
        kwargs.update({'commit': False})
        article = super(ArticleForm, self).save(**kwargs)
        article.save()
        for user in self.cleaned_data.get('contributors'):
            Contributor.objects.create(article=article, user=user)
        # fake excluding our m2m intermediated relationship before
        # calling save_m2m
        self.exclude.append('contributers');
        self.save_m2m();
        # return exclude back to normal
        self.exclude = [x for x in self.exclude if x != 'contributors']

    class Meta:
        model = Article
        # TODO I wonder if exclude is redundant since I don't list these in the layout?
        exclude = ['legacy_id', 'status_changed', ]
        widgets = {
            # TODO this is not DRY but I'm not sure how to include placeholder attr in the crispy layout
            'title': forms.TextInput(attrs={'placeholder': 'Title your compendium.'}),
            'code_data_abstract': forms.Textarea(attrs={'placeholder': 'does not need to match paper abstract'}),
        }
<|endoftext|>


<commit_before>from ..application import app
import os, time, tempfile, codecs, unicodecsv, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
from flask.ext.uploads import UploadSet, configure_uploads, TEXT, patch_request_class, UploadNotAllowed
from docx import opendocx, getdocumenttext

ENCODING = 'utf-8'

# setup file uploading
TEMP_DIR = tempfile.gettempdir()
app.config['UPLOADED_DOCS_DEST'] = TEMP_DIR
docs = UploadSet(name='docs', extensions=('txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls'))
configure_uploads(app, (docs))
patch_request_class(app, 4 * 1024 * 1024) # 4MB

basedir = os.path.dirname(os.path.abspath(__file__))
sample_data_dir_path = os.path.join(basedir,'../','../','sample-data')
logging.error("Loading sameple data from %s" % sample_data_dir_path)
sample_data_config_path = os.path.join(basedir,'../','../','config','sample-data.json')
logging.error("Loading sameple data config from %s" % sample_data_config_path)
if os.path.isdir(sample_data_dir_path) and os.path.exists(sample_data_config_path):
	samples = json.load(open(sample_data_config_path))

def write_to_temp_file(text):
	file_path = _get_temp_file()
	file = codecs.open(file_path, 'w', ENCODING)
	file.write(text)
	file.close()
	return file_path

def write_to_csv(headers, rows, file_name_suffix=None):
	file_path = _get_temp_file(file_name_suffix)
	with open(file_path, 'w') as f:
		writer = unicodecsv.writer(f, encoding=ENCODING)
		writer.writerow(headers)
		for row in rows:
			writer.writerow(row)
	return _get_file_name(file_path)

def generate_csv(file_name):
	file_path = os.path.join(TEMP_DIR, file_name)
	if not os.path.isfile(file_path):
		return abort(400)
	def generate():
		with open(file_path, 'r') as f:
			reader = unicodecsv.reader(f, encoding=ENCODING)
			for row in reader:
				yield ','.join(row) + '\n'
	return Response(generate(), headers={'Content-Disposition':'attachment;filename='+file_name},mimetype='text/csv')

def convert_to_txt(file_path):
	words = None
	ext = _get_extension(file_path)
	if ext == '.txt':
		with codecs.open(file_path, 'r', ENCODING) as myfile:
			words = myfile.read()
	elif ext == '.docx':
		words = _docx_to_txt(file_path)
	elif ext == '.rtf':
		doc = Rtf15Reader.read(open(file_path))
		words = PlaintextWriter.write(doc).getvalue()
	return words

def convert_to_csv(file_path):
	ext = _get_extension(file_path)
	if ext == '.csv':
		return file_path
	elif ext == '.xlsx' or ext == '.xls':
		wb = xlrd.open_workbook(file_path)
		sh = wb.sheet_by_index(0)
		new_file = _get_temp_file('-worksheet.csv')
		with open(new_file, 'wb') as f:
			writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
			for row in xrange(sh.nrows):
				writer.writerow(sh.row_values(row))
		return new_file
	print ext + ' could not be converted to csv'
	return file_path

def open_doc(doc):
	try:
		file_name = docs.save(doc)
		file_path = os.path.join(TEMP_DIR, file_name)
		return file_path
	except UploadNotAllowed:
		print "supported filetypes: txt, docx, rtf, csv, xlsx, xls, love"

def open_docs(docs):
	file_paths = []
	for doc in docs:
		file_paths.append(open_doc(doc))
	return file_paths

def delete_file(file_path):
	os.remove(file_path)

def open_sheet(sheet):
	first = ''
	for i, worksheet in enumerate(sheet.worksheets()):
		file_path = _get_temp_file('-worksheet' + str(i) + '.csv')
		if i == 0:
			first = file_path
		with open(file_path, 'wb') as f:
			writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
			writer.writerows(worksheet.get_all_values())
	return first

def get_samples(tool_id):
	choices = []
	texts = []
	for text in samples:
		if tool_id in text['modules']:
			texts.append((text['source'], text['title']))
	choices = texts
	return choices

def get_sample_title(path):
	for text in samples:
		if path in text['source']:
			return text['title']
	return path

def get_file_names(file_paths):
	file_names = []
	for f in file_paths:
		file_names.append(_get_file_name(f))
	return file_names

def _get_file_name(file_path):
	return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None):
	file_name = time.strftime("%Y%m%d-%H%M%S")
	if file_name_suffix is not None:
		file_name += file_name_suffix
	return os.path.join(TEMP_DIR, file_name)

def _get_extension(file_path):
	return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
	# http://davidmburke.com/2014/02/04/python-convert-documents-doc-docx-odt-pdf-to-plain-text-without-libreoffice/
	document = opendocx(file_path)
	paratextlist = getdocumenttext(document)
	newparatextlist = []
	for paratext in paratextlist:
		newparatextlist.append(paratext.encode(ENCODING))
	return '\n\n'.join(newparatextlist)
<commit_msg>max upload size 10MB<commit_after>from ..application import app
import os, time, tempfile, codecs, unicodecsv, json, xlrd, logging
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
from flask import Response, abort
from flask.ext.uploads import UploadSet, configure_uploads, TEXT, patch_request_class, UploadNotAllowed
from docx import opendocx, getdocumenttext

ENCODING = 'utf-8'

# setup file uploading
TEMP_DIR = tempfile.gettempdir()
app.config['UPLOADED_DOCS_DEST'] = TEMP_DIR
docs = UploadSet(name='docs', extensions=('txt', 'docx', 'rtf', 'csv', 'xlsx', 'xls'))
configure_uploads(app, (docs))
patch_request_class(app, 10 * 1024 * 1024) # 4MB

basedir = os.path.dirname(os.path.abspath(__file__))
sample_data_dir_path = os.path.join(basedir,'../','../','sample-data')
logging.error("Loading sameple data from %s" % sample_data_dir_path)
sample_data_config_path = os.path.join(basedir,'../','../','config','sample-data.json')
logging.error("Loading sameple data config from %s" % sample_data_config_path)
if os.path.isdir(sample_data_dir_path) and os.path.exists(sample_data_config_path):
	samples = json.load(open(sample_data_config_path))

def write_to_temp_file(text):
	file_path = _get_temp_file()
	file = codecs.open(file_path, 'w', ENCODING)
	file.write(text)
	file.close()
	return file_path

def write_to_csv(headers, rows, file_name_suffix=None):
	file_path = _get_temp_file(file_name_suffix)
	with open(file_path, 'w') as f:
		writer = unicodecsv.writer(f, encoding=ENCODING)
		writer.writerow(headers)
		for row in rows:
			writer.writerow(row)
	return _get_file_name(file_path)

def generate_csv(file_name):
	file_path = os.path.join(TEMP_DIR, file_name)
	if not os.path.isfile(file_path):
		return abort(400)
	def generate():
		with open(file_path, 'r') as f:
			reader = unicodecsv.reader(f, encoding=ENCODING)
			for row in reader:
				yield ','.join(row) + '\n'
	return Response(generate(), headers={'Content-Disposition':'attachment;filename='+file_name},mimetype='text/csv')

def convert_to_txt(file_path):
	words = None
	ext = _get_extension(file_path)
	if ext == '.txt':
		with codecs.open(file_path, 'r', ENCODING) as myfile:
			words = myfile.read()
	elif ext == '.docx':
		words = _docx_to_txt(file_path)
	elif ext == '.rtf':
		doc = Rtf15Reader.read(open(file_path))
		words = PlaintextWriter.write(doc).getvalue()
	return words

def convert_to_csv(file_path):
	ext = _get_extension(file_path)
	if ext == '.csv':
		return file_path
	elif ext == '.xlsx' or ext == '.xls':
		wb = xlrd.open_workbook(file_path)
		sh = wb.sheet_by_index(0)
		new_file = _get_temp_file('-worksheet.csv')
		with open(new_file, 'wb') as f:
			writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
			for row in xrange(sh.nrows):
				writer.writerow(sh.row_values(row))
		return new_file
	print ext + ' could not be converted to csv'
	return file_path

def open_doc(doc):
	try:
		file_name = docs.save(doc)
		file_path = os.path.join(TEMP_DIR, file_name)
		return file_path
	except UploadNotAllowed:
		print "supported filetypes: txt, docx, rtf, csv, xlsx, xls, love"

def open_docs(docs):
	file_paths = []
	for doc in docs:
		file_paths.append(open_doc(doc))
	return file_paths

def delete_file(file_path):
	os.remove(file_path)

def open_sheet(sheet):
	first = ''
	for i, worksheet in enumerate(sheet.worksheets()):
		file_path = _get_temp_file('-worksheet' + str(i) + '.csv')
		if i == 0:
			first = file_path
		with open(file_path, 'wb') as f:
			writer = unicodecsv.writer(f, encoding=ENCODING, delimiter=str(u';'), quotechar=str(u'"'))
			writer.writerows(worksheet.get_all_values())
	return first

def get_samples(tool_id):
	choices = []
	texts = []
	for text in samples:
		if tool_id in text['modules']:
			texts.append((text['source'], text['title']))
	choices = texts
	return choices

def get_sample_title(path):
	for text in samples:
		if path in text['source']:
			return text['title']
	return path

def get_file_names(file_paths):
	file_names = []
	for f in file_paths:
		file_names.append(_get_file_name(f))
	return file_names

def _get_file_name(file_path):
	return os.path.split(file_path)[1]

def _get_temp_file(file_name_suffix=None):
	file_name = time.strftime("%Y%m%d-%H%M%S")
	if file_name_suffix is not None:
		file_name += file_name_suffix
	return os.path.join(TEMP_DIR, file_name)

def _get_extension(file_path):
	return os.path.splitext(file_path)[1]

def _docx_to_txt(file_path):
	# http://davidmburke.com/2014/02/04/python-convert-documents-doc-docx-odt-pdf-to-plain-text-without-libreoffice/
	document = opendocx(file_path)
	paratextlist = getdocumenttext(document)
	newparatextlist = []
	for paratext in paratextlist:
		newparatextlist.append(paratext.encode(ENCODING))
	return '\n\n'.join(newparatextlist)
<|endoftext|>


<commit_before>import os
import random
import logging
from collections import OrderedDict
from databasic import mongo, get_base_dir
from databasic.forms import WordCounterPaste, WordCounterUpload, WordCounterSample, WordCounterLink
from databasic.logic import wordhandler, filehandler
from flask import Blueprint, render_template, request, redirect, g, abort, send_from_directory
from flask_babel import lazy_gettext as _

from databasic import NLTK_STOPWORDS_BY_LANGUAGE

mod = Blueprint('wordcounter', __name__,
                url_prefix='/<lang_code>/wordcounter',
                template_folder='../templates/wordcounter')

logger = logging.getLogger(__name__)

@mod.route('/', methods=('GET', 'POST'))
def index():

    words = None

    forms = OrderedDict()
    forms['sample'] = WordCounterSample(g.current_lang)
    forms['paste'] = WordCounterPaste('I am Sam\nSam I am\nThat Sam-I-am!\nThat Sam-I-am!\nI do not like that Sam-I-am!\nDo you like \ngreen eggs and ham?\nI do not like them, Sam-I-am.\nI do not like\ngreen eggs and ham.\nWould you like them \nhere or there?\nI would not like them\nhere or there.\nI would not like them anywhere.')
    forms['upload'] = WordCounterUpload()
    forms['link'] = WordCounterLink()

    if request.method == 'POST':
        ignore_case = True
        ignore_stopwords = True

        btn_value = request.form['btn']
        sample_id = ''
        extras_to_save = {}

        if btn_value == 'paste':
            words = forms['paste'].data['area']
            ignore_case = forms[btn_value].data['ignore_case_paste']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_paste']
            logger.debug("New from paste: %d chars", len(words) )
            title = _('your text')
        elif btn_value == 'upload':
            upload_file = forms['upload'].data['upload']
            words = process_upload(upload_file)
            ignore_case = forms[btn_value].data['ignore_case_upload']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_upload']
            title = upload_file.filename
            logger.debug("New from upload: %s", title)
        elif btn_value == 'sample':
            sample_source = forms['sample'].data['sample']
            samplename = filehandler.get_sample_title(sample_source)
            title = samplename
            ignore_case = forms[btn_value].data['ignore_case_sample']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_sample']
            sample_id = title+str(ignore_case)+str(ignore_stopwords)
            existing_doc_id = mongo.results_for_sample('wordcounter', sample_id)
            if existing_doc_id is not None:
                logger.debug("Existing from sample: %s", sample_source)
                return redirect(request.url + 'results/' + existing_doc_id)
            logger.info("New from sample: %s", sample_source)
            sample_path = filehandler.get_sample_path(sample_source)
            logger.debug("  loading from %s", sample_path)
            words = filehandler.convert_to_txt(sample_path)
            extras_to_save = filehandler.get_sample(sample_source)
        elif btn_value == 'link':
            url = forms['link'].data['link']
            # TODO: should actually accept https
            if 'https://' in url:
                url = url.replace('https', 'http')
            elif not 'http://' in url:
                url = 'http://' + url
            logger.debug("New from link: %s", url)
            content = filehandler.download_webpage(url)
            words = content['text']
            ignore_case = forms[btn_value].data['ignore_case_link']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_link']
            title = _(content['title'])

        if words is not None:
            logger.debug("  about to process words")
            counts = _process_words(words, ignore_case, ignore_stopwords, btn_value=='sample')
            logger.debug("  finished counts, about to save")
            doc_id = mongo.save_words('wordcounter', counts, ignore_case, ignore_stopwords, title, sample_id, btn_value, extras_to_save)
            logger.debug("  saved")
            return redirect(request.url + 'results/' + doc_id + '?submit=true')

    return render_template('wordcounter.html', forms=list(forms.items()), tool_name='wordcounter', max_file_size_in_mb = g.max_file_size_mb)

@mod.route('/results/<doc_id>')
def results_for_doc(doc_id):
    results = {}
    remaining_days = None

    try:
        doc = mongo.find_document('wordcounter', doc_id)
        if doc['sample_id'] == '':
            remaining_days = mongo.get_remaining_days('wordcounter', doc_id)
    except:
        logger.warning("Unable to find doc '%s'", doc_id)
        return render_template('no_results.html', tool_name='wordcounter')

    counts = doc.get('counts')

    # only render the top 40 results on the page (the csv contains all results)
    results['unique_words'] = counts['unique_words'][:40]
    results['bigrams'] = counts['bigrams'][:40]
    results['trigrams'] = counts['trigrams'][:40]

    max_index = min(20, len(results['unique_words']))
    min_index = max(0, max_index-5)
    random_unpopular_word = ['', '']
    top_word = ''
    word_in_bigrams_count = 0
    word_in_trigrams_count = 0

    if len(results['unique_words']) > 0:
        random_unpopular_word = results['unique_words'][random.randrange(min_index, max_index+1)]\
            if len(results['unique_words']) > 1 else results['unique_words'][0]

        '''
        Find the most popular word that is also present in bigrams and trigrams.
        If none can be found, just get the most popular word.
        '''

        if results['unique_words'] and results['bigrams'] and results['trigrams']:
            for word in results['unique_words']:
                top_word = word[0]
                word_in_bigrams_count = 0
                word_in_trigrams_count = 0
                for b in results['bigrams']:
                    if top_word in b[0]:
                        word_in_bigrams_count += 1
                for t in results['trigrams']:
                    if top_word in t[0]:
                        word_in_trigrams_count += 1
                if word_in_bigrams_count > 0 and word_in_trigrams_count > 0:
                    break

        if word_in_bigrams_count == 0 and word_in_trigrams_count == 0:
            top_word = results['unique_words'][0][0]

    whatnext = {}
    whatnext['top_word'] = top_word
    whatnext['word_in_bigrams_count'] = word_in_bigrams_count
    whatnext['word_in_trigrams_count'] = word_in_trigrams_count
    whatnext['random_unpopular_word'] = random_unpopular_word[0]
    whatnext['random_unpopular_word_count'] = random_unpopular_word[1]
    biography = doc['biography'] if 'biography' in doc else None

    return render_template('wordcounter/results.html',
                           results=results,
                           whatnext=whatnext,
                           tool_name='wordcounter',
                           title=doc['title'],
                           doc_id=doc_id,
                           source=doc['source'],
                           remaining_days=remaining_days,
                           total_words=counts['total_word_count'],
                           biography=biography)

@mod.route('/results/<doc_id>/download/<analysis_type>.csv')
def download_csv(doc_id, analysis_type):
    logger.debug("Download %s", analysis_type)
    if analysis_type not in ['words','bigrams','trigrams']:
        logger.warning("Requested unknown csv type: %s", analysis_type)
        abort(400)
    try:
        doc = mongo.find_document('wordcounter', doc_id)
    except:
        logger.warning("Unable to find doc '%s'", doc_id)
        abort(400)
    file_path = create_csv_file(doc.get('counts'),analysis_type)
    logger.debug('  created %s csv to download at %s', analysis_type, file_path)
    if file_path is None:
        abort(500)
    return filehandler.generate_csv(file_path)

@mod.route('/wordcounter-activity-guide.pdf')
def download_activity_guide():
    filename = "WordCounter Activity Guide.pdf"
    dir_path = os.path.join(get_base_dir(),'databasic','static','files','activity-guides',g.current_lang)
    logger.debug("download activity guide from %s/%s", dir_path, filename)
    return send_from_directory(directory=dir_path, filename=filename)

@mod.route('/run-activity')
def run_activity():
    return render_template('wordcounter/run-activity.html')

def process_upload(doc):
    file_path = filehandler.open_doc(doc)
    file_size = os.stat(file_path).st_size  # because browser might not have sent content_length
    logger.debug("Upload: %d bytes", file_size)
    words = filehandler.convert_to_txt(file_path)
    filehandler.delete_file(file_path)
    return words

def _process_words(words, ignore_case, ignore_stopwords, is_sample):
    stopwords_language = NLTK_STOPWORDS_BY_LANGUAGE[g.current_lang]
    counts = wordhandler.get_word_counts(
        words,
        ignore_case,
        ignore_stopwords,
        stopwords_language)
    return counts

def create_csv_file(counts, analysis_type):
    try:
        if analysis_type == 'words':
            return filehandler.write_to_csv(['word', 'frequency'], counts['unique_words'], '-word-counts.csv')
        elif analysis_type == 'bigrams':
            bigrams = []
            for w in counts['bigrams']:
                freq = w[1]
                phrase = " ".join(w[0])
                bigrams.append([phrase, freq])
            return filehandler.write_to_csv(['bigram phrase', 'frequency'], bigrams, '-bigram-counts.csv')
        elif analysis_type == 'trigrams':
            trigrams = []
            for w in counts['trigrams']:
                freq = w[1]
                phrase = " ".join(w[0])
                trigrams.append([phrase, freq])
            return filehandler.write_to_csv(['trigram phrase', 'frequency'], trigrams, '-trigram-counts.csv')
        logger.error("Requested unknown csv type: %s", analysis_type)
    except Exception as e:
        logger.exception(e)
    return render_template('no_results.html', tool_name='wordcounter')

def _clamp(n, minn, maxn):
    return max(min(maxn, n), minn)
<commit_msg>Support pasted text in WordCounter again #392<commit_after>import os
import random
import logging
from collections import OrderedDict
from databasic import mongo, get_base_dir
from databasic.forms import WordCounterPaste, WordCounterUpload, WordCounterSample, WordCounterLink
from databasic.logic import wordhandler, filehandler
from flask import Blueprint, render_template, request, redirect, g, abort, send_from_directory
from flask_babel import lazy_gettext as _

from databasic import NLTK_STOPWORDS_BY_LANGUAGE

mod = Blueprint('wordcounter', __name__,
                url_prefix='/<lang_code>/wordcounter',
                template_folder='../templates/wordcounter')

logger = logging.getLogger(__name__)

@mod.route('/', methods=('GET', 'POST'))
def index():

    words = None

    forms = OrderedDict()
    forms['sample'] = WordCounterSample(g.current_lang)
    forms['paste'] = WordCounterPaste('I am Sam\nSam I am\nThat Sam-I-am!\nThat Sam-I-am!\nI do not like that Sam-I-am!\nDo you like \ngreen eggs and ham?\nI do not like them, Sam-I-am.\nI do not like\ngreen eggs and ham.\nWould you like them \nhere or there?\nI would not like them\nhere or there.\nI would not like them anywhere.')
    forms['upload'] = WordCounterUpload()
    forms['link'] = WordCounterLink()

    if request.method == 'POST':
        ignore_case = True
        ignore_stopwords = True

        btn_value = request.form['btn']
        sample_id = ''
        extras_to_save = {}

        if btn_value == 'paste':
            words = forms['paste'].data['area']
            ignore_case = forms[btn_value].data['ignore_case_paste']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_paste']
            logger.debug("New from paste: %d chars", len(words) )
            title = _('your text')
        elif btn_value == 'upload':
            upload_file = forms['upload'].data['upload']
            words = process_upload(upload_file)
            ignore_case = forms[btn_value].data['ignore_case_upload']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_upload']
            title = upload_file.filename
            logger.debug("New from upload: %s", title)
        elif btn_value == 'sample':
            sample_source = forms['sample'].data['sample']
            samplename = filehandler.get_sample_title(sample_source)
            title = samplename
            ignore_case = forms[btn_value].data['ignore_case_sample']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_sample']
            sample_id = title+str(ignore_case)+str(ignore_stopwords)
            existing_doc_id = mongo.results_for_sample('wordcounter', sample_id)
            if existing_doc_id is not None:
                logger.debug("Existing from sample: %s", sample_source)
                return redirect(request.url + 'results/' + existing_doc_id)
            logger.info("New from sample: %s", sample_source)
            sample_path = filehandler.get_sample_path(sample_source)
            logger.debug("  loading from %s", sample_path)
            words = filehandler.convert_to_txt(sample_path)
            extras_to_save = filehandler.get_sample(sample_source)
        elif btn_value == 'link':
            url = forms['link'].data['link']
            # TODO: should actually accept https
            if 'https://' in url:
                url = url.replace('https', 'http')
            elif not 'http://' in url:
                url = 'http://' + url
            logger.debug("New from link: %s", url)
            content = filehandler.download_webpage(url)
            words = content['text']
            ignore_case = forms[btn_value].data['ignore_case_link']
            ignore_stopwords = forms[btn_value].data['ignore_stopwords_link']
            title = _(content['title'])

        if words is not None:
            logger.debug("  about to process words")
            counts = _process_words(words, ignore_case, ignore_stopwords, btn_value=='sample')
            logger.debug("  finished counts, about to save")
            doc_id = mongo.save_words('wordcounter', counts, ignore_case, ignore_stopwords, str(title), sample_id,
                                      btn_value, extras_to_save)
            logger.debug("  saved")
            return redirect(request.url + 'results/' + doc_id + '?submit=true')

    return render_template('wordcounter.html', forms=list(forms.items()), tool_name='wordcounter',
                           max_file_size_in_mb=g.max_file_size_mb)

@mod.route('/results/<doc_id>')
def results_for_doc(doc_id):
    results = {}
    remaining_days = None

    try:
        doc = mongo.find_document('wordcounter', doc_id)
        if doc['sample_id'] == '':
            remaining_days = mongo.get_remaining_days('wordcounter', doc_id)
    except:
        logger.warning("Unable to find doc '%s'", doc_id)
        return render_template('no_results.html', tool_name='wordcounter')

    counts = doc.get('counts')

    # only render the top 40 results on the page (the csv contains all results)
    results['unique_words'] = counts['unique_words'][:40]
    results['bigrams'] = counts['bigrams'][:40]
    results['trigrams'] = counts['trigrams'][:40]

    max_index = min(20, len(results['unique_words']))
    min_index = max(0, max_index-5)
    random_unpopular_word = ['', '']
    top_word = ''
    word_in_bigrams_count = 0
    word_in_trigrams_count = 0

    if len(results['unique_words']) > 0:
        random_unpopular_word = results['unique_words'][random.randrange(min_index, max_index+1)]\
            if len(results['unique_words']) > 1 else results['unique_words'][0]

        '''
        Find the most popular word that is also present in bigrams and trigrams.
        If none can be found, just get the most popular word.
        '''

        if results['unique_words'] and results['bigrams'] and results['trigrams']:
            for word in results['unique_words']:
                top_word = word[0]
                word_in_bigrams_count = 0
                word_in_trigrams_count = 0
                for b in results['bigrams']:
                    if top_word in b[0]:
                        word_in_bigrams_count += 1
                for t in results['trigrams']:
                    if top_word in t[0]:
                        word_in_trigrams_count += 1
                if word_in_bigrams_count > 0 and word_in_trigrams_count > 0:
                    break

        if word_in_bigrams_count == 0 and word_in_trigrams_count == 0:
            top_word = results['unique_words'][0][0]

    whatnext = {}
    whatnext['top_word'] = top_word
    whatnext['word_in_bigrams_count'] = word_in_bigrams_count
    whatnext['word_in_trigrams_count'] = word_in_trigrams_count
    whatnext['random_unpopular_word'] = random_unpopular_word[0]
    whatnext['random_unpopular_word_count'] = random_unpopular_word[1]
    biography = doc['biography'] if 'biography' in doc else None

    return render_template('wordcounter/results.html',
                           results=results,
                           whatnext=whatnext,
                           tool_name='wordcounter',
                           title=doc['title'],
                           doc_id=doc_id,
                           source=doc['source'],
                           remaining_days=remaining_days,
                           total_words=counts['total_word_count'],
                           biography=biography)

@mod.route('/results/<doc_id>/download/<analysis_type>.csv')
def download_csv(doc_id, analysis_type):
    logger.debug("Download %s", analysis_type)
    if analysis_type not in ['words','bigrams','trigrams']:
        logger.warning("Requested unknown csv type: %s", analysis_type)
        abort(400)
    try:
        doc = mongo.find_document('wordcounter', doc_id)
    except:
        logger.warning("Unable to find doc '%s'", doc_id)
        abort(400)
    file_path = create_csv_file(doc.get('counts'),analysis_type)
    logger.debug('  created %s csv to download at %s', analysis_type, file_path)
    if file_path is None:
        abort(500)
    return filehandler.generate_csv(file_path)

@mod.route('/wordcounter-activity-guide.pdf')
def download_activity_guide():
    filename = "WordCounter Activity Guide.pdf"
    dir_path = os.path.join(get_base_dir(),'databasic','static','files','activity-guides',g.current_lang)
    logger.debug("download activity guide from %s/%s", dir_path, filename)
    return send_from_directory(directory=dir_path, filename=filename)

@mod.route('/run-activity')
def run_activity():
    return render_template('wordcounter/run-activity.html')

def process_upload(doc):
    file_path = filehandler.open_doc(doc)
    file_size = os.stat(file_path).st_size  # because browser might not have sent content_length
    logger.debug("Upload: %d bytes", file_size)
    words = filehandler.convert_to_txt(file_path)
    filehandler.delete_file(file_path)
    return words

def _process_words(words, ignore_case, ignore_stopwords, is_sample):
    stopwords_language = NLTK_STOPWORDS_BY_LANGUAGE[g.current_lang]
    counts = wordhandler.get_word_counts(
        words,
        ignore_case,
        ignore_stopwords,
        stopwords_language)
    return counts

def create_csv_file(counts, analysis_type):
    try:
        if analysis_type == 'words':
            return filehandler.write_to_csv(['word', 'frequency'], counts['unique_words'], '-word-counts.csv')
        elif analysis_type == 'bigrams':
            bigrams = []
            for w in counts['bigrams']:
                freq = w[1]
                phrase = " ".join(w[0])
                bigrams.append([phrase, freq])
            return filehandler.write_to_csv(['bigram phrase', 'frequency'], bigrams, '-bigram-counts.csv')
        elif analysis_type == 'trigrams':
            trigrams = []
            for w in counts['trigrams']:
                freq = w[1]
                phrase = " ".join(w[0])
                trigrams.append([phrase, freq])
            return filehandler.write_to_csv(['trigram phrase', 'frequency'], trigrams, '-trigram-counts.csv')
        logger.error("Requested unknown csv type: %s", analysis_type)
    except Exception as e:
        logger.exception(e)
    return render_template('no_results.html', tool_name='wordcounter')

def _clamp(n, minn, maxn):
    return max(min(maxn, n), minn)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.secex.services import Product as SecexProductService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    bra_id = request.args.get('bra_id')

    #None database fields must be treated to do math operations
    #and templates with no data shall be omitted...

    #Vars to do tests:
    #section (depth == 2)
    #positon (depth == 6)
    #brazil (bra_id == None)
    #product_id = #05 #052601
    #bra_id = #None #4mg #4mg01 #4mg0000 #4mg010206 #2ce020008

    context = {
        'background_image':'static/img/bg-profile-location.jpg',
        'portrait':'static/img/mineric_product.jpg',
        'desc_general': 'Sample Text',
        'desc_international_trade': 'Sample Text',
        'desc_economic_opp': 'Sample Text'
    }

    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)
    context['depth'] = len(product_id)

    product = {}

    #criar o name service na HS
    #verificar local pci
    #verificar as tabelas usadas em cada classe

    if bra_id:
        secex_product_service = SecexProductByLocationService(bra_id=bra_id, product_id=product_id)

        #product.update(secex_product_service.location_destination_with_more_imports())
        #product.update(secex_product_service.location_origin_with_more_imports())

    else:
        secex_product_service = SecexProductService(product_id=product_id)

    product['year'] = secex_product_service.year()
    product['export_val'] = secex_product_service.export_val()
    product['import_val'] = secex_product_service.import_val()
    product['export_kg'] = secex_product_service.export_kg()
    product['import_kg'] = secex_product_service.import_kg()
    product['trade_balance'] = secex_product_service.trade_balance()
    product['export_net_weight'] = secex_product_service.export_net_weight()
    product['import_net_weight'] = secex_product_service.import_net_weight()
    product.update(secex_product_service.brazil_municipality_with_more_exports())
    product.update(secex_product_service.brazil_municipality_with_more_imports())
    product.update(secex_product_service.brazil_destination_with_more_exports())
    product.update(secex_product_service.brazil_origin_with_more_import())
    product.update(secex_product_service.name())
    product.update(secex_product_service.pci())

    #verificar campo year
    #return object pci name

    if bra_id == None:
        product['year'] = secex_product_service.year()
        product['export_val'] = secex_product_service.export_val()
        product['import_val'] = secex_product_service.import_val()
        product['export_kg'] = secex_product_service.export_kg()
        product['import_kg'] = secex_product_service.import_kg()
        product['trade_balance'] = secex_product_service.trade_balance()
        product['export_net_weight'] = secex_product_service.export_net_weight()
        product['import_net_weight'] = secex_product_service.import_net_weight()
        product.update(secex_product_service.brazil_municipality_with_more_exports())
        product.update(secex_product_service.brazil_municipality_with_more_imports())
        product.update(secex_product_service.brazil_destination_with_more_exports())
        product.update(secex_product_service.brazil_origin_with_more_import())

    else:

        if len(product_id) == 6:
            secex_product_service.
            product.update(secex_product_service.location_postion())

        elif len(product_id) == 2:
            product.update(secex_product_service.location_section())

        if len(bra_id) != 9:
            product.update(secex_product_service.location_municipality_with_more_exports())
            product.update(secex_product_service.location_municipality_with_more_imports())

    return render_template('product/index.html', body_class='perfil-estado', product=product, context=context)<commit_msg>Updating product views.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, request
from dataviva.apps.general.views import get_locale
from dataviva.api.attrs.services import Product as AttrsProductService
from dataviva.api.secex.services import Product as SecexProductService

mod = Blueprint('product', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/product',
                static_folder='static')

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/<product_id>')
def index(product_id):

    bra_id = request.args.get('bra_id')

    #None database fields must be treated to do math operations
    #and templates with no data shall be omitted...

    #Vars to do tests:
    #section (depth == 2)
    #positon (depth == 6)
    #brazil (bra_id == None)
    #product_id = #05 #052601
    #bra_id = #None #4mg #4mg01 #4mg0000 #4mg010206 #2ce020008

    context = {
        'background_image':'static/img/bg-profile-location.jpg',
        'portrait':'static/img/mineric_product.jpg',
        'desc_general': 'Sample Text',
        'desc_international_trade': 'Sample Text',
        'desc_economic_opp': 'Sample Text'
    }

    context['bra_id'] = bra_id
    if bra_id:
        context['bra_id_len'] = len(bra_id)
    context['depth'] = len(product_id)

    product = {}

    #criar o name service na HS
    #verificar local pci
    #verificar as tabelas usadas em cada classe
    #vericar imports

    secex_product_service = SecexProductByLocationService(bra_id=bra_id, product_id=product_id)
    product['name'] = secex_product_service.name()

    if bra_id:
        secex_product_service = SecexProductByLocationService(bra_id=bra_id, product_id=product_id)

        #product.update(secex_product_service.location_destination_with_more_imports())
        #product.update(secex_product_service.location_origin_with_more_imports())

    else:
        secex_product_service = SecexProductService(product_id=product_id)

    product['year'] = secex_product_service.year()
    product['export_val'] = secex_product_service.export_val()
    product['import_val'] = secex_product_service.import_val()
    product['export_kg'] = secex_product_service.export_kg()
    product['import_kg'] = secex_product_service.import_kg()
    product['trade_balance'] = secex_product_service.trade_balance()
    product['export_net_weight'] = secex_product_service.export_net_weight()
    product['import_net_weight'] = secex_product_service.import_net_weight()
    product.update(secex_product_service.brazil_municipality_with_more_exports())
    product.update(secex_product_service.brazil_municipality_with_more_imports())
    product.update(secex_product_service.brazil_destination_with_more_exports())
    product.update(secex_product_service.brazil_origin_with_more_import())
    product.update(secex_product_service.name())
    product.update(secex_product_service.pci())

    #verificar campo year
    #return object pci name

    if bra_id == None:
        product['year'] = secex_product_service.year()
        product['export_val'] = secex_product_service.export_val()
        product['import_val'] = secex_product_service.import_val()
        product['export_kg'] = secex_product_service.export_kg()
        product['import_kg'] = secex_product_service.import_kg()
        product['trade_balance'] = secex_product_service.trade_balance()
        product['export_net_weight'] = secex_product_service.export_net_weight()
        product['import_net_weight'] = secex_product_service.import_net_weight()
        product.update(secex_product_service.brazil_municipality_with_more_exports())
        product.update(secex_product_service.brazil_municipality_with_more_imports())
        product.update(secex_product_service.brazil_destination_with_more_exports())
        product.update(secex_product_service.brazil_origin_with_more_import())

    else:

        if len(product_id) == 6:
            secex_product_service.
            product.update(secex_product_service.location_postion())

        elif len(product_id) == 2:
            product.update(secex_product_service.location_section())

        if len(bra_id) != 9:
            product.update(secex_product_service.location_municipality_with_more_exports())
            product.update(secex_product_service.location_municipality_with_more_imports())

    return render_template('product/index.html', body_class='perfil-estado', product=product, context=context)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, redirect, url_for, flash, jsonify, request, send_from_directory
from dataviva.apps.general.views import get_locale
from dataviva.translations.dictionary import dictionary
from dataviva import app, db, admin_email
from dataviva.utils import upload_helper
from models import Article, AuthorScholar, KeyWord
from forms import RegistrationForm
from sqlalchemy import desc
from datetime import datetime
from flask.ext.login import login_required
from dataviva.apps.admin.views import required_roles
from dataviva.utils.send_mail import send_mail
import os
import shutil
import fnmatch

mod = Blueprint('scholar', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/scholar')

@mod.before_request
def before_request():
    g.page_type = mod.name

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/', methods=['GET'])
def index():
    articles = Article.query.filter_by(approval_status=True).order_by(desc(Article.postage_date)).all()
    return render_template('scholar/index.html', articles=articles)

@mod.route('/article/<id>', methods=['GET'])
def show(id):
    article = Article.query.filter_by(id=id).first_or_404()
    return render_template('scholar/show.html', article=article)

@mod.route('/admin', methods=['GET'])
@login_required
@required_roles(1)
def admin():
    articles = Article.query.all()
    return render_template('scholar/admin.html', articles=articles)

@mod.route('/admin', methods=['POST'])
@login_required
@required_roles(1)
def admin_update():
    for id, approval_status in request.form.iteritems():
        article = Article.query.filter_by(id=id).first_or_404()
        article.approval_status = approval_status == u'true'
        db.session.commit()
    message = u"Estudo(s) atualizados com sucesso!"
    return message

@mod.route('/admin/article/<status>/<status_value>', methods=['POST'])
@login_required
@required_roles(1)
def admin_activate(status, status_value):
    for id in request.form.getlist('ids[]'):
        article = Article.query.filter_by(id=id).first_or_404()
        setattr(article, status, status_value == u'true')
        db.session.commit()

    message = u"Artigo(s) alterada(s) com sucesso!"
    return message, 200

def new_article_advise(article):
    advise_message = render_template('scholar/mail/new_article_advise.html', article=article)
    send_mail("New Article", ["contato@dataviva.info","dataviva@fapemig.br"], advise_message)

@mod.route('/admin/article/new', methods=['GET'])
@login_required
def new():
    form = RegistrationForm()
    return render_template('scholar/new.html', form=form, action=url_for('scholar.create'))

@mod.route('/admin/article/new', methods=['POST'])
@login_required
def create():
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if not os.path.exists(upload_folder):
        flash(u'Selecione o arquivo do artigo para enviá-lo.', 'danger')
        return render_template('scholar/new.html', form=form)

    if form.validate() is False:
        return render_template('scholar/new.html', form=form)
    else:
        article = Article()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.approval_status = 0

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        db.session.add(article)
        db.session.flush()

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        new_article_advise(article)

        message = dictionary()["article_submission"]
        flash(message, 'success')

        return redirect(url_for('scholar.index'))

@mod.route('/admin/article/<id>/edit', methods=['GET'])
@login_required
@required_roles(1)
def edit(id):
    form = RegistrationForm()
    article = Article.query.filter_by(id=id).first_or_404()
    form.title.data = article.title
    form.theme.data = article.theme
    form.authors.data = article.authors_str()
    form.keywords.data = article.keywords_str()
    form.abstract.data = article.abstract
    article_url = article.url

    return render_template('scholar/edit.html', form=form, action=url_for('scholar.update', id=id), article_url=article_url)

@mod.route('/admin/article/<id>/edit', methods=['POST'])
@login_required
@required_roles(1)
def update(id):
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if form.validate() is False:
        return render_template('scholar/edit.html', form=form)
    else:
        article = Article.query.filter_by(id=id).first_or_404()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.authors = []
        article.keywords = []

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        message = u'Estudo editado com sucesso!'
        flash(message, 'success')
        return redirect(url_for('scholar.admin'))

@mod.route('/admin/article/delete', methods=['POST'])
@login_required
@required_roles(1)
def admin_delete():
    ids = request.form.getlist('ids[]')
    if ids:
        articles = Article.query.filter(Article.id.in_(ids)).all()
        for article in articles:
            upload_helper.delete_s3_folder(os.path.join(mod.name, str(article.id)))
            db.session.delete(article)

        db.session.commit()
        return u"Artigo(s) excluído(s) com sucesso!", 200
    else:
        return u'Selecione algum artigo para excluí-lo.', 205

@mod.route('/admin/articles/all', methods=['GET'])
@login_required
@required_roles(1)
def all():
    result = Article.query.all()
    articles = []
    for row in result:
        articles += [(row.id, row.title, row.authors_str(),
                      row.postage_date.strftime('%d/%m/%Y'), row.approval_status)]
    return jsonify(articles=articles)

@mod.route('/admin/article/upload', methods=['POST'])
@login_required
def upload():

    csrf_token = request.values['csrf_token']

    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        if [file for file in os.listdir(upload_folder)][0]:
            file_name = [file for file in os.listdir(upload_folder)][0]
            os.remove(os.path.join(upload_folder, file_name))
    else:
        os.makedirs(upload_folder)

    if request.files:
        file_name = request.files.keys()[0]
        file = request.files[file_name]
        file.save(os.path.join(upload_folder, file_name))
        return 'Arquivo salvo com sucesso!', 200
    else:
        return 'Não foi possível salvar o arquivo devido a um erro no servidor.', 400

@mod.route('/admin/article/delete', methods=['DELETE'])
@login_required
def delete():
    csrf_token = request.data
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        try:
            shutil.rmtree(os.path.split(upload_folder)[0])
            return 'Arquivo removido com sucesso!', 200
        except:
            return 'Não foi possível remover o arquivo devido a um erro no servidor.', 400

# serve static files on server
@mod.route('/admin/file/<string:csrf_token1>/<string:csrf_token2>', methods=['GET'])
@login_required
def get_file(csrf_token1, csrf_token2):
    import pdb; pdb.set_trace()
    csrf_token = csrf_token1 + '##' + csrf_token2
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')
    file_name = [file for file in os.listdir(upload_folder)][0]
    return send_from_directory(upload_folder, file_name)

# show static files on server before send to s3
@mod.route('/admin/data', methods=['GET'])
@login_required
@required_roles(1)
def data():
    matches = []
    for root, dirnames, filenames in os.walk(os.path.join(app.config['UPLOAD_FOLDER'], mod.name)):
        for filename in fnmatch.filter(filenames, '*.pdf'):
            matches.append(filename)
    return jsonify(file_names=matches)
<commit_msg>In views of scholar, trying to fix the link send by email with study link in new study advise.<commit_after># -*- coding: utf-8 -*-
from flask import Blueprint, render_template, g, redirect, url_for, flash, jsonify, request, send_from_directory
from dataviva.apps.general.views import get_locale
from dataviva.translations.dictionary import dictionary
from dataviva import app, db, admin_email
from dataviva.utils import upload_helper
from models import Article, AuthorScholar, KeyWord
from forms import RegistrationForm
from sqlalchemy import desc
from datetime import datetime
from flask.ext.login import login_required
from dataviva.apps.admin.views import required_roles
from dataviva.utils.send_mail import send_mail
import os
import shutil
import fnmatch

mod = Blueprint('scholar', __name__,
                template_folder='templates',
                url_prefix='/<lang_code>/scholar')

@mod.before_request
def before_request():
    g.page_type = mod.name

@mod.url_value_preprocessor
def pull_lang_code(endpoint, values):
    g.locale = values.pop('lang_code')

@mod.url_defaults
def add_language_code(endpoint, values):
    values.setdefault('lang_code', get_locale())

@mod.route('/', methods=['GET'])
def index():
    articles = Article.query.filter_by(approval_status=True).order_by(desc(Article.postage_date)).all()
    return render_template('scholar/index.html', articles=articles)

@mod.route('/article/<id>', methods=['GET'])
def show(id):
    article = Article.query.filter_by(id=id).first_or_404()
    return render_template('scholar/show.html', article=article)

@mod.route('/admin', methods=['GET'])
@login_required
@required_roles(1)
def admin():
    articles = Article.query.all()
    return render_template('scholar/admin.html', articles=articles)

@mod.route('/admin', methods=['POST'])
@login_required
@required_roles(1)
def admin_update():
    for id, approval_status in request.form.iteritems():
        article = Article.query.filter_by(id=id).first_or_404()
        article.approval_status = approval_status == u'true'
        db.session.commit()
    message = u"Estudo(s) atualizados com sucesso!"
    return message

@mod.route('/admin/article/<status>/<status_value>', methods=['POST'])
@login_required
@required_roles(1)
def admin_activate(status, status_value):
    for id in request.form.getlist('ids[]'):
        article = Article.query.filter_by(id=id).first_or_404()
        setattr(article, status, status_value == u'true')
        db.session.commit()

    message = u"Artigo(s) alterada(s) com sucesso!"
    return message, 200

def creat_study_url(article_id):
    url = request.url + str(article_id)
    url2 = url.replace("/admin","",1).replace("new","",1)
    return url2

def new_article_advise(article):
    url = creat_study_url(article.id)
    advise_message = render_template('scholar/mail/new_article_advise.html', article=article, url=url)
    send_mail("Novo estudo [" + article.date_str() + "]", [admin_email], advise_message)

@mod.route('/admin/article/new', methods=['GET'])
@login_required
def new():
    form = RegistrationForm()
    return render_template('scholar/new.html', form=form, action=url_for('scholar.create'))

@mod.route('/admin/article/new', methods=['POST'])
@login_required
def create():
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if not os.path.exists(upload_folder):
        flash(u'Selecione o arquivo do artigo para enviá-lo.', 'danger')
        return render_template('scholar/new.html', form=form)

    if form.validate() is False:
        return render_template('scholar/new.html', form=form)
    else:
        article = Article()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.approval_status = 0

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        db.session.add(article)
        db.session.flush()

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        new_article_advise(article)

        message = dictionary()["article_submission"]
        flash(message, 'success')

        return redirect(url_for('scholar.index'))

@mod.route('/admin/article/<id>/edit', methods=['GET'])
@login_required
@required_roles(1)
def edit(id):
    form = RegistrationForm()
    article = Article.query.filter_by(id=id).first_or_404()
    form.title.data = article.title
    form.theme.data = article.theme
    form.authors.data = article.authors_str()
    form.keywords.data = article.keywords_str()
    form.abstract.data = article.abstract
    article_url = article.url

    return render_template('scholar/edit.html', form=form, action=url_for('scholar.update', id=id), article_url=article_url)

@mod.route('/admin/article/<id>/edit', methods=['POST'])
@login_required
@required_roles(1)
def update(id):
    csrf_token = request.form.get('csrf_token')
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    form = RegistrationForm()

    if form.validate() is False:
        return render_template('scholar/edit.html', form=form)
    else:
        article = Article.query.filter_by(id=id).first_or_404()
        article.title = form.title.data
        article.theme = form.theme.data
        article.abstract = form.abstract.data
        article.postage_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        article.authors = []
        article.keywords = []

        author_input_list = form.authors.data.split(',')
        for author_input in author_input_list:
            article.authors.append(AuthorScholar(author_input))

        keyword_input_list = form.keywords.data.split(',')
        for keyword_input in keyword_input_list:
            keyword = KeyWord.query.filter_by(name=keyword_input).first()

            if not keyword:
                article.keywords.append(KeyWord(keyword_input))
            else:
                article.keywords.append(keyword)

        if os.path.exists(upload_folder):

            file_name = [file for file in os.listdir(upload_folder)][0]

            article.url = upload_helper.upload_s3_file(
                os.path.join(upload_folder, file_name),
                os.path.join('scholar/', str(article.id), 'files/', 'article'),
                {
                    'ContentType': "application/pdf",
                    'ContentDisposition': 'attachment; filename=dataviva-article-' + str(article.id) + '.pdf'
                }
            )

            shutil.rmtree(os.path.split(upload_folder)[0])

        db.session.commit()

        message = u'Estudo editado com sucesso!'
        flash(message, 'success')
        return redirect(url_for('scholar.admin'))

@mod.route('/admin/article/delete', methods=['POST'])
@login_required
@required_roles(1)
def admin_delete():
    ids = request.form.getlist('ids[]')
    if ids:
        articles = Article.query.filter(Article.id.in_(ids)).all()
        for article in articles:
            upload_helper.delete_s3_folder(os.path.join(mod.name, str(article.id)))
            db.session.delete(article)

        db.session.commit()
        return u"Artigo(s) excluído(s) com sucesso!", 200
    else:
        return u'Selecione algum artigo para excluí-lo.', 205

@mod.route('/admin/articles/all', methods=['GET'])
@login_required
@required_roles(1)
def all():
    result = Article.query.all()
    articles = []
    for row in result:
        articles += [(row.id, row.title, row.authors_str(),
                      row.postage_date.strftime('%d/%m/%Y'), row.approval_status)]
    return jsonify(articles=articles)

@mod.route('/admin/article/upload', methods=['POST'])
@login_required
def upload():

    csrf_token = request.values['csrf_token']

    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        if [file for file in os.listdir(upload_folder)][0]:
            file_name = [file for file in os.listdir(upload_folder)][0]
            os.remove(os.path.join(upload_folder, file_name))
    else:
        os.makedirs(upload_folder)

    if request.files:
        file_name = request.files.keys()[0]
        file = request.files[file_name]
        file.save(os.path.join(upload_folder, file_name))
        return 'Arquivo salvo com sucesso!', 200
    else:
        return 'Não foi possível salvar o arquivo devido a um erro no servidor.', 400

@mod.route('/admin/article/delete', methods=['DELETE'])
@login_required
def delete():
    csrf_token = request.data
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')

    if os.path.exists(upload_folder):
        try:
            shutil.rmtree(os.path.split(upload_folder)[0])
            return 'Arquivo removido com sucesso!', 200
        except:
            return 'Não foi possível remover o arquivo devido a um erro no servidor.', 400

# serve static files on server
@mod.route('/admin/file/<string:csrf_token1>/<string:csrf_token2>', methods=['GET'])
@login_required
def get_file(csrf_token1, csrf_token2):
    csrf_token = csrf_token1 + '##' + csrf_token2
    upload_folder = os.path.join(app.config['UPLOAD_FOLDER'], mod.name, csrf_token, 'files')
    file_name = [file for file in os.listdir(upload_folder)][0]
    return send_from_directory(upload_folder, file_name)

# show static files on server before send to s3
@mod.route('/admin/data', methods=['GET'])
@login_required
@required_roles(1)
def data():
    matches = []
    for root, dirnames, filenames in os.walk(os.path.join(app.config['UPLOAD_FOLDER'], mod.name)):
        for filename in fnmatch.filter(filenames, '*.pdf'):
            matches.append(filename)
    return jsonify(file_names=matches)
<|endoftext|>