<commit_before># -*- coding: utf-8 -*-
# Django settings for basic pinax project.

import os.path
import datetime
import posixpath
import time

from django.utils.http import http_date

import djcelery

djcelery.setup_loader()

PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))

DEBUG = False
TEMPLATE_DEBUG = True

SERVE_MEDIA = DEBUG
SERVE_LESS = DEBUG

# django-compressor is turned off by default due to deployment overhead for
# most users. See <URL> for more information
COMPRESS = False

INTERNAL_IPS = [
    "127.0.0.1",
]

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql_psycopg2",
        "NAME": "crate",
    }
}

TIME_ZONE = "UTC"
LANGUAGE_CODE = "en-us"

USE_I18N = False
USE_L10N = True
USE_TZ = True

MEDIA_ROOT = os.path.join(PROJECT_ROOT, "site_media", "media")
MEDIA_URL = "/site_media/media/"

STATIC_ROOT = os.path.join(PROJECT_ROOT, "site_media", "static")
STATIC_URL = "/site_media/static/"

ADMIN_MEDIA_PREFIX = posixpath.join(STATIC_URL, "admin/")

STATICFILES_DIRS = [
    os.path.join(PROJECT_ROOT, "static"),
]

STATICFILES_FINDERS = [
    "staticfiles.finders.FileSystemFinder",
    "staticfiles.finders.AppDirectoriesFinder",
    "staticfiles.finders.LegacyAppDirectoriesFinder",
    "compressor.finders.CompressorFinder",
]

COMPRESS_OUTPUT_DIR = "cache"

SECRET_KEY = ")ys)^6l18seiq)veen^41ox%l@6#o-2t*9-8n5$7$f+qi*x03$"

TEMPLATE_LOADERS = [
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
]

MIDDLEWARE_CLASSES = [
    "django_hosts.middleware.HostsMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django_openid.consumer.SessionConsumer",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",

    "pagination.middleware.PaginationMiddleware",

    "pinax.apps.account.middleware.LocaleMiddleware",
    "pinax.middleware.security.HideSensistiveFieldsMiddleware",
]

ROOT_URLCONF = "crate_project.urls"
ROOT_HOSTCONF = "crate_project.hosts"

DEFAULT_HOST = "default"

WSGI_APPLICATION = "crate_project.wsgi.application"

TEMPLATE_DIRS = [
    os.path.join(PROJECT_ROOT, "templates"),
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    "django.contrib.messages.context_processors.messages",

    "staticfiles.context_processors.static",

    "pinax.core.context_processors.pinax_settings",

    "pinax.apps.account.context_processors.account",

    # "notification.context_processors.notification",
    "announcements.context_processors.site_wide_announcements",
]

INSTALLED_APPS = [
    # Admin Dashboard
    "admin_tools",
    "admin_tools.theming",
    "admin_tools.menu",
    "admin_tools.dashboard",

    # Django
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.messages",
    "django.contrib.humanize",
    "django.contrib.markup",

    "pinax.templatetags",

    # theme
    "pinax_theme_bootstrap",

    # external (Pinax)
    # "notification",  # must be first
    "staticfiles",
    "pagination",
    "compressor",
    "django_openid",
    "timezones",
    "emailconfirmation",
    "announcements",
    "idios",
    "metron",

    # Pinax
    "pinax.apps.account",
    "pinax.apps.signup_codes",

    # external (Project)
    "south",
    "djcelery",
    "django_hosts",
    "haystack",
    "storages",
    "intercom",
    "celery_haystack",
    "tastypie",

    # project
    "about",
    "profiles",
    "packages",
    "pypi",
    "search",
    "crate",
    "evaluator",
    "privatebeta",
    "helpdocs",
]

FIXTURE_DIRS = [
    os.path.join(PROJECT_ROOT, "fixtures"),
]

MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage"

ABSOLUTE_URL_OVERRIDES = {
    "auth.user": lambda o: "/profiles/profile/%s/" % o.username,
}

AUTH_PROFILE_MODULE = "profiles.Profile"
NOTIFICATION_LANGUAGE_MODULE = "account.Account"

ACCOUNT_OPEN_SIGNUP = True
ACCOUNT_USE_OPENID = False
ACCOUNT_REQUIRED_EMAIL = True
ACCOUNT_EMAIL_VERIFICATION = True
ACCOUNT_EMAIL_AUTHENTICATION = False
ACCOUNT_UNIQUE_EMAIL = EMAIL_CONFIRMATION_UNIQUE_EMAIL = True

AUTHENTICATION_BACKENDS = [
    "pinax.apps.account.auth_backends.AuthenticationBackend",
]

LOGIN_URL = "/account/login/"
LOGIN_REDIRECT_URLNAME = "what_next"
LOGOUT_REDIRECT_URLNAME = "search"

EMAIL_CONFIRMATION_DAYS = 2
EMAIL_DEBUG = DEBUG

DEBUG_TOOLBAR_CONFIG = {
    "INTERCEPT_REDIRECTS": False,
}

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_PUBLISH_RETRY = True

CELERY_TASK_RESULT_EXPIRES = 7 * 24 * 60 * 60  # 7 Days

CELERYD_HIJACK_ROOT_LOGGER = False

CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 15

AWS_QUERYSTRING_AUTH = False
AWS_S3_SECURE_URLS = False

AWS_HEADERS = {
    "Expires": lambda: http_date(time.mktime((datetime.datetime.now() + datetime.timedelta(days=365)).timetuple())),
    "Cache-Control": "max-age=31556926",
}

METRON_SETTINGS = {
    "mixpanel": {3: "a280fbfb3039709d8a5ac15834a16e5e"},
    "google": {3: "UA-28759418-1"},
    "gauges": {3: "4f1e4cd0613f5d7003000002"}
}

ADMIN_TOOLS_INDEX_DASHBOARD = "crate.dashboard.CrateIndexDashboard"
<commit_msg>set a task to ignore celery task results<commit_after># -*- coding: utf-8 -*-
# Django settings for basic pinax project.

import os.path
import datetime
import posixpath
import time

from django.utils.http import http_date

import djcelery

djcelery.setup_loader()

PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))

DEBUG = False
TEMPLATE_DEBUG = True

SERVE_MEDIA = DEBUG
SERVE_LESS = DEBUG

# django-compressor is turned off by default due to deployment overhead for
# most users. See <URL> for more information
COMPRESS = False

INTERNAL_IPS = [
    "127.0.0.1",
]

DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql_psycopg2",
        "NAME": "crate",
    }
}

TIME_ZONE = "UTC"
LANGUAGE_CODE = "en-us"

USE_I18N = False
USE_L10N = True
USE_TZ = True

MEDIA_ROOT = os.path.join(PROJECT_ROOT, "site_media", "media")
MEDIA_URL = "/site_media/media/"

STATIC_ROOT = os.path.join(PROJECT_ROOT, "site_media", "static")
STATIC_URL = "/site_media/static/"

ADMIN_MEDIA_PREFIX = posixpath.join(STATIC_URL, "admin/")

STATICFILES_DIRS = [
    os.path.join(PROJECT_ROOT, "static"),
]

STATICFILES_FINDERS = [
    "staticfiles.finders.FileSystemFinder",
    "staticfiles.finders.AppDirectoriesFinder",
    "staticfiles.finders.LegacyAppDirectoriesFinder",
    "compressor.finders.CompressorFinder",
]

COMPRESS_OUTPUT_DIR = "cache"

SECRET_KEY = ")ys)^6l18seiq)veen^41ox%l@6#o-2t*9-8n5$7$f+qi*x03$"

TEMPLATE_LOADERS = [
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
]

MIDDLEWARE_CLASSES = [
    "django_hosts.middleware.HostsMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django_openid.consumer.SessionConsumer",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",

    "pagination.middleware.PaginationMiddleware",

    "pinax.apps.account.middleware.LocaleMiddleware",
    "pinax.middleware.security.HideSensistiveFieldsMiddleware",
]

ROOT_URLCONF = "crate_project.urls"
ROOT_HOSTCONF = "crate_project.hosts"

DEFAULT_HOST = "default"

WSGI_APPLICATION = "crate_project.wsgi.application"

TEMPLATE_DIRS = [
    os.path.join(PROJECT_ROOT, "templates"),
]

TEMPLATE_CONTEXT_PROCESSORS = [
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.request",
    "django.contrib.messages.context_processors.messages",

    "staticfiles.context_processors.static",

    "pinax.core.context_processors.pinax_settings",

    "pinax.apps.account.context_processors.account",

    # "notification.context_processors.notification",
    "announcements.context_processors.site_wide_announcements",
]

INSTALLED_APPS = [
    # Admin Dashboard
    "admin_tools",
    "admin_tools.theming",
    "admin_tools.menu",
    "admin_tools.dashboard",

    # Django
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.sites",
    "django.contrib.messages",
    "django.contrib.humanize",
    "django.contrib.markup",

    "pinax.templatetags",

    # theme
    "pinax_theme_bootstrap",

    # external (Pinax)
    # "notification",  # must be first
    "staticfiles",
    "pagination",
    "compressor",
    "django_openid",
    "timezones",
    "emailconfirmation",
    "announcements",
    "idios",
    "metron",

    # Pinax
    "pinax.apps.account",
    "pinax.apps.signup_codes",

    # external (Project)
    "south",
    "djcelery",
    "django_hosts",
    "haystack",
    "storages",
    "intercom",
    "celery_haystack",
    "tastypie",

    # project
    "about",
    "profiles",
    "packages",
    "pypi",
    "search",
    "crate",
    "evaluator",
    "privatebeta",
    "helpdocs",
]

FIXTURE_DIRS = [
    os.path.join(PROJECT_ROOT, "fixtures"),
]

MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage"

ABSOLUTE_URL_OVERRIDES = {
    "auth.user": lambda o: "/profiles/profile/%s/" % o.username,
}

AUTH_PROFILE_MODULE = "profiles.Profile"
NOTIFICATION_LANGUAGE_MODULE = "account.Account"

ACCOUNT_OPEN_SIGNUP = True
ACCOUNT_USE_OPENID = False
ACCOUNT_REQUIRED_EMAIL = True
ACCOUNT_EMAIL_VERIFICATION = True
ACCOUNT_EMAIL_AUTHENTICATION = False
ACCOUNT_UNIQUE_EMAIL = EMAIL_CONFIRMATION_UNIQUE_EMAIL = True

AUTHENTICATION_BACKENDS = [
    "pinax.apps.account.auth_backends.AuthenticationBackend",
]

LOGIN_URL = "/account/login/"
LOGIN_REDIRECT_URLNAME = "what_next"
LOGOUT_REDIRECT_URLNAME = "search"

EMAIL_CONFIRMATION_DAYS = 2
EMAIL_DEBUG = DEBUG

DEBUG_TOOLBAR_CONFIG = {
    "INTERCEPT_REDIRECTS": False,
}

CELERY_SEND_TASK_ERROR_EMAILS = True
CELERY_DISABLE_RATE_LIMITS = True
CELERY_TASK_PUBLISH_RETRY = True

CELERY_IGNORE_RESULT = True

CELERY_TASK_RESULT_EXPIRES = 7 * 24 * 60 * 60  # 7 Days

CELERYD_HIJACK_ROOT_LOGGER = False

CELERYBEAT_SCHEDULER = "djcelery.schedulers.DatabaseScheduler"

HAYSTACK_SEARCH_RESULTS_PER_PAGE = 15

AWS_QUERYSTRING_AUTH = False
AWS_S3_SECURE_URLS = False

AWS_HEADERS = {
    "Expires": lambda: http_date(time.mktime((datetime.datetime.now() + datetime.timedelta(days=365)).timetuple())),
    "Cache-Control": "max-age=31556926",
}

METRON_SETTINGS = {
    "mixpanel": {3: "a280fbfb3039709d8a5ac15834a16e5e"},
    "google": {3: "UA-28759418-1"},
    "gauges": {3: "4f1e4cd0613f5d7003000002"}
}

ADMIN_TOOLS_INDEX_DASHBOARD = "crate.dashboard.CrateIndexDashboard"
<|endoftext|>


<commit_before>python runtests_bootstrap.py
python runtests_uniform.py
<commit_msg>Redoing runtests for being proper Python<commit_after>#!/usr/bin/env python

execfile('runtests_bootstrap.py')
execfile('runtests_uniform.py')
<|endoftext|>


<commit_before>from flask import current_app
from croplands_api import celery
from croplands_api.models import db, Record
from croplands_api.utils.google.fusion import replace_rows
from croplands_api.utils.google.gee import extract, ee
from croplands_api.utils.mappings import get_crop_label, get_intensity_label, get_land_cover_label, \
    get_water_label
import StringIO
import csv
from datetime import datetime, timedelta

def status(so_far, total):
    print '%d bytes transferred out of %d' % (so_far, total)

def convert_to_labels(row):
    row['water_source'] = get_water_label(row['water_source'])
    row['crop_primary'] = get_crop_label(row['crop_primary'])
    row['crop_secondary'] = get_crop_label(row['crop_secondary'])
    row['land_cover'] = get_land_cover_label(row['land_cover'])
    row['intensity'] = get_intensity_label(row['intensity'])
    return row

def median(A):
    if len(A) == 0:
        return None
    return sorted(A)[len(A) / 2]

def mean(A):
    if len(A) == 0:
        return None
    total = 0.0
    count = 0

    for i in A:
        if i is not None:
            total += i
            count += 1

    if count == 0:
        return 0

    return total / count

@celery.task(rate_limit="120/m", time_limit=300)
def get_ndvi(id=None, record=None):
    """
    Gets the ndvi time series for a record and insert into array(23 elements) for record.
    Also computes the mean ndvi.

    :param id: record.id optional
    :param record: Record
    :return: None
    """
    eta = datetime.utcnow() + timedelta(days=150)

    if id is not None and record is None:
        record = db.session.query(Record).filter(Record.id == id).first()

    geometry = ee.Geometry.Point(record.location.lon, record.location.lat)
    start, end = str(record.year) + "-01-01", str(record.year) + "-12-31"
    collection = ee.ImageCollection('MODIS/MOD13Q1').filterDate(start, end).select(['NDVI'])
    results = extract(geometry, collection)

    series23 = [(row['NDVI'] / 10) if row['NDVI'] is not None else None for row in results]

    if len(series23) < 23:
        series23 += [None for i in range(23 - len(series23))]
        eta = datetime.utcnow() + timedelta(days=10)

    assert len(series23) == 23

    record.ndvi_mean = int(mean(series23))
    record.ndvi = series23

    print("Record #%d NDVI Series Updated. Mean: %d" % (record.id, record.ndvi_mean))

    db.session.commit()

    # todo fix for testing...
    if current_app.config['ENV'] != 'TESTING':
        get_ndvi.apply_async(args=[id, record], eta=eta)

@celery.task()
def sum_ratings_for_record(id):
    try:
        db.session.execute(
            """
            UPDATE record
            SET rating = (SELECT sum(rating)
                          FROM record_rating
                          WHERE record_rating.record_id = :id)
            WHERE id = :id
            """,
            {'id': id}
        )
        db.session.commit()
    except Exception as e:
        print e
        pass

@celery.task(rate_limit="15/h", time_limit=300)
def build_fusion_tables():
    cmd = """
          SELECT  record.id as id,
                  location.id AS location_id,
                  location.lat AS lat,
                  location.lon AS lon,
                  record.rating as rating,
                  record.year as year,
                  record.month as month,
                  record.land_use_type as land_cover,
                  record.crop_primary as crop_primary,
                  record.crop_secondary as crop_secondary,
                  record.water as water_source,
                  record.intensity as intensity,
                  location.country as country,
                  record.source_description as source_description,
                  record.source_type as source_type,
                  record.source_id as source_id,
                  record.source_class as source_class,
                  images.image_1 as image_1,
                  images.image_2 as image_2,
                  images.image_3 as image_3,
                  location.use_validation as use_validation,
                  location.use_private as use_private
          FROM record
          LEFT JOIN location ON location.id = record.location_id
          LEFT OUTER JOIN (SELECT * from crosstab (
            $$SELECT location_id, url, replace(url, 'images/', 'http://images.croplands.org/')
             FROM   image
             WHERE  url not like '%%digital_globe%%'
             $$)
             AS t ( location_id int,
              image_1 text, --varchar changed to text with replace function above
              image_2 text,
              image_3 text
            )
          ) images on location.id = images.location_id
          WHERE location.use_deleted is false AND location.use_invalid is false
          """

    result = db.engine.execute(cmd)
    columns = result.keys()

    all_results = [convert_to_labels(dict(row.items())) for row in result]

    training = StringIO.StringIO()
    validation = StringIO.StringIO()
    public = StringIO.StringIO()

    writer_training = csv.DictWriter(training, fieldnames=columns[0:-2], extrasaction='ignore')
    writer_validation = csv.DictWriter(validation, fieldnames=columns[0:-2], extrasaction='ignore')
    writer_public = csv.DictWriter(public, fieldnames=columns[0:-2], extrasaction='ignore')

    writer_training.writeheader()
    writer_validation.writeheader()
    writer_public.writeheader()

    print len(all_results)

    for row in all_results:
        try:
            if not row['use_private']:
                writer_public.writerow(row)
            if row['use_validation']:
                writer_validation.writerow(row)
            else:
                row['intensity'] = 'Unknown'
                writer_training.writerow(row)
        except UnicodeEncodeError as e:
            print e, row

    replace_rows('1C_gFvQmd3AGtB0Q0XgnKk5ESUARSH79FB9Un8sF2', training, startLine=1)
    replace_rows('12WLGpk7o1ic_j88NQfmrUEILVWDlrJaqZCAqEDeo', validation, startLine=1)
    replace_rows('1jQjTg7zXhwmLGJdfPCavgdifnyNTqJGi3Bn3RwWF', public, startLine=1)
<commit_msg>allow records to have intensity<commit_after>from flask import current_app
from croplands_api import celery
from croplands_api.models import db, Record
from croplands_api.utils.google.fusion import replace_rows
from croplands_api.utils.google.gee import extract, ee
from croplands_api.utils.mappings import get_crop_label, get_intensity_label, get_land_cover_label, \
    get_water_label
import StringIO
import csv
from datetime import datetime, timedelta

def status(so_far, total):
    print '%d bytes transferred out of %d' % (so_far, total)

def convert_to_labels(row):
    row['water_source'] = get_water_label(row['water_source'])
    row['crop_primary'] = get_crop_label(row['crop_primary'])
    row['crop_secondary'] = get_crop_label(row['crop_secondary'])
    row['land_cover'] = get_land_cover_label(row['land_cover'])
    row['intensity'] = get_intensity_label(row['intensity'])
    return row

def median(A):
    if len(A) == 0:
        return None
    return sorted(A)[len(A) / 2]

def mean(A):
    if len(A) == 0:
        return None
    total = 0.0
    count = 0

    for i in A:
        if i is not None:
            total += i
            count += 1

    if count == 0:
        return 0

    return total / count

@celery.task(rate_limit="120/m", time_limit=300)
def get_ndvi(id=None, record=None):
    """
    Gets the ndvi time series for a record and insert into array(23 elements) for record.
    Also computes the mean ndvi.

    :param id: record.id optional
    :param record: Record
    :return: None
    """
    eta = datetime.utcnow() + timedelta(days=150)

    if id is not None and record is None:
        record = db.session.query(Record).filter(Record.id == id).first()

    geometry = ee.Geometry.Point(record.location.lon, record.location.lat)
    start, end = str(record.year) + "-01-01", str(record.year) + "-12-31"
    collection = ee.ImageCollection('MODIS/MOD13Q1').filterDate(start, end).select(['NDVI'])
    results = extract(geometry, collection)

    series23 = [(row['NDVI'] / 10) if row['NDVI'] is not None else None for row in results]

    if len(series23) < 23:
        series23 += [None for i in range(23 - len(series23))]
        eta = datetime.utcnow() + timedelta(days=10)

    assert len(series23) == 23

    record.ndvi_mean = int(mean(series23))
    record.ndvi = series23

    print("Record #%d NDVI Series Updated. Mean: %d" % (record.id, record.ndvi_mean))

    db.session.commit()

    # todo fix for testing...
    if current_app.config['ENV'] != 'TESTING':
        get_ndvi.apply_async(args=[id, record], eta=eta)

@celery.task()
def sum_ratings_for_record(id):
    try:
        db.session.execute(
            """
            UPDATE record
            SET rating = (SELECT sum(rating)
                          FROM record_rating
                          WHERE record_rating.record_id = :id)
            WHERE id = :id
            """,
            {'id': id}
        )
        db.session.commit()
    except Exception as e:
        print e
        pass

@celery.task(rate_limit="15/h", time_limit=300)
def build_fusion_tables():
    cmd = """
          SELECT  record.id as id,
                  location.id AS location_id,
                  location.lat AS lat,
                  location.lon AS lon,
                  record.rating as rating,
                  record.year as year,
                  record.month as month,
                  record.land_use_type as land_cover,
                  record.crop_primary as crop_primary,
                  record.crop_secondary as crop_secondary,
                  record.water as water_source,
                  record.intensity as intensity,
                  location.country as country,
                  record.source_description as source_description,
                  record.source_type as source_type,
                  record.source_id as source_id,
                  record.source_class as source_class,
                  images.image_1 as image_1,
                  images.image_2 as image_2,
                  images.image_3 as image_3,
                  location.use_validation as use_validation,
                  location.use_private as use_private
          FROM record
          LEFT JOIN location ON location.id = record.location_id
          LEFT OUTER JOIN (SELECT * from crosstab (
            $$SELECT location_id, url, replace(url, 'images/', 'http://images.croplands.org/')
             FROM   image
             WHERE  url not like '%%digital_globe%%'
             $$)
             AS t ( location_id int,
              image_1 text, --varchar changed to text with replace function above
              image_2 text,
              image_3 text
            )
          ) images on location.id = images.location_id
          WHERE location.use_deleted is false AND location.use_invalid is false
          """

    result = db.engine.execute(cmd)
    columns = result.keys()

    all_results = [convert_to_labels(dict(row.items())) for row in result]

    training = StringIO.StringIO()
    validation = StringIO.StringIO()
    public = StringIO.StringIO()

    writer_training = csv.DictWriter(training, fieldnames=columns[0:-2], extrasaction='ignore')
    writer_validation = csv.DictWriter(validation, fieldnames=columns[0:-2], extrasaction='ignore')
    writer_public = csv.DictWriter(public, fieldnames=columns[0:-2], extrasaction='ignore')

    writer_training.writeheader()
    writer_validation.writeheader()
    writer_public.writeheader()

    print len(all_results)

    for row in all_results:
        try:
            if not row['use_private']:
                writer_public.writerow(row)
            if row['use_validation']:
                writer_validation.writerow(row)
            else:
                writer_training.writerow(row)
        except UnicodeEncodeError as e:
            print e, row

    replace_rows('1C_gFvQmd3AGtB0Q0XgnKk5ESUARSH79FB9Un8sF2', training, startLine=1)
    replace_rows('12WLGpk7o1ic_j88NQfmrUEILVWDlrJaqZCAqEDeo', validation, startLine=1)
    replace_rows('1jQjTg7zXhwmLGJdfPCavgdifnyNTqJGi3Bn3RwWF', public, startLine=1)
<|endoftext|>


<commit_before>"""Configuration for Django system."""

__version__ = "0.5.1"
<commit_msg>Increment version number to 0.5.2<commit_after>"""Configuration for Django system."""

__version__ = "0.5.2"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2010-2012 Cidadania S. Coop. Galega
#
# This file is part of e-cidadania.
#
# e-cidadania is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# e-cidadania is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with e-cidadania. If not, see <http://www.gnu.org/licenses/>.

"""
The proposal administration allows to edit every proposal made in the system.
"""

from django.contrib import admin
from django.utils.translation import ugettext_lazy as _

from apps.ecidadania.proposals.models import Proposal

class ProposalAdmin(admin.ModelAdmin):

    """
    Basic proposal administration interface since most of the work is done
    in the website.

    :list display: title, author, tags
    :search: none:
    """
    list_display = ('title', 'author', 'tags')

    fieldsets = [
        (None, {'fields':
            ['code', 'title', 'description', 'tags', 'support_votes']}),

        (_('Location'), {'fields':
            ['latitude', 'longitude']}),

        (_('Relations'), {'fields':
            [('space', 'author')]}),

        (_('Other'), {'fields':
            ['budget', 'closed', 'close_reason', 'closed_by']}),

        (_('Options'), {'fields':
            ['anon_allowed', 'refurbished']}),

    ]

admin.site.register(Proposal, ProposalAdmin)
<commit_msg>ProposalSet model admin add<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2010-2012 Cidadania S. Coop. Galega
#
# This file is part of e-cidadania.
#
# e-cidadania is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# e-cidadania is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with e-cidadania. If not, see <http://www.gnu.org/licenses/>.

"""
The proposal administration allows to edit every proposal made in the system.
"""

from django.contrib import admin
from django.utils.translation import ugettext_lazy as _

from apps.ecidadania.proposals.models import Proposal, ProposalSet

class ProposalSetAdmin(admin.ModelAdmin):

    """
    Basic ProposalSet administration interface.

    :list_display: name, author
    :search: none
    """

    list_display = ('name', 'author')

    fieldsets = [
        (None, {'fields':
            ['name']})
    ]

class ProposalAdmin(admin.ModelAdmin):

    """
    Basic proposal administration interface since most of the work is done
    in the website.

    :list display: title, author, tags
    :search: none:
    """
    list_display = ('title', 'author', 'tags')

    fieldsets = [
        (None, {'fields':
            ['code', 'title', 'description', 'tags', 'support_votes']}),

        (_('Location'), {'fields':
            ['latitude', 'longitude']}),

        (_('Relations'), {'fields':
            [('space', 'author')]}),

        (_('Other'), {'fields':
            ['budget', 'closed', 'close_reason', 'closed_by']}),

        (_('Options'), {'fields':
            ['anon_allowed', 'refurbished']}),

    ]

admin.site.register(ProposalSet, ProposalSetAdmin)
admin.site.register(Proposal, ProposalAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# this file is released under public domain and you can use without limitations
import re

#########################################################################
## This is a samples controller
## - index is the default action of any application
## - user is required for authentication and authorization
## - download is for downloading files uploaded in the db (does streaming)
## - call exposes all registered services (none by default)
#########################################################################

def index():
    """
    Show an introduction page for visitors, or personalized curation dashboard for
    a logged-in user.
    """
    #response.flash = T("Welcome to web2py!")

    if False:  ## auth.is_logged_in():
        # user is logged in, bounce to their personal dashboard
        redirect(URL('dashboard'))
    else:
        # anonymous visitor, show a general info page
        return dict()

def error():
    return dict()

@auth.requires_login()
def dashboard():
    return dict(message="My Curation Activity")

def user():
    """
    exposes:
    http://..../[app]/default/user/login
    http://..../[app]/default/user/logout
    http://..../[app]/default/user/register
    http://..../[app]/default/user/profile
    http://..../[app]/default/user/retrieve_password
    http://..../[app]/default/user/change_password
    use @auth.requires_login()
        @auth.requires_membership('group name')
        @auth.requires_permission('read','table name',record_id)
    to decorate functions that need access control
    """
    return dict(form=auth())

def download():
    """
    allows downloading of uploaded files
    http://..../[app]/default/download/[filename]
    """
    return response.download(request, db)

def call():
    """
    exposes services. for example:
    http://..../[app]/default/call/jsonrpc
    decorate with @services.jsonrpc the functions to expose
    supports xml, json, xmlrpc, jsonrpc, amfrpc, rss, csv
    """
    return service()

@auth.requires_signature()
def data():
    """
    http://..../[app]/default/data/tables
    http://..../[app]/default/data/create/[table]
    http://..../[app]/default/data/read/[table]/[id]
    http://..../[app]/default/data/update/[table]/[id]
    http://..../[app]/default/data/delete/[table]/[id]
    http://..../[app]/default/data/select/[table]
    http://..../[app]/default/data/search/[table]
    but URLs must be signed, i.e. linked with
      A('table',_href=URL('data/tables',user_signature=True))
    or with the signed load operator
      LOAD('default','data.load',args='tables',ajax=True,user_signature=True)
    """
    return dict(form=crud())

UPLOADID_PAT = re.compile(r'^[-_.a-zA-Z0-9]{5,85}$')
def to_nexml():
    global UPLOADID_PAT
    from externalproc import get_external_proc_dir_for_upload, get_logger, invoc_status, \
            ExternalProcStatus, get_conf, write_input_files, write_ext_proc_content, do_ext_proc_launch
    import os
    import datetime
    import codecs
    import locket
    import shutil
    from xml2jsonbadgerfish import to_badgerfish_dict, cull_for_ot_nexson
    '''
    Controller for conversion of NEXUS, newick, or NeXML to NeXSON
    Required arguments:
        "uploadid" - A unique string for this upload. It needs
            to match the pattern '^[-_.a-zA-Z0-9]{5,85}$'
            clash with a previously used ID will cause data from the
            previous upload to be returned.
        "file" should be a multipart-encoded file to be translated to NexSON
          OR
        "content" which is a string that contains the content of the file
            format. "content" is checked if "file" is not provided.
    Optional arguments:
        "output" one of ['ot:nexson', 'nexson', 'nexml', 'input', 'provenance']
            the default is ot:nexson. This specifies what is to be returned.
            Possible values are:
            "ot:nexson" is the Open Tree NexSON (with character data culled).
                This should be the first call to this uploadid. Subsequent
                calls can retrieve intermediates. JSON.
            "nexson" is the complete NexSON version of the data. JSON.
            "nexml" is the NeXML version of the file. This is an intermediate
                for the NexSON. XML.
            "input" returns the uploaded file. text/plain.
            "provenance" returns a simple, ad-hoc JSON with initial call details.
        "dataDeposit" should be a URL that should be added to the meta element
            in the Open Tree NexSON object.
        "inputformat" should be "nexus", "newick", or "nexml"
            default is "nexus"

    '''
    _LOG = get_logger(request, 'to_nexml')
    try:
        unique_id = request.vars.uploadid
        assert unique_id is not None
    except:
        raise HTTP(400, 'Expecting an "uploadid" argument with a unique ID for this upload')
    if not UPLOADID_PAT.match(unique_id):
        raise HTTP(400, 'uploadid must be series of letters, numbers, dots or dashes between 5 and 85 characters long. "{u}" does not match this pattern'.format(u=unique_id))
    output = request.vars.output or 'ot:nexson'
    output = output.lower()
    output_choices = ['ot:nexson', 'nexson', 'nexml', 'input', 'provenance']
    if output not in output_choices:
        raise HTTP(400, 'The "output" should be one of: "{c}"'.format(c='", "'.join(output_choices)))
    try:
        working_dir = get_external_proc_dir_for_upload(request, '2nexml', unique_id)
    except Exception, x:
        raise HTTP(404)
    if working_dir is None or not os.path.exists(working_dir):
        raise HTTP(404)
    INPUT_FILENAME = 'in.nex'
    PROV_FILENAME = 'provenance.json'
    NEXML_FILENAME = 'out.xml'
    ERR_FILENAME = 'err.txt'

    INPUT_FILEPATH = os.path.join(working_dir, INPUT_FILENAME)
    INP_LOCKFILEPATH = os.path.join(working_dir, INPUT_FILENAME + '.lock')
    inpfp = os.path.join(working_dir, INPUT_FILENAME)
    input_choices = ['nexus', 'newick', 'nexml']
    with locket.lock_file(INP_LOCKFILEPATH):
        if not os.path.exists(INPUT_FILEPATH):
            if output != 'ot:nexson':
                raise HTTP(400, 'The "output" argument should be "ot:nexson" in the first call with each "uploadid"')
            if request.vars.file is not None:
                upf = request.vars.file
                upload_stream = upf.file
                filename = upf.filename
            elif request.vars.content is not None:
                upload_stream = request.vars.content # stream is a bad name, but write_input_files does the write thing.
                filename = '<content provided as a string in a "content" rather than a file upload>'
            else:
                raise HTTP(400, 'Expecting a "file" argument with an input file or a "content" argument with the contents of in input file')
            inp_format = request.vars.inputformat or 'nexus'
            inp_format = inp_format.lower()
            if inp_format not in input_choices:
                raise HTTP(400, 'inputformat should be one of: "{c}"'.format(c='", "'.join(input_choices)))
            write_input_files(request, working_dir, [(INPUT_FILENAME, upload_stream)])
            prov_info = {
                'filename' : filename,
                'date-created': datetime.datetime.utcnow().isoformat(),
            }
            if request.vars.dataDeposit:
                prov_info['data-deposit'] = request.vars.dataDeposit
            write_ext_proc_content(request,
                                   working_dir,
                                   [(PROV_FILENAME, json.dumps(prov_info))],
                                   encoding='utf-8')
    if output == 'provenance':
        PROV_FILEPATH =  os.path.join(working_dir, PROV_FILENAME)
        response.view = 'generic.json'
        return json.load(codecs.open(PROV_FILEPATH, 'rU', encoding='utf-8'))
    if output == 'input':
        response.headers['Content-Type'] = 'text/plain'
        return open(INPUT_FILEPATH, 'rU').read()
    NEXML_FILEPATH = os.path.join(working_dir, NEXML_FILENAME)
    block = True
    timeout_duration = 0.1 #@TEMPORARY should not be hard coded`
    #@TEMPORARY could be refactored into a launch_or_get_status() call
    status = invoc_status(request, working_dir)
    launched_this_call = False
    if status == ExternalProcStatus.NOT_FOUND:
        inp_format = request.vars.inputformat or 'nexus'
        inp_format = inp_format.lower()
        if inp_format not in input_choices:
            raise HTTP(400, 'inputformat should be one of: "{c}"'.format(c='", "'.join(input_choices)))
        if inp_format == 'newick':
            inp_format = 'relaxedphyliptree'
        if inp_format == 'nexml':
            shutil.copyfile(INPUT_FILEPATH, NEXML_FILEPATH)
        else:
            try:
                try:
                    exe_path = get_conf(request).get("external", "2nexml")
                except:
                    _LOG.warn("Config does not have external/2nexml setting")
                    raise
                assert(os.path.exists(exe_path))
            except:
                response.view = 'generic.json'; return {'hb':exe_path}
                _LOG.warn("Could not find the 2nexml executable")
                raise HTTP(501, T("Server is not configured to allow 2nexml conversion"))
            do_ext_proc_launch(request,
                               working_dir,
                               [exe_path, '-f{f}'.format(f=inp_format),'in.nex'],
                               NEXML_FILENAME,
                               ERR_FILENAME,
                               wait=block)
            if not block:
                time.sleep(timeout_duration)
            status = invoc_status(request, working_dir)
            assert(status != ExternalProcStatus.NOT_FOUND)
            launched_this_call = True
    if status == ExternalProcStatus.RUNNING:
        if not launched_this_call:
            time.sleep(timeout_duration)
            status = invoc_status(request, working_dir)
        if status == ExternalProcStatus.RUNNING:
            return HTTP(102, "Process still running")
    if status == ExternalProcStatus.FAILED:
        try:
            ERR_FILEPATH = os.path.join(working_dir, ERR_FILENAME)
            err_content = 'Error message:\n ' + open(ERR_FILEPATH, 'rU').read()
        except:
            err_content = ''
        response.headers['Content-Type'] = 'text/plain'
        raise HTTP(501, T("Conversion to NeXML failed.\n" + err_content))
    if output == 'nexml':
        response.headers['Content-Type'] = 'text/xml'
        return open(NEXML_FILEPATH, 'rU').read()
    NEXSON_FILENAME = 'nexson.json'
    NEXSON_FILEPATH = os.path.join(working_dir, NEXSON_FILENAME)
    NEXSON_DONE_FILEPATH = NEXSON_FILEPATH + '.written'
    NEXSON_LOCKFILEPATH = NEXSON_FILEPATH+ '.lock'
    if not os.path.exists(NEXSON_DONE_FILEPATH):
        try:
            with locket.lock_file(NEXSON_LOCKFILEPATH, timeout=0):
                if not os.path.exists(NEXSON_DONE_FILEPATH):
                    dfj = to_badgerfish_dict(NEXML_FILEPATH)
                    out = codecs.open(NEXSON_FILEPATH, 'w', encoding='utf-8')
                    json.dump(dfj, out, indent=0, sort_keys=True)
                    out.write('\n')
                    out.close()
                    out = open(NEXSON_DONE_FILEPATH, 'w')
                    out.write('0\n')
                    out.close()
        except locket.LockError:
            return HTTP(102, "Conversion to NexSON still running")
    if output == 'nexson':
        response.view = 'generic.json'
        return json.load(codecs.open(NEXSON_FILEPATH, 'rU', encoding='utf-8'))
    # Open Tree NeXSON - cull characters
    OT_NEXSON_FILENAME = 'ot_nexson.json'
    OT_NEXSON_FILEPATH = os.path.join(working_dir, OT_NEXSON_FILENAME)
    OT_NEXSON_DONE_FILEPATH = OT_NEXSON_FILEPATH + '.written'
    OT_NEXSON_LOCKFILEPATH = OT_NEXSON_FILEPATH+ '.lock'
    if not os.path.exists(OT_NEXSON_DONE_FILEPATH):
        try:
            with locket.lock_file(OT_NEXSON_LOCKFILEPATH, timeout=0):
                if not os.path.exists(OT_NEXSON_DONE_FILEPATH):
                    inf = codecs.open(NEXSON_FILEPATH, 'rU', encoding='utf-8')
                    full_nexson = json.load(inf)
                    cull_for_ot_nexson(full_nexson, data_deposit=request.vars.dataDeposit)
                    out = codecs.open(OT_NEXSON_FILEPATH, 'w', encoding='utf-8')
                    json.dump(full_nexson, out, indent=0, sort_keys=True)
                    out.write('\n')
                    out.close()
                    out = open(OT_NEXSON_DONE_FILEPATH, 'w')
                    out.write('0\n')
                    out.close()
        except locket.LockError:
            return HTTP(102, "Conversion to ot:NexSON still running")
    assert (output == 'ot:nexson')
    response.view = 'generic.json'
    return json.load(codecs.open(OT_NEXSON_FILEPATH, 'rU', encoding='utf-8'))
    <commit_msg>Watch for unwanted encoding of 'ot:nexson'<commit_after># -*- coding: utf-8 -*-
# this file is released under public domain and you can use without limitations
import re

#########################################################################
## This is a samples controller
## - index is the default action of any application
## - user is required for authentication and authorization
## - download is for downloading files uploaded in the db (does streaming)
## - call exposes all registered services (none by default)
#########################################################################

def index():
    """
    Show an introduction page for visitors, or personalized curation dashboard for
    a logged-in user.
    """
    #response.flash = T("Welcome to web2py!")

    if False:  ## auth.is_logged_in():
        # user is logged in, bounce to their personal dashboard
        redirect(URL('dashboard'))
    else:
        # anonymous visitor, show a general info page
        return dict()

def error():
    return dict()

@auth.requires_login()
def dashboard():
    return dict(message="My Curation Activity")

def user():
    """
    exposes:
    http://..../[app]/default/user/login
    http://..../[app]/default/user/logout
    http://..../[app]/default/user/register
    http://..../[app]/default/user/profile
    http://..../[app]/default/user/retrieve_password
    http://..../[app]/default/user/change_password
    use @auth.requires_login()
        @auth.requires_membership('group name')
        @auth.requires_permission('read','table name',record_id)
    to decorate functions that need access control
    """
    return dict(form=auth())

def download():
    """
    allows downloading of uploaded files
    http://..../[app]/default/download/[filename]
    """
    return response.download(request, db)

def call():
    """
    exposes services. for example:
    http://..../[app]/default/call/jsonrpc
    decorate with @services.jsonrpc the functions to expose
    supports xml, json, xmlrpc, jsonrpc, amfrpc, rss, csv
    """
    return service()

@auth.requires_signature()
def data():
    """
    http://..../[app]/default/data/tables
    http://..../[app]/default/data/create/[table]
    http://..../[app]/default/data/read/[table]/[id]
    http://..../[app]/default/data/update/[table]/[id]
    http://..../[app]/default/data/delete/[table]/[id]
    http://..../[app]/default/data/select/[table]
    http://..../[app]/default/data/search/[table]
    but URLs must be signed, i.e. linked with
      A('table',_href=URL('data/tables',user_signature=True))
    or with the signed load operator
      LOAD('default','data.load',args='tables',ajax=True,user_signature=True)
    """
    return dict(form=crud())

UPLOADID_PAT = re.compile(r'^[-_.a-zA-Z0-9]{5,85}$')
def to_nexml():
    global UPLOADID_PAT
    from externalproc import get_external_proc_dir_for_upload, get_logger, invoc_status, \
            ExternalProcStatus, get_conf, write_input_files, write_ext_proc_content, do_ext_proc_launch
    import os
    import datetime
    import codecs
    import locket
    import shutil
    from xml2jsonbadgerfish import to_badgerfish_dict, cull_for_ot_nexson
    '''
    Controller for conversion of NEXUS, newick, or NeXML to NeXSON
    Required arguments:
        "uploadid" - A unique string for this upload. It needs
            to match the pattern '^[-_.a-zA-Z0-9]{5,85}$'
            clash with a previously used ID will cause data from the
            previous upload to be returned.
        "file" should be a multipart-encoded file to be translated to NexSON
          OR
        "content" which is a string that contains the content of the file
            format. "content" is checked if "file" is not provided.
    Optional arguments:
        "output" one of ['ot:nexson', 'nexson', 'nexml', 'input', 'provenance']
            the default is ot:nexson. This specifies what is to be returned.
            Possible values are:
            "ot:nexson" is the Open Tree NexSON (with character data culled).
                This should be the first call to this uploadid. Subsequent
                calls can retrieve intermediates. JSON.
            "nexson" is the complete NexSON version of the data. JSON.
            "nexml" is the NeXML version of the file. This is an intermediate
                for the NexSON. XML.
            "input" returns the uploaded file. text/plain.
            "provenance" returns a simple, ad-hoc JSON with initial call details.
        "dataDeposit" should be a URL that should be added to the meta element
            in the Open Tree NexSON object.
        "inputformat" should be "nexus", "newick", or "nexml"
            default is "nexus"

    '''
    _LOG = get_logger(request, 'to_nexml')
    try:
        unique_id = request.vars.uploadid
        assert unique_id is not None
    except:
        raise HTTP(400, 'Expecting an "uploadid" argument with a unique ID for this upload')
    if not UPLOADID_PAT.match(unique_id):
        raise HTTP(400, 'uploadid must be series of letters, numbers, dots or dashes between 5 and 85 characters long. "{u}" does not match this pattern'.format(u=unique_id))
    output = request.vars.output or 'ot:nexson'
    output = output.lower()
    if output == 'ot%3anexson':
        # handle unwanted encoding (happens if file submitted)
        output = 'ot:nexson'
    output_choices = ['ot:nexson', 'nexson', 'nexml', 'input', 'provenance']
    if output not in output_choices:
        raise HTTP(400, 'The "output" should be one of: "{c}"'.format(c='", "'.join(output_choices)))
    try:
        working_dir = get_external_proc_dir_for_upload(request, '2nexml', unique_id)
    except Exception, x:
        raise HTTP(404)
    if working_dir is None or not os.path.exists(working_dir):
        raise HTTP(404)
    INPUT_FILENAME = 'in.nex'
    PROV_FILENAME = 'provenance.json'
    NEXML_FILENAME = 'out.xml'
    ERR_FILENAME = 'err.txt'

    INPUT_FILEPATH = os.path.join(working_dir, INPUT_FILENAME)
    INP_LOCKFILEPATH = os.path.join(working_dir, INPUT_FILENAME + '.lock')
    inpfp = os.path.join(working_dir, INPUT_FILENAME)
    input_choices = ['nexus', 'newick', 'nexml']
    with locket.lock_file(INP_LOCKFILEPATH):
        if not os.path.exists(INPUT_FILEPATH):
            if output != 'ot:nexson':
                raise HTTP(400, 'The "output" argument should be "ot:nexson" in the first call with each "uploadid"')
            if request.vars.file is not None:
                upf = request.vars.file
                upload_stream = upf.file
                filename = upf.filename
            elif request.vars.content is not None:
                upload_stream = request.vars.content # stream is a bad name, but write_input_files does the write thing.
                filename = '<content provided as a string in a "content" rather than a file upload>'
            else:
                raise HTTP(400, 'Expecting a "file" argument with an input file or a "content" argument with the contents of in input file')
            inp_format = request.vars.inputformat or 'nexus'
            inp_format = inp_format.lower()
            if inp_format not in input_choices:
                raise HTTP(400, 'inputformat should be one of: "{c}"'.format(c='", "'.join(input_choices)))
            write_input_files(request, working_dir, [(INPUT_FILENAME, upload_stream)])
            prov_info = {
                'filename' : filename,
                'date-created': datetime.datetime.utcnow().isoformat(),
            }
            if request.vars.dataDeposit:
                prov_info['data-deposit'] = request.vars.dataDeposit
            write_ext_proc_content(request,
                                   working_dir,
                                   [(PROV_FILENAME, json.dumps(prov_info))],
                                   encoding='utf-8')
    if output == 'provenance':
        PROV_FILEPATH =  os.path.join(working_dir, PROV_FILENAME)
        response.view = 'generic.json'
        return json.load(codecs.open(PROV_FILEPATH, 'rU', encoding='utf-8'))
    if output == 'input':
        response.headers['Content-Type'] = 'text/plain'
        return open(INPUT_FILEPATH, 'rU').read()
    NEXML_FILEPATH = os.path.join(working_dir, NEXML_FILENAME)
    block = True
    timeout_duration = 0.1 #@TEMPORARY should not be hard coded`
    #@TEMPORARY could be refactored into a launch_or_get_status() call
    status = invoc_status(request, working_dir)
    launched_this_call = False
    if status == ExternalProcStatus.NOT_FOUND:
        inp_format = request.vars.inputformat or 'nexus'
        inp_format = inp_format.lower()
        if inp_format not in input_choices:
            raise HTTP(400, 'inputformat should be one of: "{c}"'.format(c='", "'.join(input_choices)))
        if inp_format == 'newick':
            inp_format = 'relaxedphyliptree'
        if inp_format == 'nexml':
            shutil.copyfile(INPUT_FILEPATH, NEXML_FILEPATH)
        else:
            try:
                try:
                    exe_path = get_conf(request).get("external", "2nexml")
                except:
                    _LOG.warn("Config does not have external/2nexml setting")
                    raise
                assert(os.path.exists(exe_path))
            except:
                response.view = 'generic.json'; return {'hb':exe_path}
                _LOG.warn("Could not find the 2nexml executable")
                raise HTTP(501, T("Server is not configured to allow 2nexml conversion"))
            do_ext_proc_launch(request,
                               working_dir,
                               [exe_path, '-f{f}'.format(f=inp_format),'in.nex'],
                               NEXML_FILENAME,
                               ERR_FILENAME,
                               wait=block)
            if not block:
                time.sleep(timeout_duration)
            status = invoc_status(request, working_dir)
            assert(status != ExternalProcStatus.NOT_FOUND)
            launched_this_call = True
    if status == ExternalProcStatus.RUNNING:
        if not launched_this_call:
            time.sleep(timeout_duration)
            status = invoc_status(request, working_dir)
        if status == ExternalProcStatus.RUNNING:
            return HTTP(102, "Process still running")
    if status == ExternalProcStatus.FAILED:
        try:
            ERR_FILEPATH = os.path.join(working_dir, ERR_FILENAME)
            err_content = 'Error message:\n ' + open(ERR_FILEPATH, 'rU').read()
        except:
            err_content = ''
        response.headers['Content-Type'] = 'text/plain'
        raise HTTP(501, T("Conversion to NeXML failed.\n" + err_content))
    if output == 'nexml':
        response.headers['Content-Type'] = 'text/xml'
        return open(NEXML_FILEPATH, 'rU').read()
    NEXSON_FILENAME = 'nexson.json'
    NEXSON_FILEPATH = os.path.join(working_dir, NEXSON_FILENAME)
    NEXSON_DONE_FILEPATH = NEXSON_FILEPATH + '.written'
    NEXSON_LOCKFILEPATH = NEXSON_FILEPATH+ '.lock'
    if not os.path.exists(NEXSON_DONE_FILEPATH):
        try:
            with locket.lock_file(NEXSON_LOCKFILEPATH, timeout=0):
                if not os.path.exists(NEXSON_DONE_FILEPATH):
                    dfj = to_badgerfish_dict(NEXML_FILEPATH)
                    out = codecs.open(NEXSON_FILEPATH, 'w', encoding='utf-8')
                    json.dump(dfj, out, indent=0, sort_keys=True)
                    out.write('\n')
                    out.close()
                    out = open(NEXSON_DONE_FILEPATH, 'w')
                    out.write('0\n')
                    out.close()
        except locket.LockError:
            return HTTP(102, "Conversion to NexSON still running")
    if output == 'nexson':
        response.view = 'generic.json'
        return json.load(codecs.open(NEXSON_FILEPATH, 'rU', encoding='utf-8'))
    # Open Tree NeXSON - cull characters
    OT_NEXSON_FILENAME = 'ot_nexson.json'
    OT_NEXSON_FILEPATH = os.path.join(working_dir, OT_NEXSON_FILENAME)
    OT_NEXSON_DONE_FILEPATH = OT_NEXSON_FILEPATH + '.written'
    OT_NEXSON_LOCKFILEPATH = OT_NEXSON_FILEPATH+ '.lock'
    if not os.path.exists(OT_NEXSON_DONE_FILEPATH):
        try:
            with locket.lock_file(OT_NEXSON_LOCKFILEPATH, timeout=0):
                if not os.path.exists(OT_NEXSON_DONE_FILEPATH):
                    inf = codecs.open(NEXSON_FILEPATH, 'rU', encoding='utf-8')
                    full_nexson = json.load(inf)
                    cull_for_ot_nexson(full_nexson, data_deposit=request.vars.dataDeposit)
                    out = codecs.open(OT_NEXSON_FILEPATH, 'w', encoding='utf-8')
                    json.dump(full_nexson, out, indent=0, sort_keys=True)
                    out.write('\n')
                    out.close()
                    out = open(OT_NEXSON_DONE_FILEPATH, 'w')
                    out.write('0\n')
                    out.close()
        except locket.LockError:
            return HTTP(102, "Conversion to ot:NexSON still running")
    assert (output == 'ot:nexson')
    response.view = 'generic.json'
    return json.load(codecs.open(OT_NEXSON_FILEPATH, 'rU', encoding='utf-8'))

<|endoftext|>


<commit_before>from Products.CMFPlone.utils import getToolByName
from bda.plone.cart import readcookie
from bda.plone.checkout import CheckoutDone
from bda.plone.checkout.interfaces import CheckoutError
from bda.plone.checkout.interfaces import ICheckoutAdapter
from bda.plone.checkout.interfaces import IFieldsProvider
from bda.plone.checkout.vocabularies import country_vocabulary
from bda.plone.checkout.vocabularies import gender_vocabulary
from bda.plone.payment import Payments
from bda.plone.shipping import Shippings
from yafowil.base import ExtractionError
from yafowil.base import UNSET
from yafowil.base import factory
from yafowil.plone.form import Form
from yafowil.yaml import parse_from_YAML
from zExceptions import Redirect
from zope.component import getMultiAdapter
from zope.event import notify
from zope.i18n import translate
from zope.i18nmessageid import MessageFactory
from zope.interface import implementer

import transaction

_ = MessageFactory('bda.plone.checkout')

def get_prop_from_member(member, name, prefix=None):
    """Get property from memberdata.
    If a prefix is given, add it to the name to get the value from the
    property sheet.
    """
    default = UNSET
    if member:
        if prefix:
            name = '%s%s' % (prefix, name)
        default = member.getProperty(name, None)
    return default

class ProviderRegistry(object):

    def __init__(self):
        self.providers = list()

    def add(self, factory):
        self.providers.append(factory)

    def __iter__(self):
        return self.providers.__iter__()

provider_registry = ProviderRegistry()

CHECKOUT = 0
CONFIRM = 1

class FormContext(object):

    @property
    def form_context(self):
        confirm = self.request.get('checkout_confirm') or\
            self.request.get('action.checkout.finish')
        return confirm and CONFIRM or CHECKOUT

    @property
    def mode(self):
        return self.form_context is CONFIRM and 'display' or 'edit'

@implementer(IFieldsProvider)
class FieldsProvider(FormContext):
    fields_template = None
    fields_name = ''
    message_factory = _
    ignore_on_save = False
    memberdata_prefix = None

    def __init__(self, context, request):
        self.context = context
        self.request = request
        member = None
        membership = getToolByName(self.context, 'portal_membership', None)
        if membership and not membership.isAnonymousUser():
            member = membership.getAuthenticatedMember()
        self.member = member

    def get_value(self, widget, data):
        default = get_prop_from_member(self.member, widget.name,
                                       prefix=self.memberdata_prefix)
        ret = None
        if 'checkbox' in widget.blueprints:
            # for selected checkboxes, not the value but only the input name
            # (dottedpath) is in request.form, otherwise they are completly
            # ommited.
            ret = widget.dottedpath in self.request or default
        else:
            ret = self.request.get(widget.dottedpath, default)
        return ret

    def extend(self, form):
        fields = parse_from_YAML(self.fields_template,
                                 self, self.message_factory)
        form[self.fields_name] = fields

class CartSummary(FieldsProvider):
    fields_name = 'cart_summary'

    def extend(self, form):
        if not self.form_context == CONFIRM:
            return
        compound = form[self.fields_name] = factory('compound', props={
            'structural': True})
        compound['heading'] = factory('tag', props={
            'structural': True,
            'tag': 'h2',
            'text': _('heading_cart_summary', 'Cart')})
        compound['overview'] = factory('tag', props={
            'structural': True,
            'class': 'cart_overview',
            'tag': 'div',
            'text': self.context.restrictedTraverse('@@cart_overview')()})

provider_registry.add(CartSummary)

class PersonalData(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/personal_data.yaml'
    fields_name = 'personal_data'

    @property
    def gender_vocabulary(self):
        return gender_vocabulary()

provider_registry.add(PersonalData)

class BillingAddress(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/billing_address.yaml'
    fields_name = 'billing_address'

    @property
    def country_vocabulary(self):
        return country_vocabulary()

provider_registry.add(BillingAddress)

class DeliveryAddress(BillingAddress):
    fields_template = 'bda.plone.checkout.browser:forms/delivery_address.yaml'
    fields_name = 'delivery_address'
    memberdata_prefix = 'delivery_'

    def conditional_required(self, widget, data):
        if data.parent['alternative_delivery'].extracted\
                and not data.extracted:
            raise ExtractionError(widget.attrs['conditional_required'])
        return data.extracted

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.delivery_address.alternative_delivery'
        if self.request.get(name) is None:
            return 'hidden'
        return ''

    @property
    def alternative_delivery_vocab(self):
        return {
            True: _('yes', 'Yes'),
            False: _('no', 'No'),
            UNSET: _('not set', 'not set'),
        }

provider_registry.add(DeliveryAddress)

class ShippingSelection(FieldsProvider):
    fields_template =\
        'bda.plone.checkout.browser:forms/shipping_selection.yaml'
    fields_name = 'shipping_selection'

    @property
    def shippings(self):
        return Shippings(self.context)

    @property
    def shipping_vocabulary(self):
        return self.shippings.vocab

    def get_shipping(self, widget, data):
        return self.request.get(widget.dottedpath, self.shippings.default)

provider_registry.add(ShippingSelection)

class PaymentSelection(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/payment_selection.yaml'
    fields_name = 'payment_selection'

    @property
    def payments(self):
        return Payments(self.context)

    @property
    def payment_vocabulary(self):
        return self.payments.vocab

    def get_payment(self, widget, data):
        return self.request.get(widget.dottedpath, self.payments.default)

provider_registry.add(PaymentSelection)

class OrderComment(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/order_comment.yaml'
    fields_name = 'order_comment'

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.order_comment.comment'
        if not self.request.get(name):
            return 'hidden'
        return ''

provider_registry.add(OrderComment)

class AcceptTermsAndConditions(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/accept_terms.yaml'
    fields_name = 'accept_terms_and_conditions'
    ignore_on_save = True

    @property
    def accept_label(self):
        # XXX: url from config
        tac_url = '%s/agb' % self.context.absolute_url()
        tac_label = _('terms_and_conditions', 'Terms and conditions')
        tac_label = translate(tac_label, context=self.request)
        tac_link = '%s'
        tac_link = tac_link % (tac_url, tac_label)
        tac_text = _('terms_and_conditions_text',
                     'I have read and accept the ${terms_and_conditions}',
                     mapping={'terms_and_conditions': tac_link})
        return tac_text

    @property
    def mode(self):
        if self.request.get('action.checkout.finish')\
                or self.form_context is CONFIRM:
            return 'edit'
        return 'skip'

    def validate_accept(self, widget, data):
        if not data.extracted:
            raise ExtractionError(_('error_accept_terms_and_conditiond',
                                    'Please accept our terms and conditions.'))
        return data.extracted

provider_registry.add(AcceptTermsAndConditions)

class CheckoutForm(Form, FormContext):
    action_resource = '@@checkout'
    # in order to provide your own registry, subclass CheckoutForm and and
    # override ``provider_registry``
    provider_registry = provider_registry

    def prepare(self):
        if not readcookie(self.request):
            raise Redirect(self.context.absolute_url())
        checkout = self.form_context is CHECKOUT
        form_class = checkout and 'mode_edit' or 'mode_display'
        self.form = factory('#form', name='checkout', props={
            'action': self.form_action,
            'class_add': form_class})
        for fields_factory in self.provider_registry:
            fields_factory(self.context, self.request).extend(self.form)
        # checkout data input
        if checkout:
            self.form['checkout_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'checkout_back',
                'handler': None,
                'next': self.checkout_back,
                'skip': True})
            self.form['next'] = factory('submit', props={
                'label': _('next', 'Next'),
                'action': 'next',
                'handler': None,
                'next': self.checkout_summary})
        # checkout confirmation
        else:
            self.form['confirm_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'confirm_back',
                'handler': None,
                'next': self.confirm_back})
            self.form['finish'] = factory('submit', props={
                'class': 'prevent_if_no_longer_available',
                'label': _('finish', 'Order now'),
                'action': 'finish',
                'handler': self.finish,
                'next': self.checkout_done})

    def checkout_back(self, request):
        raise Redirect('%s/@@cart' % self.context.absolute_url())

    def confirm_back(self, request):
        self.prepare()
        return self.form(request=request)

    def checkout_summary(self, request):
        self.request['checkout_confirm'] = '1'
        self.prepare()
        return self.form(request=request)

    def checkout_done(self, request):
        transaction.commit()
        raise Redirect(self.finish_redirect_url)

    def finish(self, widget, data):
        providers = [fields_factory(self.context, self.request)
                     for fields_factory in self.provider_registry]
        to_adapt = (self.context, self.request)
        checkout_adapter = getMultiAdapter(to_adapt, ICheckoutAdapter)
        try:
            uid = checkout_adapter.save(providers, widget, data)
        except CheckoutError:
            transaction.abort()
            self.checkout_back(self.request)
        checkout_adapter.clear_session()
        if checkout_adapter.skip_payment:
            self.finish_redirect_url = \
                checkout_adapter.skip_payment_redirect_url
        else:
            p_name = data.fetch('checkout.payment_selection.payment').extracted
            payments = Payments(self.context)
            payment = payments.get(p_name)
            self.finish_redirect_url = payment.init_url(str(uid))
        event = CheckoutDone(self.context, self.request, uid)
        notify(event)
<commit_msg>import central message_factory<commit_after>from Products.CMFPlone.utils import getToolByName
from bda.plone.cart import readcookie
from bda.plone.checkout import CheckoutDone
from bda.plone.checkout import message_factory as _
from bda.plone.checkout.interfaces import CheckoutError
from bda.plone.checkout.interfaces import ICheckoutAdapter
from bda.plone.checkout.interfaces import IFieldsProvider
from bda.plone.checkout.vocabularies import country_vocabulary
from bda.plone.checkout.vocabularies import gender_vocabulary
from bda.plone.payment import Payments
from bda.plone.shipping import Shippings
from yafowil.base import ExtractionError
from yafowil.base import UNSET
from yafowil.base import factory
from yafowil.plone.form import Form
from yafowil.yaml import parse_from_YAML
from zExceptions import Redirect
from zope.component import getMultiAdapter
from zope.event import notify
from zope.i18n import translate
from zope.interface import implementer

import transaction

def get_prop_from_member(member, name, prefix=None):
    """Get property from memberdata.
    If a prefix is given, add it to the name to get the value from the
    property sheet.
    """
    default = UNSET
    if member:
        if prefix:
            name = '%s%s' % (prefix, name)
        default = member.getProperty(name, None)
    return default

class ProviderRegistry(object):

    def __init__(self):
        self.providers = list()

    def add(self, factory):
        self.providers.append(factory)

    def __iter__(self):
        return self.providers.__iter__()

provider_registry = ProviderRegistry()

CHECKOUT = 0
CONFIRM = 1

class FormContext(object):

    @property
    def form_context(self):
        confirm = self.request.get('checkout_confirm') or\
            self.request.get('action.checkout.finish')
        return confirm and CONFIRM or CHECKOUT

    @property
    def mode(self):
        return self.form_context is CONFIRM and 'display' or 'edit'

@implementer(IFieldsProvider)
class FieldsProvider(FormContext):
    fields_template = None
    fields_name = ''
    message_factory = _
    ignore_on_save = False
    memberdata_prefix = None

    def __init__(self, context, request):
        self.context = context
        self.request = request
        member = None
        membership = getToolByName(self.context, 'portal_membership', None)
        if membership and not membership.isAnonymousUser():
            member = membership.getAuthenticatedMember()
        self.member = member

    def get_value(self, widget, data):
        default = get_prop_from_member(self.member, widget.name,
                                       prefix=self.memberdata_prefix)
        ret = None
        if 'checkbox' in widget.blueprints:
            # for selected checkboxes, not the value but only the input name
            # (dottedpath) is in request.form, otherwise they are completly
            # ommited.
            ret = widget.dottedpath in self.request or default
        else:
            ret = self.request.get(widget.dottedpath, default)
        return ret

    def extend(self, form):
        fields = parse_from_YAML(self.fields_template,
                                 self, self.message_factory)
        form[self.fields_name] = fields

class CartSummary(FieldsProvider):
    fields_name = 'cart_summary'

    def extend(self, form):
        if not self.form_context == CONFIRM:
            return
        compound = form[self.fields_name] = factory('compound', props={
            'structural': True})
        compound['heading'] = factory('tag', props={
            'structural': True,
            'tag': 'h2',
            'text': _('heading_cart_summary', 'Cart')})
        compound['overview'] = factory('tag', props={
            'structural': True,
            'class': 'cart_overview',
            'tag': 'div',
            'text': self.context.restrictedTraverse('@@cart_overview')()})

provider_registry.add(CartSummary)

class PersonalData(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/personal_data.yaml'
    fields_name = 'personal_data'

    @property
    def gender_vocabulary(self):
        return gender_vocabulary()

provider_registry.add(PersonalData)

class BillingAddress(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/billing_address.yaml'
    fields_name = 'billing_address'

    @property
    def country_vocabulary(self):
        return country_vocabulary()

provider_registry.add(BillingAddress)

class DeliveryAddress(BillingAddress):
    fields_template = 'bda.plone.checkout.browser:forms/delivery_address.yaml'
    fields_name = 'delivery_address'
    memberdata_prefix = 'delivery_'

    def conditional_required(self, widget, data):
        if data.parent['alternative_delivery'].extracted\
                and not data.extracted:
            raise ExtractionError(widget.attrs['conditional_required'])
        return data.extracted

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.delivery_address.alternative_delivery'
        if self.request.get(name) is None:
            return 'hidden'
        return ''

    @property
    def alternative_delivery_vocab(self):
        return {
            True: _('yes', 'Yes'),
            False: _('no', 'No'),
            UNSET: _('not set', 'not set'),
        }

provider_registry.add(DeliveryAddress)

class ShippingSelection(FieldsProvider):
    fields_template =\
        'bda.plone.checkout.browser:forms/shipping_selection.yaml'
    fields_name = 'shipping_selection'

    @property
    def shippings(self):
        return Shippings(self.context)

    @property
    def shipping_vocabulary(self):
        return self.shippings.vocab

    def get_shipping(self, widget, data):
        return self.request.get(widget.dottedpath, self.shippings.default)

provider_registry.add(ShippingSelection)

class PaymentSelection(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/payment_selection.yaml'
    fields_name = 'payment_selection'

    @property
    def payments(self):
        return Payments(self.context)

    @property
    def payment_vocabulary(self):
        return self.payments.vocab

    def get_payment(self, widget, data):
        return self.request.get(widget.dottedpath, self.payments.default)

provider_registry.add(PaymentSelection)

class OrderComment(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/order_comment.yaml'
    fields_name = 'order_comment'

    @property
    def hidden_class(self):
        if self.form_context is CHECKOUT:
            return ''
        name = 'checkout.order_comment.comment'
        if not self.request.get(name):
            return 'hidden'
        return ''

provider_registry.add(OrderComment)

class AcceptTermsAndConditions(FieldsProvider):
    fields_template = 'bda.plone.checkout.browser:forms/accept_terms.yaml'
    fields_name = 'accept_terms_and_conditions'
    ignore_on_save = True

    @property
    def accept_label(self):
        # XXX: url from config
        tac_url = '%s/agb' % self.context.absolute_url()
        tac_label = _('terms_and_conditions', 'Terms and conditions')
        tac_label = translate(tac_label, context=self.request)
        tac_link = '%s'
        tac_link = tac_link % (tac_url, tac_label)
        tac_text = _('terms_and_conditions_text',
                     'I have read and accept the ${terms_and_conditions}',
                     mapping={'terms_and_conditions': tac_link})
        return tac_text

    @property
    def mode(self):
        if self.request.get('action.checkout.finish')\
                or self.form_context is CONFIRM:
            return 'edit'
        return 'skip'

    def validate_accept(self, widget, data):
        if not data.extracted:
            raise ExtractionError(_('error_accept_terms_and_conditiond',
                                    'Please accept our terms and conditions.'))
        return data.extracted

provider_registry.add(AcceptTermsAndConditions)

class CheckoutForm(Form, FormContext):
    action_resource = '@@checkout'
    # in order to provide your own registry, subclass CheckoutForm and and
    # override ``provider_registry``
    provider_registry = provider_registry

    def prepare(self):
        if not readcookie(self.request):
            raise Redirect(self.context.absolute_url())
        checkout = self.form_context is CHECKOUT
        form_class = checkout and 'mode_edit' or 'mode_display'
        self.form = factory('#form', name='checkout', props={
            'action': self.form_action,
            'class_add': form_class})
        for fields_factory in self.provider_registry:
            fields_factory(self.context, self.request).extend(self.form)
        # checkout data input
        if checkout:
            self.form['checkout_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'checkout_back',
                'handler': None,
                'next': self.checkout_back,
                'skip': True})
            self.form['next'] = factory('submit', props={
                'label': _('next', 'Next'),
                'action': 'next',
                'handler': None,
                'next': self.checkout_summary})
        # checkout confirmation
        else:
            self.form['confirm_back'] = factory('submit', props={
                'label': _('back', 'Back'),
                'action': 'confirm_back',
                'handler': None,
                'next': self.confirm_back})
            self.form['finish'] = factory('submit', props={
                'class': 'prevent_if_no_longer_available',
                'label': _('finish', 'Order now'),
                'action': 'finish',
                'handler': self.finish,
                'next': self.checkout_done})

    def checkout_back(self, request):
        raise Redirect('%s/@@cart' % self.context.absolute_url())

    def confirm_back(self, request):
        self.prepare()
        return self.form(request=request)

    def checkout_summary(self, request):
        self.request['checkout_confirm'] = '1'
        self.prepare()
        return self.form(request=request)

    def checkout_done(self, request):
        transaction.commit()
        raise Redirect(self.finish_redirect_url)

    def finish(self, widget, data):
        providers = [fields_factory(self.context, self.request)
                     for fields_factory in self.provider_registry]
        to_adapt = (self.context, self.request)
        checkout_adapter = getMultiAdapter(to_adapt, ICheckoutAdapter)
        try:
            uid = checkout_adapter.save(providers, widget, data)
        except CheckoutError:
            transaction.abort()
            self.checkout_back(self.request)
        checkout_adapter.clear_session()
        if checkout_adapter.skip_payment:
            self.finish_redirect_url = \
                checkout_adapter.skip_payment_redirect_url
        else:
            p_name = data.fetch('checkout.payment_selection.payment').extracted
            payments = Payments(self.context)
            payment = payments.get(p_name)
            self.finish_redirect_url = payment.init_url(str(uid))
        event = CheckoutDone(self.context, self.request, uid)
        notify(event)
<|endoftext|>