<commit_before>"""
Django settings for amweekly project.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

import dj_database_url
from decouple import config

BASE_DIR = os.path.dirname(os.path.dirname(__file__))
PROJECT_ROOT = os.path.dirname(BASE_DIR)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default=False, cast=bool)

ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='',
                       cast=lambda v: [s.strip() for s in v.split(',')])

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'import_export',

    'amweekly.apps.AmweeklyConfig',
    'amweekly.shares',
)

# FIXME https://github.com/jazzband/django-debug-toolbar/issues/853
# if DEBUG:
#     INSTALLED_APPS += ('debug_toolbar',)

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'amweekly.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
            'debug': config('TEMPLATE_DEBUG', default=DEBUG, cast=bool),
        },
    },
]

WSGI_APPLICATION = 'amweekly.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(conn_max_age=600),
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'public', 'static')

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Media

MEDIA_ROOT = os.path.join(BASE_DIR, 'public', 'media')

# SSL

SECURE_SSL_REDIRECT = config('SECURE_SSL_REDIRECT', default=True, cast=bool)
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
SESSION_COOKIE_SECURE = SECURE_SSL_REDIRECT
CSRF_COOKIE_SECURE = SECURE_SSL_REDIRECT

# redis

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': config('REDIS_URL', default='redis://localhost:6379/0'),
        'OPTIONS': {
            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
        }
    },
}

# rq

INSTALLED_APPS += ('django_rq',)

RQ_QUEUES = {
    'default': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
    'high': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
    'low': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
}
RQ_SHOW_ADMIN_LINK = True

# Sentry

SENTRY_DSN = config('SENTRY_DSN', default=None)

if SENTRY_DSN:
    INSTALLED_APPS += ('raven.contrib.django.raven_compat',)
    RAVEN_CONFIG = {
        'dsn': SENTRY_DSN,
    }

# Facebook

FACEBOOK_CLIENT_ID=config('FACEBOOK_CLIENT_ID', default='')
FACEBOOK_CLIENT_SECRET=config('FACEBOOK_CLIENT_SECRET', default='')
<commit_msg>change comments<commit_after>"""
Django settings for amweekly project.

For more information on this file, see
https://docs.djangoproject.com/en/1.10/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.10/ref/settings/
"""

import os

import dj_database_url
from decouple import config

BASE_DIR = os.path.dirname(os.path.dirname(__file__))
PROJECT_ROOT = os.path.dirname(BASE_DIR)

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default=False, cast=bool)

ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='',
                       cast=lambda v: [s.strip() for s in v.split(',')])

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',

    'import_export',

    'amweekly.apps.AmweeklyConfig',
    'amweekly.shares',
)

# FIXME https://github.com/jazzband/django-debug-toolbar/issues/853
# if DEBUG:
#     INSTALLED_APPS += ('debug_toolbar',)

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'amweekly.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
            'debug': config('TEMPLATE_DEBUG', default=DEBUG, cast=bool),
        },
    },
]

WSGI_APPLICATION = 'amweekly.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(conn_max_age=600),
}

# Internationalization
# https://docs.djangoproject.com/en/1.10/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'public', 'static')

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Media

MEDIA_ROOT = os.path.join(BASE_DIR, 'public', 'media')

# SSL

SECURE_SSL_REDIRECT = config('SECURE_SSL_REDIRECT', default=True, cast=bool)
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
SESSION_COOKIE_SECURE = SECURE_SSL_REDIRECT
CSRF_COOKIE_SECURE = SECURE_SSL_REDIRECT

# redis

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
CACHES = {
    'default': {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': config('REDIS_URL', default='redis://localhost:6379/0'),
        'OPTIONS': {
            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
        }
    },
}

# rq

INSTALLED_APPS += ('django_rq',)

RQ_QUEUES = {
    'default': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
    'high': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
    'low': {
        'URL': config('REDIS_URL', default='redis://localhost:6379/0'),
    },
}
RQ_SHOW_ADMIN_LINK = True

# Sentry

SENTRY_DSN = config('SENTRY_DSN', default=None)

if SENTRY_DSN:
    INSTALLED_APPS += ('raven.contrib.django.raven_compat',)
    RAVEN_CONFIG = {
        'dsn': SENTRY_DSN,
    }

# Facebook

FACEBOOK_CLIENT_ID=config('FACEBOOK_CLIENT_ID', default='')
FACEBOOK_CLIENT_SECRET=config('FACEBOOK_CLIENT_SECRET', default='')
<|endoftext|>


<commit_before>import os
import json
import time, datetime
from backend.app import app, db

from backend import models
from backend.models import *
import parsers
from sqlalchemy import types

import requests

import logging
logger = logging.getLogger(__name__)
logging.getLogger('sqlalchemy.engine').level = logging.WARN

STATIC_HOST = app.config['STATIC_HOST']
db.echo = False

def strip_filpath(filepath):

    return "/".join(filepath.split("/")[1::])

def dump_db(name):
    try:
        os.remove(name)
    except Exception as e:
        logger.debug(e)
        pass
    return

def read_data(filename):
    start = time.time()

    logger.debug("reading " + filename)
    with open('data/' + filename, 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            records.append(json.loads(line))
    return records

def db_date_from_utime(utime):
    try:
        return datetime.datetime.fromtimestamp(float(utime))
    except:
        try:
            datetime.strptime(utime, '%Y-%m-%dT%H:%m:%s')
        except:
            return None

def construct_obj(obj, mappings):
    """
    Returns a result with the properties mapped by mapping, including all revisions
    """
    result_obj = {}
    # print obj
    for key in mappings.keys():
        if (obj.has_key(key)):
            result_obj[key] = obj[key]
    if (obj.has_key("revisions") and (type(obj["revisions"]) is list) and (len(obj["revisions"]) > 0)):
        for revision in obj["revisions"]:
            tmp = construct_obj(revision, mappings)
            result_obj.update(tmp)
    return result_obj

def find_files(obj):
    files = []
    if (obj.has_key("files") and (type(obj["files"]) is list) and (len(obj["files"]) > 0)):
        # print obj["files"]
        for f in obj["files"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "http://eu-west-1-pmg.s3-website-eu-west-1.amazonaws.com/" + f["filepath"],
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    if (obj.has_key("audio") and (type(obj["audio"]) is list) and (len(obj["audio"]) > 0)):
        # print obj["audio"]
        for f in obj["audio"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "http://eu-west-1-pmg.s3-website-eu-west-1.amazonaws.com/audio/" + f["filename"],
                    playtime = f["playtime"],
                    description = f["title_format"]
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    return files

def find_committee(obj):
    organisations = []
    if (obj.has_key("terms") and (type(obj["terms"]) is list) and (len(obj["terms"]) > 0)):
        # print obj["terms"]
        for term in obj["terms"]:
            organisation = Organisation.query.filter_by(name = term).first()
            # print organisation, term
            if organisation:
                organisations.append(organisation)
            # else:
            #     print term
    return organisations

def prep_table(tablename):
    Model = getattr(models, tablename.capitalize())
    print "Deleted rows: ", Model.query.delete()

def rebuild_table(tablename, mappings):
    logger.debug("Rebuilding %s" % tablename)
    Model = getattr(models, tablename.capitalize())
    prep_table(tablename)
    i = 0
    with open('data/' + tablename + '.json', 'r') as f:
        records = []
        lines = f.readlines()
        logger.debug("Found %i records" % (len(lines)))
        for line in lines:
            obj = json.loads(line)
            newobj = construct_obj(obj, mappings)
            # Check for Dates
            for mapping in mappings.keys():
                row_type = getattr(Model, mapping).property.columns[0].type
                if (row_type.__str__() == "DATE"):
                    if (newobj.has_key(mappings[mapping])):
                        newobj[mappings[mapping]] = db_date_from_utime(newobj[mappings[mapping]])
            model = Model()
            files = find_files(obj)
            committees = find_committee(obj)
            if (committees):
                for committee in committees:
                    model.committee.append(committee)
            if (len(files)):
                for f in files:
                    model.files.append(f)
            for key,val in newobj.iteritems():
                setattr(model, key, val)
            db.session.add(model)
            i += 1
            if (i == 100):
                db.session.commit()
                i = 0
                logger.debug("Wrote 100 rows...")
        db.session.commit()

def rebuild_db():
    """
    Save json fixtures into a structured database, intended for use in our app.
    """

    db.drop_all()
    db.create_all()

    start = time.time()

    # populate houses of parliament
    joint_obj = House(name="Joint (NA + NCOP)", name_short="Joint")
    ncop_obj = House(name="National Council of Provinces", name_short="NCOP")
    na_obj = House(name="National Assembly", name_short="NA")
    for obj in [joint_obj, ncop_obj, na_obj]:
        db.session.add(obj)
    db.session.commit()

    # populate membership_type
    chairperson = MembershipType(name="chairperson")
    db.session.add(chairperson)
    db.session.commit()

    # populate committees
    committees = {}
    drupal_recs = read_data('comm_info_page.json')
    for rec in drupal_recs:
        if rec['terms']:
            name = rec['terms'][0].strip()
            if not committees.get('name'):
                committees[name] = {}
            if rec['comm_info_type'] == '"Contact"':
                committees[name]["contact"] = rec["revisions"][0]["body"]
            elif rec['comm_info_type'] == '"About"':
                committees[name]["about"] = rec["revisions"][0]["body"]
            if len(rec["revisions"]) > 1:
                logger.debug("ERROR: MULTIPLE REVISIONS PRESENT FOR A COMMITTEE")
    for key, val in committees.iteritems():
        logger.debug(key)
        organisation = Organisation()
        organisation.name = key
        organisation.type = "committee"
        organisation.version = 0

        # set parent relation
        if "ncop" in organisation.name.lower():
            organisation.house_id = ncop_obj.id
        elif "joint" in organisation.name.lower():
            organisation.house_id = joint_obj.id
        else:
            organisation.house_id = na_obj.id

        committee_info = CommitteeInfo()
        if val.get("about"):
            committee_info.about = val["about"]
        if val.get("contact"):
            committee_info.contact_details = val["contact"]
        committee_info.organization = organisation

        db.session.add(organisation)
        db.session.add(committee_info)
        val['model'] = organisation
    db.session.commit()

    # populate committee members
    members = read_data('committee_member.json')
    for member in members:
        member_obj = Member(
            name=member['title'].strip(),
            version=0,
            start_date = db_date_from_utime(member['start_date'])
        )
        if member.get('files'):
            # logger.debug(json.dumps(member['files'], indent=4)
            member_obj.profile_pic_url = STATIC_HOST + strip_filpath(member["files"][-1]['filepath'])
        logger.debug(member_obj.name)
        logger.debug(member_obj.profile_pic_url)

        # extract bio info
        if member['revisions']:
            bio = member['revisions'][0]['body']
            if bio:
                index = bio.find("Further information will be provided shortly on:")
                if index and index > 0:
                    bio = bio[0:index].strip()
                    logger.debug(bio)
                    member_obj.bio = bio

        # set committee relationships
        for term in member['terms']:
            if committees.get(term):
                org_model = committees[term]['model']
                membership_obj = Membership(organisation=org_model)
                member_obj.memberships.append(membership_obj)
            else:
                logger.debug("committee not found: " + term)

        # set party membership
        party = member['mp_party']
        if party:
            logger.debug(party)
            party_obj = Party.query.filter_by(name=party).first()
            if not party_obj:
                party_obj = Party(name=party)
                db.session.add(party_obj)
                db.session.commit()
            member_obj.party_id = party_obj.id

        # set house membership
        house = member['mp_province']
        if house == "National Assembly":
            member_obj.house_id = na_obj.id
        elif house and "NCOP" in house:
            member_obj.house_id = ncop_obj.id
            # logger.debug(house)
            province_obj = Province.query.filter_by(name=house[5::]).first()
            if not province_obj:
                province_obj = Province(name=house[5::])
                db.session.add(province_obj)
                db.session.commit()
            member_obj.province_id = province_obj.id

        db.session.add(member_obj)
        logger.debug('')
    db.session.commit()

    # select a random chairperson for each committee
    tmp_committees = Organisation.query.filter_by(type="committee").all()
    for committee in tmp_committees:
        if committee.memberships:
            membership_obj = committee.memberships[0]
            membership_obj.type_id = 1
            db.session.add(membership_obj)
    db.session.commit()

    # populate committee reports
    i = 0
    db.session.commit()
    logger.debug("reading report.js")
    with open('data/report.json', 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            report = json.loads(line)
            parsed_report = parsers.MeetingReportParser(report)

            committee_obj = committees.get(parsed_report.committee)
            if committee_obj:
                committee_obj = committee_obj['model']

            report_obj = CommitteeMeeting(
                body=parsed_report.body,
                summary=parsed_report.summary,
                organisation=committee_obj,
                date=parsed_report.date,
                title=parsed_report.title,
                version=0
            )
            db.session.add(report_obj)

            for item in parsed_report.related_docs:
                doc_obj = Content(
                    event=report_obj,
                    type=item["filemime"],
                )
                doc_obj.file_path=item["filepath"]
                doc_obj.title=item["title"]
                db.session.add(doc_obj)

            for item in parsed_report.audio:
                audio_obj = Content(
                    event=report_obj,
                    type=item["filemime"],
                )
                if item["filepath"].startswith('files/'):
                    audio_obj.file_path=item["filepath"][6::]
                else:
                    audio_obj.file_path=item["filepath"]
                if item.get("title_format"):
                    audio_obj.title=item["title_format"]
                elif item.get("filename"):
                    audio_obj.title=item["filename"]
                elif item.get("origname"):
                    audio_obj.title=item["origname"]
                else:
                    audio_obj.title="Unnamed audio"
                db.session.add(audio_obj)

            i += 1
            if i % 500 == 0:
                logger.debug("writing 500 reports...")
                db.session.commit()
    return

def addChild(Child, val):
    if (val):
        tmpobj = Child.query.filter_by(name = val).first()
        if not tmpobj:
            tmpobj = Child(name=val)
            db.session.add(tmpobj)
            db.session.commit()
        return tmpobj.id
    return None

def bills():
    prep_table("bill")
    r = requests.get('http://billsapi.pmg.org.za/bill/')
    bills = r.json()
    for billobj in bills:
        bill = Bill()
        bill.title = billobj["name"]
        bill.bill_code = billobj["code"]
        # act_name = db.Column(db.String(250))
        bill.number = billobj["number"]
        bill.year = billobj["year"]
        # date_of_introduction = db.Column(db.Date)
        # date_of_assent = db.Column(db.Date)
        # effective_date = db.Column(db.Date)
        bill.objective = billobj["objective"]
        bill.is_deleted = billobj["is_deleted"]
        bill.status_id = addChild(BillStatus, billobj["status"])
        bill.bill_type_id = addChild(BillType, billobj["bill_type"])
        # place_of_introduction_id = db.Column(db.Integer, db.ForeignKey('organisation.id'))
        # place_of_introduction = db.relationship('Organisation')
        # introduced_by_id = db.Column(db.Integer, db.ForeignKey('member.id'))
        # introduced_by = db.relationship('Member')
        db.session.add(bill)
    db.session.commit()

def add_featured():
    committeemeetings = CommitteeMeeting.query.limit(5)
    tabledreports = Tabled_committee_report.query.limit(5)
    featured = Featured()
    for committeemeeting in committeemeetings:
        featured.committee_meeting.append(committeemeeting)
    for tabledreport in tabledreports:
        featured.tabled_committee_report.append(tabledreport)
    featured.title = "LivemagSA Launched Live From Parliament"
    featured.blurb = "For the next six months, LiveMagSA be teaming up with PMG to report directly from parliament, bringing you the highlights and telling you about the policy decisions that affect you."
    featured.link = "http://livemag.co.za/welcome-parliament/"
    db.session.add(featured)
    db.session.commit()

def clear_db():
    logger.debug("Dropping all")
    db.drop_all()
    logger.debug("Creating all")
    db.create_all()

if __name__ == '__main__':
    clear_db()
    rebuild_db()
    rebuild_table("hansard", { "title": "title", "meeting_date": "meeting_date", "start_date": "start_date", "body": "body" })
    bills()
    rebuild_table("briefing", {"title": "title", "briefing_date": "briefing_date", "summary": "summary", "minutes": "minutes", "presentation": "presentation", "start_date": "start_date" })
    rebuild_table("questions_replies", {"title": "title", "body": "body", "start_date": "start_date", "question_number": "question_number"})
    rebuild_table("tabled_committee_report", { "title": "title", "start_date": "start_date", "body": "body", "summary": "teaser", "nid": "nid" })
    rebuild_table("calls_for_comment", {
        "title": "title", "start_date": "start_date", "end_date": "comment_exp", "body": "body", "summary": "teaser", "nid": "nid"
        })
    rebuild_table("policy_document", { "title": "title", "effective_date": "effective_date", "start_date": "start_date" })
    rebuild_table("gazette", { "title": "title", "effective_date": "effective_date", "start_date": "start_date" })
    rebuild_table("book", { "title": "title", "summary": "teaser", "start_date": "start_date", "body": "body" })
    add_featured()

<commit_msg>Remove files from urls and add audio for audio, and don't include cdn<commit_after>import os
import json
import time, datetime
from backend.app import app, db

from backend import models
from backend.models import *
import parsers
from sqlalchemy import types

import requests

import logging
logger = logging.getLogger(__name__)
logging.getLogger('sqlalchemy.engine').level = logging.WARN

STATIC_HOST = app.config['STATIC_HOST']
db.echo = False

def strip_filpath(filepath):

    return "/".join(filepath.split("/")[1::])

def dump_db(name):
    try:
        os.remove(name)
    except Exception as e:
        logger.debug(e)
        pass
    return

def read_data(filename):
    start = time.time()

    logger.debug("reading " + filename)
    with open('data/' + filename, 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            records.append(json.loads(line))
    return records

def db_date_from_utime(utime):
    try:
        return datetime.datetime.fromtimestamp(float(utime))
    except:
        try:
            datetime.strptime(utime, '%Y-%m-%dT%H:%m:%s')
        except:
            return None

def construct_obj(obj, mappings):
    """
    Returns a result with the properties mapped by mapping, including all revisions
    """
    result_obj = {}
    # print obj
    for key in mappings.keys():
        if (obj.has_key(key)):
            result_obj[key] = obj[key]
    if (obj.has_key("revisions") and (type(obj["revisions"]) is list) and (len(obj["revisions"]) > 0)):
        for revision in obj["revisions"]:
            tmp = construct_obj(revision, mappings)
            result_obj.update(tmp)
    return result_obj

def find_files(obj):
    files = []
    if (obj.has_key("files") and (type(obj["files"]) is list) and (len(obj["files"]) > 0)):
        # print obj["files"]
        for f in obj["files"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = f["filepath"].replace("files/", ""),
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    if (obj.has_key("audio") and (type(obj["audio"]) is list) and (len(obj["audio"]) > 0)):
        # print obj["audio"]
        for f in obj["audio"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "audio/" + f["filename"].replace("files/", ""),
                    playtime = f["playtime"],
                    description = f["title_format"]
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    return files

def find_committee(obj):
    organisations = []
    if (obj.has_key("terms") and (type(obj["terms"]) is list) and (len(obj["terms"]) > 0)):
        # print obj["terms"]
        for term in obj["terms"]:
            organisation = Organisation.query.filter_by(name = term).first()
            # print organisation, term
            if organisation:
                organisations.append(organisation)
            # else:
            #     print term
    return organisations

def prep_table(tablename):
    Model = getattr(models, tablename.capitalize())
    print "Deleted rows: ", Model.query.delete()

def rebuild_table(tablename, mappings):
    logger.debug("Rebuilding %s" % tablename)
    Model = getattr(models, tablename.capitalize())
    prep_table(tablename)
    i = 0
    with open('data/' + tablename + '.json', 'r') as f:
        records = []
        lines = f.readlines()
        logger.debug("Found %i records" % (len(lines)))
        for line in lines:
            obj = json.loads(line)
            newobj = construct_obj(obj, mappings)
            # Check for Dates
            for mapping in mappings.keys():
                row_type = getattr(Model, mapping).property.columns[0].type
                if (row_type.__str__() == "DATE"):
                    if (newobj.has_key(mappings[mapping])):
                        newobj[mappings[mapping]] = db_date_from_utime(newobj[mappings[mapping]])
            model = Model()
            files = find_files(obj)
            committees = find_committee(obj)
            if (committees):
                for committee in committees:
                    model.committee.append(committee)
            if (len(files)):
                for f in files:
                    model.files.append(f)
            for key,val in newobj.iteritems():
                setattr(model, key, val)
            db.session.add(model)
            i += 1
            if (i == 100):
                db.session.commit()
                i = 0
                logger.debug("Wrote 100 rows...")
        db.session.commit()

def rebuild_db():
    """
    Save json fixtures into a structured database, intended for use in our app.
    """

    db.drop_all()
    db.create_all()

    start = time.time()

    # populate houses of parliament
    joint_obj = House(name="Joint (NA + NCOP)", name_short="Joint")
    ncop_obj = House(name="National Council of Provinces", name_short="NCOP")
    na_obj = House(name="National Assembly", name_short="NA")
    for obj in [joint_obj, ncop_obj, na_obj]:
        db.session.add(obj)
    db.session.commit()

    # populate membership_type
    chairperson = MembershipType(name="chairperson")
    db.session.add(chairperson)
    db.session.commit()

    # populate committees
    committees = {}
    drupal_recs = read_data('comm_info_page.json')
    for rec in drupal_recs:
        if rec['terms']:
            name = rec['terms'][0].strip()
            if not committees.get('name'):
                committees[name] = {}
            if rec['comm_info_type'] == '"Contact"':
                committees[name]["contact"] = rec["revisions"][0]["body"]
            elif rec['comm_info_type'] == '"About"':
                committees[name]["about"] = rec["revisions"][0]["body"]
            if len(rec["revisions"]) > 1:
                logger.debug("ERROR: MULTIPLE REVISIONS PRESENT FOR A COMMITTEE")
    for key, val in committees.iteritems():
        logger.debug(key)
        organisation = Organisation()
        organisation.name = key
        organisation.type = "committee"
        organisation.version = 0

        # set parent relation
        if "ncop" in organisation.name.lower():
            organisation.house_id = ncop_obj.id
        elif "joint" in organisation.name.lower():
            organisation.house_id = joint_obj.id
        else:
            organisation.house_id = na_obj.id

        committee_info = CommitteeInfo()
        if val.get("about"):
            committee_info.about = val["about"]
        if val.get("contact"):
            committee_info.contact_details = val["contact"]
        committee_info.organization = organisation

        db.session.add(organisation)
        db.session.add(committee_info)
        val['model'] = organisation
    db.session.commit()

    # populate committee members
    members = read_data('committee_member.json')
    for member in members:
        member_obj = Member(
            name=member['title'].strip(),
            version=0,
            start_date = db_date_from_utime(member['start_date'])
        )
        if member.get('files'):
            # logger.debug(json.dumps(member['files'], indent=4)
            member_obj.profile_pic_url = STATIC_HOST + strip_filpath(member["files"][-1]['filepath'])
        logger.debug(member_obj.name)
        logger.debug(member_obj.profile_pic_url)

        # extract bio info
        if member['revisions']:
            bio = member['revisions'][0]['body']
            if bio:
                index = bio.find("Further information will be provided shortly on:")
                if index and index > 0:
                    bio = bio[0:index].strip()
                    logger.debug(bio)
                    member_obj.bio = bio

        # set committee relationships
        for term in member['terms']:
            if committees.get(term):
                org_model = committees[term]['model']
                membership_obj = Membership(organisation=org_model)
                member_obj.memberships.append(membership_obj)
            else:
                logger.debug("committee not found: " + term)

        # set party membership
        party = member['mp_party']
        if party:
            logger.debug(party)
            party_obj = Party.query.filter_by(name=party).first()
            if not party_obj:
                party_obj = Party(name=party)
                db.session.add(party_obj)
                db.session.commit()
            member_obj.party_id = party_obj.id

        # set house membership
        house = member['mp_province']
        if house == "National Assembly":
            member_obj.house_id = na_obj.id
        elif house and "NCOP" in house:
            member_obj.house_id = ncop_obj.id
            # logger.debug(house)
            province_obj = Province.query.filter_by(name=house[5::]).first()
            if not province_obj:
                province_obj = Province(name=house[5::])
                db.session.add(province_obj)
                db.session.commit()
            member_obj.province_id = province_obj.id

        db.session.add(member_obj)
        logger.debug('')
    db.session.commit()

    # select a random chairperson for each committee
    tmp_committees = Organisation.query.filter_by(type="committee").all()
    for committee in tmp_committees:
        if committee.memberships:
            membership_obj = committee.memberships[0]
            membership_obj.type_id = 1
            db.session.add(membership_obj)
    db.session.commit()

    # populate committee reports
    i = 0
    db.session.commit()
    logger.debug("reading report.js")
    with open('data/report.json', 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            report = json.loads(line)
            parsed_report = parsers.MeetingReportParser(report)

            committee_obj = committees.get(parsed_report.committee)
            if committee_obj:
                committee_obj = committee_obj['model']

            report_obj = CommitteeMeeting(
                body=parsed_report.body,
                summary=parsed_report.summary,
                organisation=committee_obj,
                date=parsed_report.date,
                title=parsed_report.title,
                version=0
            )
            db.session.add(report_obj)

            for item in parsed_report.related_docs:
                doc_obj = Content(
                    event=report_obj,
                    type=item["filemime"],
                )
                doc_obj.file_path=item["filepath"]
                doc_obj.title=item["title"]
                db.session.add(doc_obj)

            for item in parsed_report.audio:
                audio_obj = Content(
                    event=report_obj,
                    type=item["filemime"],
                )
                if item["filepath"].startswith('files/'):
                    audio_obj.file_path=item["filepath"][6::]
                else:
                    audio_obj.file_path=item["filepath"]
                if item.get("title_format"):
                    audio_obj.title=item["title_format"]
                elif item.get("filename"):
                    audio_obj.title=item["filename"]
                elif item.get("origname"):
                    audio_obj.title=item["origname"]
                else:
                    audio_obj.title="Unnamed audio"
                db.session.add(audio_obj)

            i += 1
            if i % 500 == 0:
                logger.debug("writing 500 reports...")
                db.session.commit()
    return

def addChild(Child, val):
    if (val):
        tmpobj = Child.query.filter_by(name = val).first()
        if not tmpobj:
            tmpobj = Child(name=val)
            db.session.add(tmpobj)
            db.session.commit()
        return tmpobj.id
    return None

def bills():
    prep_table("bill")
    r = requests.get('http://billsapi.pmg.org.za/bill/')
    bills = r.json()
    for billobj in bills:
        bill = Bill()
        bill.title = billobj["name"]
        bill.bill_code = billobj["code"]
        # act_name = db.Column(db.String(250))
        bill.number = billobj["number"]
        bill.year = billobj["year"]
        # date_of_introduction = db.Column(db.Date)
        # date_of_assent = db.Column(db.Date)
        # effective_date = db.Column(db.Date)
        bill.objective = billobj["objective"]
        bill.is_deleted = billobj["is_deleted"]
        bill.status_id = addChild(BillStatus, billobj["status"])
        bill.bill_type_id = addChild(BillType, billobj["bill_type"])
        # place_of_introduction_id = db.Column(db.Integer, db.ForeignKey('organisation.id'))
        # place_of_introduction = db.relationship('Organisation')
        # introduced_by_id = db.Column(db.Integer, db.ForeignKey('member.id'))
        # introduced_by = db.relationship('Member')
        db.session.add(bill)
    db.session.commit()

def add_featured():
    committeemeetings = CommitteeMeeting.query.limit(5)
    tabledreports = Tabled_committee_report.query.limit(5)
    featured = Featured()
    for committeemeeting in committeemeetings:
        featured.committee_meeting.append(committeemeeting)
    for tabledreport in tabledreports:
        featured.tabled_committee_report.append(tabledreport)
    featured.title = "LivemagSA Launched Live From Parliament"
    featured.blurb = "For the next six months, LiveMagSA be teaming up with PMG to report directly from parliament, bringing you the highlights and telling you about the policy decisions that affect you."
    featured.link = "http://livemag.co.za/welcome-parliament/"
    db.session.add(featured)
    db.session.commit()

def clear_db():
    logger.debug("Dropping all")
    db.drop_all()
    logger.debug("Creating all")
    db.create_all()

if __name__ == '__main__':
    clear_db()
    rebuild_db()
    rebuild_table("hansard", { "title": "title", "meeting_date": "meeting_date", "start_date": "start_date", "body": "body" })
    bills()
    rebuild_table("briefing", {"title": "title", "briefing_date": "briefing_date", "summary": "summary", "minutes": "minutes", "presentation": "presentation", "start_date": "start_date" })
    rebuild_table("questions_replies", {"title": "title", "body": "body", "start_date": "start_date", "question_number": "question_number"})
    rebuild_table("tabled_committee_report", { "title": "title", "start_date": "start_date", "body": "body", "summary": "teaser", "nid": "nid" })
    rebuild_table("calls_for_comment", {
        "title": "title", "start_date": "start_date", "end_date": "comment_exp", "body": "body", "summary": "teaser", "nid": "nid"
        })
    rebuild_table("policy_document", { "title": "title", "effective_date": "effective_date", "start_date": "start_date" })
    rebuild_table("gazette", { "title": "title", "effective_date": "effective_date", "start_date": "start_date" })
    rebuild_table("book", { "title": "title", "summary": "teaser", "start_date": "start_date", "body": "body" })
    add_featured()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License (MIT)

# Copyright (c) 2017 Juan Cabral

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# =============================================================================
# FUTURE
# =============================================================================

from __future__ import unicode_literals

# =============================================================================
# DOC
# =============================================================================

__doc__ = """All feets base tests"""

# =============================================================================
# IMPORTS
# =============================================================================

import numpy as np

from .. import MPFeatureSpace, FeatureSpace

from .core import FeetsTestCase

# =============================================================================
# BASE CLASS
# =============================================================================

class FeatureSpaceTestCase(FeetsTestCase):

    FeatureSpaceCls = FeatureSpace

    def setUp(self):
        self.space = self.FeatureSpaceCls(only=["Amplitude"])

    def test_extract_one(self):
        data = np.array([[
            0.46057565, 0.51372940, 0.70136533, 0.21454228,
            0.54792300, 0.33433717, 0.44879870, 0.55571062,
            0.24388037, 0.44793366, 0.30175873, 0.88326381,
            0.12208977, 0.37088649, 0.59457310, 0.74705894,
            0.24551664, 0.36009236, 0.80661981, 0.04961063,
            0.87747311, 0.97388975, 0.95775496, 0.34195989,
            0.54201036, 0.87854618, 0.07388174, 0.21543205,
            0.59295337, 0.56771493]])

        features, values = self.space.extract_one(data)
        self.assertTrue(len(features) == 1 and features[0] == "Amplitude")
        self.assertAllClose(values[features == "Amplitude"], 0.45203809)

    def test_extract(self):
        data = np.array([[
            0.46057565, 0.51372940, 0.70136533, 0.21454228,
            0.54792300, 0.33433717, 0.44879870, 0.55571062,
            0.24388037, 0.44793366, 0.30175873, 0.88326381,
            0.12208977, 0.37088649, 0.59457310, 0.74705894,
            0.24551664, 0.36009236, 0.80661981, 0.04961063,
            0.87747311, 0.97388975, 0.95775496, 0.34195989,
            0.54201036, 0.87854618, 0.07388174, 0.21543205,
            0.59295337, 0.56771493]])

        features, values_col = self.space.extract([data])
        for values in values_col:
            self.assertTrue(len(features) == 1 and features[0] == "Amplitude")
            self.assertAllClose(values[features == "Amplitude"], 0.45203809)

class MPFeatureSpaceTestCase(FeatureSpaceTestCase):

    FeatureSpaceCls = MPFeatureSpace
<commit_msg>simplified tests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# The MIT License (MIT)

# Copyright (c) 2017 Juan Cabral

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# =============================================================================
# FUTURE
# =============================================================================

from __future__ import unicode_literals

# =============================================================================
# DOC
# =============================================================================

__doc__ = """All feets base tests"""

# =============================================================================
# IMPORTS
# =============================================================================

import numpy as np

import mock

from .. import MPFeatureSpace, FeatureSpace, Extractor, register_extractor

from .core import FeetsTestCase

# =============================================================================
# BASE CLASS
# =============================================================================

class FeatureSpaceTestCase(FeetsTestCase):

    FeatureSpaceCls = FeatureSpace

    def setUp(self):
        self.space = self.FeatureSpaceCls(only=["Amplitude"])

    def test_extract_one(self):
        data = np.array([[
            0.46057565, 0.51372940, 0.70136533, 0.21454228,
            0.54792300, 0.33433717, 0.44879870, 0.55571062,
            0.24388037, 0.44793366, 0.30175873, 0.88326381,
            0.12208977, 0.37088649, 0.59457310, 0.74705894,
            0.24551664, 0.36009236, 0.80661981, 0.04961063,
            0.87747311, 0.97388975, 0.95775496, 0.34195989,
            0.54201036, 0.87854618, 0.07388174, 0.21543205,
            0.59295337, 0.56771493]])

        features, values = self.space.extract_one(data)
        self.assertTrue(len(features) == 1 and features[0] == "Amplitude")
        self.assertAllClose(values[features == "Amplitude"], 0.45203809)

    def test_extract(self):
        data = np.array([[
            0.46057565, 0.51372940, 0.70136533, 0.21454228,
            0.54792300, 0.33433717, 0.44879870, 0.55571062,
            0.24388037, 0.44793366, 0.30175873, 0.88326381,
            0.12208977, 0.37088649, 0.59457310, 0.74705894,
            0.24551664, 0.36009236, 0.80661981, 0.04961063,
            0.87747311, 0.97388975, 0.95775496, 0.34195989,
            0.54201036, 0.87854618, 0.07388174, 0.21543205,
            0.59295337, 0.56771493]])

        features, values_col = self.space.extract([data])
        for values in values_col:
            self.assertTrue(len(features) == 1 and features[0] == "Amplitude")
            self.assertAllClose(values[features == "Amplitude"], 0.45203809)

    @mock.patch("feets.extractors._extractors", {})
    def test_features_order(self):

        @register_extractor
        class ReturnSame(Extractor):
            data = ["magnitude"]
            features = ["Same"]

            def fit(self, magnitude):
                return {"Same": magnitude[0]}

        self.space = self.FeatureSpaceCls(only=["Same"])

        data = np.array([
            [4], [1], [2], [3]
        ])

        features, values_col = self.space.extract(data)

        import ipdb; ipdb.set_trace()

class MPFeatureSpaceTestCase(FeatureSpaceTestCase):

    FeatureSpaceCls = MPFeatureSpace
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

""":mod:`openquake.hazardlib.calc.hazard_curve` implements
:func:`calc_hazard_curves`. Here is an example of a classical PSHA
parallel calculator computing the hazard curves per each realization in less
than 20 lines of code:

.. code-block:: python

   import sys
   from openquake.commonlib import logs
   from openquake.calculators.base import get_calc

   def main(job_ini):
       calc_id = logs.init()
       calc = get_calc(job_ini, calc_id)
       calc.run(individual_curves='true', shutdown=True)
       print('The hazard curves are in %s::/hcurves-rlzs'
             % calc.datastore.filename)

   if __name__ == '__main__':
       main(sys.argv[1])  # path to a job.ini file

NB: the implementation in the engine is smarter and more
efficient. Here we start a parallel computation per each realization,
the engine manages all the realizations at once.
"""

import operator
from openquake.baselib.performance import Monitor
from openquake.baselib.parallel import sequential_apply
from openquake.baselib.general import DictArray, groupby
from openquake.hazardlib.probability_map import ProbabilityMap
from openquake.hazardlib.gsim.base import ContextMaker, PmapMaker
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.sourceconverter import SourceGroup
from openquake.hazardlib.tom import FatedTOM

def _cluster(imtls, tom, gsims, pmap):
    """
    Computes the probability map in case of a cluster group
    """
    L, G = len(imtls.array), len(gsims)
    pmapclu = ProbabilityMap(L, G)
    # Get temporal occurrence model
    # Number of occurrences for the cluster
    first = True
    for nocc in range(0, 50):
        # TODO fix this once the occurrence rate will be used just as
        # an object attribute
        ocr = tom.occurrence_rate
        prob_n_occ = tom.get_probability_n_occurrences(ocr, nocc)
        if first:
            pmapclu = (~pmap)**nocc * prob_n_occ
            first = False
        else:
            pmapclu += (~pmap)**nocc * prob_n_occ
    pmap = ~pmapclu
    return pmap

def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary with keys pmap, calc_times, rup_data, extra
    """
    if not hasattr(src_filter, 'sitecol'):  # do not filter
        src_filter = SourceFilter(src_filter, {})

    # Get the parameters assigned to the group
    src_mutex = getattr(group, 'src_interdep', None) == 'mutex'
    cluster = getattr(group, 'cluster', None)
    trts = set()
    maxradius = 0
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures may not be set, so it is set here
            src.num_ruptures = src.count_ruptures()
        # set the proper TOM in case of a cluster
        if cluster:
            src.temporal_occurrence_model = FatedTOM(time_span=1)
        trts.add(src.tectonic_region_type)
        if hasattr(src, 'radius'):  # for prefiltered point sources
            maxradius = max(maxradius, src.radius)

    param['maximum_distance'] = src_filter.integration_distance
    [trt] = trts  # there must be a single tectonic region type
    cmaker = ContextMaker(trt, gsims, param, monitor)
    pmap, rup_data, calc_times, extra = PmapMaker(
        cmaker, src_filter, group).make()
    extra['task_no'] = getattr(monitor, 'task_no', 0)
    extra['trt'] = trt
    extra['source_id'] = src.source_id
    extra['grp_id'] = src.grp_id
    extra['maxradius'] = maxradius
    group_probability = getattr(group, 'grp_probability', None)
    if src_mutex and group_probability:
        pmap *= group_probability

    if cluster:
        tom = getattr(group, 'temporal_occurrence_model')
        pmap = _cluster(param['imtls'], tom, gsims, pmap)
    return dict(pmap=pmap, calc_times=calc_times,
                rup_data=rup_data, extra=extra)

# not used in the engine, only in tests and possibly notebooks
def calc_hazard_curves(
        groups, srcfilter, imtls, gsim_by_trt, truncation_level=None,
        apply=sequential_apply, reqv=None, **kwargs):
    """
    Compute hazard curves on a list of sites, given a set of seismic source
    groups and a dictionary of ground shaking intensity models (one per
    tectonic region type).

    Probability of ground motion exceedance is computed in different ways
    depending if the sources are independent or mutually exclusive.

    :param groups:
        A sequence of groups of seismic sources objects (instances of
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param srcfilter:
        A source filter over the site collection or the site collection itself
    :param imtls:
        Dictionary mapping intensity measure type strings
        to lists of intensity measure levels.
    :param gsim_by_trt:
        Dictionary mapping tectonic region types (members
        of :class:`openquake.hazardlib.const.TRT`) to
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE` objects.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param apply:
        apply function to use (default sequential_apply)
    :param reqv:
        If not None, an instance of RjbEquivalent
    :returns:
        An array of size N, where N is the number of sites, which elements
        are records with fields given by the intensity measure types; the
        size of each field is given by the number of levels in ``imtls``.
    """
    # This is ensuring backward compatibility i.e. processing a list of
    # sources
    if not isinstance(groups[0], SourceGroup):  # sent a list of sources
        odic = groupby(groups, operator.attrgetter('tectonic_region_type'))
        groups = [SourceGroup(trt, odic[trt], 'src_group', 'indep', 'indep')
                  for trt in odic]
    # ensure the sources have the right et_id
    idx = 0
    for i, grp in enumerate(groups):
        for src in grp:
            if not hasattr(src, 'et_id'):
                src.et_id = i  # fix et_id
            src.grp_id = i
            src.id = idx
            idx += 1
    imtls = DictArray(imtls)
    shift_hypo = kwargs['shift_hypo'] if 'shift_hypo' in kwargs else False
    param = dict(imtls=imtls, truncation_level=truncation_level, reqv=reqv,
                 cluster=grp.cluster, shift_hypo=shift_hypo)
    pmap = ProbabilityMap(len(imtls.array), 1)
    # Processing groups with homogeneous tectonic region
    mon = Monitor()
    for group in groups:
        for src in group:
            if not src.nsites:  # not set
                src.nsites = 1
        gsim = gsim_by_trt[group[0].tectonic_region_type]
        if group.atomic:  # do not split
            it = [classical(group, srcfilter, [gsim], param, mon)]
        else:  # split the group and apply `classical` in parallel
            it = apply(
                classical, (group.sources, srcfilter, [gsim], param),
                weight=operator.attrgetter('weight'))
        for dic in it:
            pmap |= dic['pmap']
    sitecol = getattr(srcfilter, 'sitecol', srcfilter)
    return pmap.convert(imtls, len(sitecol.complete))

# called in adv-manual/developing.rst
def calc_hazard_curve(site1, src, gsim, oqparam):
    """
    :param site1: site collection with a single site
    :param src: a seismic source object
    :param gsim: a GSIM object
    :param oqparam: an object with attributes .maximum_distance, .imtls
    :returns: a ProbabilityCurve object
    """
    assert len(site1) == 1, site1
    trt = src.tectonic_region_type
    cmaker = ContextMaker(trt, {gsim: [0]}, vars(oqparam))
    srcfilter = SourceFilter(site1, oqparam.maximum_distance)
    pmap, rup_data, calc_times, extra = PmapMaker(
        cmaker, srcfilter, [src]).make()
    return pmap[0]  # pcurve with shape (L, G) on site 0
<commit_msg>Managed the case of zero curves<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2012-2020 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

""":mod:`openquake.hazardlib.calc.hazard_curve` implements
:func:`calc_hazard_curves`. Here is an example of a classical PSHA
parallel calculator computing the hazard curves per each realization in less
than 20 lines of code:

.. code-block:: python

   import sys
   from openquake.commonlib import logs
   from openquake.calculators.base import get_calc

   def main(job_ini):
       calc_id = logs.init()
       calc = get_calc(job_ini, calc_id)
       calc.run(individual_curves='true', shutdown=True)
       print('The hazard curves are in %s::/hcurves-rlzs'
             % calc.datastore.filename)

   if __name__ == '__main__':
       main(sys.argv[1])  # path to a job.ini file

NB: the implementation in the engine is smarter and more
efficient. Here we start a parallel computation per each realization,
the engine manages all the realizations at once.
"""

import operator
import numpy
from openquake.baselib.performance import Monitor
from openquake.baselib.parallel import sequential_apply
from openquake.baselib.general import DictArray, groupby
from openquake.hazardlib.probability_map import (
    ProbabilityMap, ProbabilityCurve)
from openquake.hazardlib.gsim.base import ContextMaker, PmapMaker
from openquake.hazardlib.calc.filters import SourceFilter
from openquake.hazardlib.sourceconverter import SourceGroup
from openquake.hazardlib.tom import FatedTOM

def _cluster(imtls, tom, gsims, pmap):
    """
    Computes the probability map in case of a cluster group
    """
    L, G = len(imtls.array), len(gsims)
    pmapclu = ProbabilityMap(L, G)
    # Get temporal occurrence model
    # Number of occurrences for the cluster
    first = True
    for nocc in range(0, 50):
        # TODO fix this once the occurrence rate will be used just as
        # an object attribute
        ocr = tom.occurrence_rate
        prob_n_occ = tom.get_probability_n_occurrences(ocr, nocc)
        if first:
            pmapclu = (~pmap)**nocc * prob_n_occ
            first = False
        else:
            pmapclu += (~pmap)**nocc * prob_n_occ
    pmap = ~pmapclu
    return pmap

def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary with keys pmap, calc_times, rup_data, extra
    """
    if not hasattr(src_filter, 'sitecol'):  # do not filter
        src_filter = SourceFilter(src_filter, {})

    # Get the parameters assigned to the group
    src_mutex = getattr(group, 'src_interdep', None) == 'mutex'
    cluster = getattr(group, 'cluster', None)
    trts = set()
    maxradius = 0
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures may not be set, so it is set here
            src.num_ruptures = src.count_ruptures()
        # set the proper TOM in case of a cluster
        if cluster:
            src.temporal_occurrence_model = FatedTOM(time_span=1)
        trts.add(src.tectonic_region_type)
        if hasattr(src, 'radius'):  # for prefiltered point sources
            maxradius = max(maxradius, src.radius)

    param['maximum_distance'] = src_filter.integration_distance
    [trt] = trts  # there must be a single tectonic region type
    cmaker = ContextMaker(trt, gsims, param, monitor)
    pmap, rup_data, calc_times, extra = PmapMaker(
        cmaker, src_filter, group).make()
    extra['task_no'] = getattr(monitor, 'task_no', 0)
    extra['trt'] = trt
    extra['source_id'] = src.source_id
    extra['grp_id'] = src.grp_id
    extra['maxradius'] = maxradius
    group_probability = getattr(group, 'grp_probability', None)
    if src_mutex and group_probability:
        pmap *= group_probability

    if cluster:
        tom = getattr(group, 'temporal_occurrence_model')
        pmap = _cluster(param['imtls'], tom, gsims, pmap)
    return dict(pmap=pmap, calc_times=calc_times,
                rup_data=rup_data, extra=extra)

# not used in the engine, only in tests and possibly notebooks
def calc_hazard_curves(
        groups, srcfilter, imtls, gsim_by_trt, truncation_level=None,
        apply=sequential_apply, reqv=None, **kwargs):
    """
    Compute hazard curves on a list of sites, given a set of seismic source
    groups and a dictionary of ground shaking intensity models (one per
    tectonic region type).

    Probability of ground motion exceedance is computed in different ways
    depending if the sources are independent or mutually exclusive.

    :param groups:
        A sequence of groups of seismic sources objects (instances of
        of :class:`~openquake.hazardlib.source.base.BaseSeismicSource`).
    :param srcfilter:
        A source filter over the site collection or the site collection itself
    :param imtls:
        Dictionary mapping intensity measure type strings
        to lists of intensity measure levels.
    :param gsim_by_trt:
        Dictionary mapping tectonic region types (members
        of :class:`openquake.hazardlib.const.TRT`) to
        :class:`~openquake.hazardlib.gsim.base.GMPE` or
        :class:`~openquake.hazardlib.gsim.base.IPE` objects.
    :param truncation_level:
        Float, number of standard deviations for truncation of the intensity
        distribution.
    :param apply:
        apply function to use (default sequential_apply)
    :param reqv:
        If not None, an instance of RjbEquivalent
    :returns:
        An array of size N, where N is the number of sites, which elements
        are records with fields given by the intensity measure types; the
        size of each field is given by the number of levels in ``imtls``.
    """
    # This is ensuring backward compatibility i.e. processing a list of
    # sources
    if not isinstance(groups[0], SourceGroup):  # sent a list of sources
        odic = groupby(groups, operator.attrgetter('tectonic_region_type'))
        groups = [SourceGroup(trt, odic[trt], 'src_group', 'indep', 'indep')
                  for trt in odic]
    # ensure the sources have the right et_id
    idx = 0
    for i, grp in enumerate(groups):
        for src in grp:
            if not hasattr(src, 'et_id'):
                src.et_id = i  # fix et_id
            src.grp_id = i
            src.id = idx
            idx += 1
    imtls = DictArray(imtls)
    shift_hypo = kwargs['shift_hypo'] if 'shift_hypo' in kwargs else False
    param = dict(imtls=imtls, truncation_level=truncation_level, reqv=reqv,
                 cluster=grp.cluster, shift_hypo=shift_hypo)
    pmap = ProbabilityMap(len(imtls.array), 1)
    # Processing groups with homogeneous tectonic region
    mon = Monitor()
    for group in groups:
        for src in group:
            if not src.nsites:  # not set
                src.nsites = 1
        gsim = gsim_by_trt[group[0].tectonic_region_type]
        if group.atomic:  # do not split
            it = [classical(group, srcfilter, [gsim], param, mon)]
        else:  # split the group and apply `classical` in parallel
            it = apply(
                classical, (group.sources, srcfilter, [gsim], param),
                weight=operator.attrgetter('weight'))
        for dic in it:
            pmap |= dic['pmap']
    sitecol = getattr(srcfilter, 'sitecol', srcfilter)
    return pmap.convert(imtls, len(sitecol.complete))

# called in adv-manual/developing.rst
def calc_hazard_curve(site1, src, gsim, oqparam):
    """
    :param site1: site collection with a single site
    :param src: a seismic source object
    :param gsim: a GSIM object
    :param oqparam: an object with attributes .maximum_distance, .imtls
    :returns: a ProbabilityCurve object
    """
    assert len(site1) == 1, site1
    trt = src.tectonic_region_type
    cmaker = ContextMaker(trt, {gsim: [0]}, vars(oqparam))
    srcfilter = SourceFilter(site1, oqparam.maximum_distance)
    pmap, rup_data, calc_times, extra = PmapMaker(
        cmaker, srcfilter, [src]).make()
    if not pmap:  # filtered away
        zero = numpy.zeros((len(oqparam.imtls.array), 1))
        return ProbabilityCurve(zero)
    return pmap[0]  # pcurve with shape (L, G) on site 0
<|endoftext|>


<commit_before># coding: utf-8

# imports
import os, re, decimal
from time import gmtime, strftime, localtime, mktime, time
from urlparse import urlparse

# django imports
from django.utils.translation import ugettext as _
from django.utils.safestring import mark_safe
from django.core.files import File
from django.core.files.storage import default_storage

# filebrowser imports
from filebrowser.settings import *

# PIL import
if STRICT_PIL:
    from PIL import Image
else:
    try:
        from PIL import Image
    except ImportError:
        import Image

def url_to_path(value):
    """
    Change URL to PATH.
    Value has to be an URL relative to MEDIA URL or a full URL (including MEDIA_URL).

    Returns a PATH relative to MEDIA_ROOT.
    """

    mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
    value = mediaurl_re.sub('', value)
    return value

def path_to_url(value):
    """
    Change PATH to URL.
    Value has to be a PATH relative to MEDIA_ROOT.

    Return an URL relative to MEDIA_ROOT.
    """

    mediaroot_re = re.compile(r'^(%s)' % (MEDIA_ROOT))
    value = mediaroot_re.sub('', value)
    return url_join(MEDIA_URL, value)

def dir_from_url(value):
    """
    Get the relative server directory from a URL.
    URL has to be an absolute URL including MEDIA_URL or
    an URL relative to MEDIA_URL.
    """

    mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
    value = mediaurl_re.sub('', value)
    directory_re = re.compile(r'^(%s)' % (DIRECTORY))
    value = directory_re.sub('', value)
    return os.path.split(value)[0]

def get_version_path(value, version_prefix):
    """
    Construct the PATH to an Image version.
    Value has to be server-path, relative to MEDIA_ROOT.

    version_filename = filename + version_prefix + ext
    Returns a path relative to MEDIA_ROOT.
    """

    if os.path.isfile(os.path.join(MEDIA_ROOT, value)):
        path, filename = os.path.split(value)
        filename, ext = os.path.splitext(filename)
        version_filename = filename + "_" + version_prefix + ext
        return os.path.join(VERSIONS_BASEDIR, path, version_filename)
    else:
        return None

def sort_by_attr(seq, attr):
    """
    Sort the sequence of objects by object's attribute

    Arguments:
    seq  - the list or any sequence (including immutable one) of objects to sort.
    attr - the name of attribute to sort by

    Returns:
    the sorted list of objects.
    """
    import operator

    # Use the "Schwartzian transform"
    # Create the auxiliary list of tuples where every i-th tuple has form
    # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
    # only to provide stable sorting, but mainly to eliminate comparison of objects
    # (which can be expensive or prohibited) in case of equal attribute values.
    intermed = map(None, map(getattr, seq, (attr,)*len(seq)), xrange(len(seq)), seq)
    intermed.sort()
    return map(operator.getitem, intermed, (-1,) * len(intermed))

def url_join(*args):
    """
    URL join routine.
    """

    if args[0].startswith("http://"):
        url = "http://"
    else:
        url = "/"
    for arg in args:
        arg = unicode(arg).replace("\\", "/")
        arg_split = arg.split("/")
        for elem in arg_split:
            if elem != "" and elem != "http:":
                url = url + elem + "/"
    # remove trailing slash for filenames
    if os.path.splitext(args[-1])[1]:
        url = url.rstrip("/")
    return url

def get_path(path):
    """
    Get Path.
    """

    if path.startswith('.') or os.path.isabs(path) or not os.path.isdir(os.path.join(MEDIA_ROOT, DIRECTORY, path)):
        return None
    return path

def get_file(path, filename):
    """
    Get File.
    """

    if not os.path.isfile(os.path.join(MEDIA_ROOT, DIRECTORY, path, filename)) and not os.path.isdir(os.path.join(MEDIA_ROOT, DIRECTORY, path, filename)):
        return None
    return filename

def get_breadcrumbs(query, path):
    """
    Get breadcrumbs.
    """

    breadcrumbs = []
    dir_query = ""
    if path:
        for item in path.split(os.sep):
            dir_query = os.path.join(dir_query,item)
            breadcrumbs.append([item,dir_query])
    return breadcrumbs

def get_filterdate(filterDate, dateTime):
    """
    Get filterdate.
    """

    returnvalue = ''
    dateYear = strftime("%Y", gmtime(dateTime))
    dateMonth = strftime("%m", gmtime(dateTime))
    dateDay = strftime("%d", gmtime(dateTime))
    if filterDate == 'today' and int(dateYear) == int(localtime()[0]) and int(dateMonth) == int(localtime()[1]) and int(dateDay) == int(localtime()[2]): returnvalue = 'true'
    elif filterDate == 'thismonth' and dateTime >= time()-2592000: returnvalue = 'true'
    elif filterDate == 'thisyear' and int(dateYear) == int(localtime()[0]): returnvalue = 'true'
    elif filterDate == 'past7days' and dateTime >= time()-604800: returnvalue = 'true'
    elif filterDate == '': returnvalue = 'true'
    return returnvalue

def get_settings_var():
    """
    Get settings variables used for FileBrowser listing.
    """

    settings_var = {}
    # Main
    settings_var['DEBUG'] = DEBUG
    settings_var['MEDIA_ROOT'] = MEDIA_ROOT
    settings_var['MEDIA_URL'] = MEDIA_URL
    settings_var['DIRECTORY'] = DIRECTORY
    # FileBrowser
    settings_var['URL_FILEBROWSER_MEDIA'] = URL_FILEBROWSER_MEDIA
    settings_var['PATH_FILEBROWSER_MEDIA'] = PATH_FILEBROWSER_MEDIA
    # TinyMCE
    settings_var['URL_TINYMCE'] = URL_TINYMCE
    settings_var['PATH_TINYMCE'] = PATH_TINYMCE
    # Extensions/Formats (for FileBrowseField)
    settings_var['EXTENSIONS'] = EXTENSIONS
    settings_var['SELECT_FORMATS'] = SELECT_FORMATS
    # Versions
    settings_var['VERSIONS_BASEDIR'] = VERSIONS_BASEDIR
    settings_var['VERSIONS'] = VERSIONS
    settings_var['ADMIN_VERSIONS'] = ADMIN_VERSIONS
    settings_var['ADMIN_THUMBNAIL'] = ADMIN_THUMBNAIL
    # FileBrowser Options
    settings_var['MAX_UPLOAD_SIZE'] = MAX_UPLOAD_SIZE
    # Convert Filenames
    settings_var['CONVERT_FILENAME'] = CONVERT_FILENAME
    return settings_var

def handle_file_upload(path, file):
    """
    Handle File Upload.
    """

    file_path = os.path.join(path, file.name)
    uploadedfile = default_storage.save(file_path, file)
    return uploadedfile

def get_file_type(filename):
    """
    Get file type as defined in EXTENSIONS.
    """

    file_extension = os.path.splitext(filename)[1].lower()
    file_type = ''
    for k,v in EXTENSIONS.iteritems():
        for extension in v:
            if file_extension == extension.lower():
                file_type = k
    return file_type

def is_selectable(filename, selecttype):
    """
    Get select type as defined in FORMATS.
    """

    file_extension = os.path.splitext(filename)[1].lower()
    select_types = []
    for k,v in SELECT_FORMATS.iteritems():
        for extension in v:
            if file_extension == extension.lower():
                select_types.append(k)
    return select_types

def version_generator(value, version_prefix, force=None):
    """
    Generate Version for an Image.
    value has to be a serverpath relative to MEDIA_ROOT.
    """

    # PIL's Error "Suspension not allowed here" work around:
    # s. http://mail.python.org/pipermail/image-sig/1999-August/000816.html
    if STRICT_PIL:
        from PIL import ImageFile
    else:
        try:
            from PIL import ImageFile
        except ImportError:
            import ImageFile
    ImageFile.MAXBLOCK = IMAGE_MAXBLOCK # default is 64k

    try:
        im = Image.open(os.path.join(MEDIA_ROOT, value))
        version_path = get_version_path(value, version_prefix)
        absolute_version_path = os.path.join(MEDIA_ROOT, version_path)
        version_dir = os.path.split(absolute_version_path)[0]
        if not os.path.isdir(version_dir):
            os.makedirs(version_dir)
            os.chmod(version_dir, 0775)
        version = scale_and_crop(im, VERSIONS[version_prefix]['width'], VERSIONS[version_prefix]['height'], VERSIONS[version_prefix]['opts'])
        try:
            version.save(absolute_version_path, quality=90, optimize=(os.path.splitext(version_path)[1].lower() != '.gif'))
        except IOError:
            version.save(absolute_version_path, quality=90)
        return version_path
    except:
        return None

def scale_and_crop(im, width, height, opts):
    """
    Scale and Crop.
    """

    x, y   = [float(v) for v in im.size]
    if width:
        xr = float(width)
    else:
        xr = float(x*height/y)
    if height:
        yr = float(height)
    else:
        yr = float(y*width/x)

    if 'crop' in opts:
        r = max(xr/x, yr/y)
    else:
        r = min(xr/x, yr/y)

    if r < 1.0 or (r > 1.0 and 'upscale' in opts):
        im = im.resize((int(x*r), int(y*r)), resample=Image.ANTIALIAS)

    if 'crop' in opts:
        x, y   = [float(v) for v in im.size]
        ex, ey = (x-min(x, xr))/2, (y-min(y, yr))/2
        if ex or ey:
            im = im.crop((int(ex), int(ey), int(x-ex), int(y-ey)))
    return im

    # if 'crop' in opts:
    #     if 'top_left' in opts:
    #         #draw cropping box from upper left corner of image
    #         box = (0, 0, int(min(x, xr)), int(min(y, yr)))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'top_right' in opts:
    #         #draw cropping box from upper right corner of image
    #         box = (int(x-min(x, xr)), 0, int(x), int(min(y, yr)))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'bottom_left' in opts:
    #         #draw cropping box from lower left corner of image
    #         box = (0, int(y-min(y, yr)), int(xr), int(y))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'bottom_right' in opts:
    #         #draw cropping box from lower right corner of image
    #         (int(x-min(x, xr)), int(y-min(y, yr)), int(x), int(y))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     else:
    #         ex, ey = (x-min(x, xr))/2, (y-min(y, yr))/2
    #         if ex or ey:
    #             box = (int(ex), int(ey), int(x-ex), int(y-ey))
    #             im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    # return im

scale_and_crop.valid_options = ('crop', 'upscale')

def convert_filename(value):
    """
    Convert Filename.
    """

    if CONVERT_FILENAME:
        return value.replace(" ", "_").lower()
    else:
        return value

<commit_msg>fixes issue 200<commit_after># coding: utf-8

# imports
import os, re, decimal
from time import gmtime, strftime, localtime, mktime, time
from urlparse import urlparse

# django imports
from django.utils.translation import ugettext as _
from django.utils.safestring import mark_safe
from django.core.files import File
from django.core.files.storage import default_storage

# filebrowser imports
from filebrowser.settings import *

# PIL import
if STRICT_PIL:
    from PIL import Image
else:
    try:
        from PIL import Image
    except ImportError:
        import Image

def url_to_path(value):
    """
    Change URL to PATH.
    Value has to be an URL relative to MEDIA URL or a full URL (including MEDIA_URL).

    Returns a PATH relative to MEDIA_ROOT.
    """

    mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
    value = mediaurl_re.sub('', value)
    return value

def path_to_url(value):
    """
    Change PATH to URL.
    Value has to be a PATH relative to MEDIA_ROOT.

    Return an URL relative to MEDIA_ROOT.
    """

    mediaroot_re = re.compile(r'^(%s)' % (MEDIA_ROOT))
    value = mediaroot_re.sub('', value)
    return url_join(MEDIA_URL, value)

def dir_from_url(value):
    """
    Get the relative server directory from a URL.
    URL has to be an absolute URL including MEDIA_URL or
    an URL relative to MEDIA_URL.
    """

    mediaurl_re = re.compile(r'^(%s)' % (MEDIA_URL))
    value = mediaurl_re.sub('', value)
    directory_re = re.compile(r'^(%s)' % (DIRECTORY))
    value = directory_re.sub('', value)
    return os.path.split(value)[0]

def get_version_path(value, version_prefix):
    """
    Construct the PATH to an Image version.
    Value has to be server-path, relative to MEDIA_ROOT.

    version_filename = filename + version_prefix + ext
    Returns a path relative to MEDIA_ROOT.
    """

    if os.path.isfile(os.path.join(MEDIA_ROOT, value)):
        path, filename = os.path.split(value)
        filename, ext = os.path.splitext(filename)

        # check if this file is a version of an other file
        # to return filename_<version>.ext instead of filename_<version>_<version>.ext
        tmp = filename.split("_")
        if tmp[len(tmp)-1] in ADMIN_VERSIONS:
            # it seems like the "original" is actually a version of an other original
            # so we strip the suffix (aka. version_perfix)
            new_filename = filename.replace("_" + tmp[len(tmp)-1], "")
            # check if the version exists when we use the new_filename
            if os.path.isfile(os.path.join(MEDIA_ROOT, path, new_filename + "_" + version_prefix + ext)):
                # our "original" filename seem to be filename_<version> construct
                # so we replace it with the new_filename
                filename = new_filename
                # if a VERSIONS_BASEDIR is set we need to strip it from the path
                # or we get a <VERSIONS_BASEDIR>/<VERSIONS_BASEDIR>/... construct
                if VERSIONS_BASEDIR != "":
                        path = path.replace(VERSIONS_BASEDIR + "/", "")

        version_filename = filename + "_" + version_prefix + ext
        return os.path.join(VERSIONS_BASEDIR, path, version_filename)
    else:
        return None

def sort_by_attr(seq, attr):
    """
    Sort the sequence of objects by object's attribute

    Arguments:
    seq  - the list or any sequence (including immutable one) of objects to sort.
    attr - the name of attribute to sort by

    Returns:
    the sorted list of objects.
    """
    import operator

    # Use the "Schwartzian transform"
    # Create the auxiliary list of tuples where every i-th tuple has form
    # (seq[i].attr, i, seq[i]) and sort it. The second item of tuple is needed not
    # only to provide stable sorting, but mainly to eliminate comparison of objects
    # (which can be expensive or prohibited) in case of equal attribute values.
    intermed = map(None, map(getattr, seq, (attr,)*len(seq)), xrange(len(seq)), seq)
    intermed.sort()
    return map(operator.getitem, intermed, (-1,) * len(intermed))

def url_join(*args):
    """
    URL join routine.
    """

    if args[0].startswith("http://"):
        url = "http://"
    else:
        url = "/"
    for arg in args:
        arg = unicode(arg).replace("\\", "/")
        arg_split = arg.split("/")
        for elem in arg_split:
            if elem != "" and elem != "http:":
                url = url + elem + "/"
    # remove trailing slash for filenames
    if os.path.splitext(args[-1])[1]:
        url = url.rstrip("/")
    return url

def get_path(path):
    """
    Get Path.
    """

    if path.startswith('.') or os.path.isabs(path) or not os.path.isdir(os.path.join(MEDIA_ROOT, DIRECTORY, path)):
        return None
    return path

def get_file(path, filename):
    """
    Get File.
    """

    if not os.path.isfile(os.path.join(MEDIA_ROOT, DIRECTORY, path, filename)) and not os.path.isdir(os.path.join(MEDIA_ROOT, DIRECTORY, path, filename)):
        return None
    return filename

def get_breadcrumbs(query, path):
    """
    Get breadcrumbs.
    """

    breadcrumbs = []
    dir_query = ""
    if path:
        for item in path.split(os.sep):
            dir_query = os.path.join(dir_query,item)
            breadcrumbs.append([item,dir_query])
    return breadcrumbs

def get_filterdate(filterDate, dateTime):
    """
    Get filterdate.
    """

    returnvalue = ''
    dateYear = strftime("%Y", gmtime(dateTime))
    dateMonth = strftime("%m", gmtime(dateTime))
    dateDay = strftime("%d", gmtime(dateTime))
    if filterDate == 'today' and int(dateYear) == int(localtime()[0]) and int(dateMonth) == int(localtime()[1]) and int(dateDay) == int(localtime()[2]): returnvalue = 'true'
    elif filterDate == 'thismonth' and dateTime >= time()-2592000: returnvalue = 'true'
    elif filterDate == 'thisyear' and int(dateYear) == int(localtime()[0]): returnvalue = 'true'
    elif filterDate == 'past7days' and dateTime >= time()-604800: returnvalue = 'true'
    elif filterDate == '': returnvalue = 'true'
    return returnvalue

def get_settings_var():
    """
    Get settings variables used for FileBrowser listing.
    """

    settings_var = {}
    # Main
    settings_var['DEBUG'] = DEBUG
    settings_var['MEDIA_ROOT'] = MEDIA_ROOT
    settings_var['MEDIA_URL'] = MEDIA_URL
    settings_var['DIRECTORY'] = DIRECTORY
    # FileBrowser
    settings_var['URL_FILEBROWSER_MEDIA'] = URL_FILEBROWSER_MEDIA
    settings_var['PATH_FILEBROWSER_MEDIA'] = PATH_FILEBROWSER_MEDIA
    # TinyMCE
    settings_var['URL_TINYMCE'] = URL_TINYMCE
    settings_var['PATH_TINYMCE'] = PATH_TINYMCE
    # Extensions/Formats (for FileBrowseField)
    settings_var['EXTENSIONS'] = EXTENSIONS
    settings_var['SELECT_FORMATS'] = SELECT_FORMATS
    # Versions
    settings_var['VERSIONS_BASEDIR'] = VERSIONS_BASEDIR
    settings_var['VERSIONS'] = VERSIONS
    settings_var['ADMIN_VERSIONS'] = ADMIN_VERSIONS
    settings_var['ADMIN_THUMBNAIL'] = ADMIN_THUMBNAIL
    # FileBrowser Options
    settings_var['MAX_UPLOAD_SIZE'] = MAX_UPLOAD_SIZE
    # Convert Filenames
    settings_var['CONVERT_FILENAME'] = CONVERT_FILENAME
    return settings_var

def handle_file_upload(path, file):
    """
    Handle File Upload.
    """

    file_path = os.path.join(path, file.name)
    uploadedfile = default_storage.save(file_path, file)
    return uploadedfile

def get_file_type(filename):
    """
    Get file type as defined in EXTENSIONS.
    """

    file_extension = os.path.splitext(filename)[1].lower()
    file_type = ''
    for k,v in EXTENSIONS.iteritems():
        for extension in v:
            if file_extension == extension.lower():
                file_type = k
    return file_type

def is_selectable(filename, selecttype):
    """
    Get select type as defined in FORMATS.
    """

    file_extension = os.path.splitext(filename)[1].lower()
    select_types = []
    for k,v in SELECT_FORMATS.iteritems():
        for extension in v:
            if file_extension == extension.lower():
                select_types.append(k)
    return select_types

def version_generator(value, version_prefix, force=None):
    """
    Generate Version for an Image.
    value has to be a serverpath relative to MEDIA_ROOT.
    """

    # PIL's Error "Suspension not allowed here" work around:
    # s. http://mail.python.org/pipermail/image-sig/1999-August/000816.html
    if STRICT_PIL:
        from PIL import ImageFile
    else:
        try:
            from PIL import ImageFile
        except ImportError:
            import ImageFile
    ImageFile.MAXBLOCK = IMAGE_MAXBLOCK # default is 64k

    try:
        im = Image.open(os.path.join(MEDIA_ROOT, value))
        version_path = get_version_path(value, version_prefix)
        absolute_version_path = os.path.join(MEDIA_ROOT, version_path)
        version_dir = os.path.split(absolute_version_path)[0]
        if not os.path.isdir(version_dir):
            os.makedirs(version_dir)
            os.chmod(version_dir, 0775)
        version = scale_and_crop(im, VERSIONS[version_prefix]['width'], VERSIONS[version_prefix]['height'], VERSIONS[version_prefix]['opts'])
        try:
            version.save(absolute_version_path, quality=90, optimize=(os.path.splitext(version_path)[1].lower() != '.gif'))
        except IOError:
            version.save(absolute_version_path, quality=90)
        return version_path
    except:
        return None

def scale_and_crop(im, width, height, opts):
    """
    Scale and Crop.
    """

    x, y   = [float(v) for v in im.size]
    if width:
        xr = float(width)
    else:
        xr = float(x*height/y)
    if height:
        yr = float(height)
    else:
        yr = float(y*width/x)

    if 'crop' in opts:
        r = max(xr/x, yr/y)
    else:
        r = min(xr/x, yr/y)

    if r < 1.0 or (r > 1.0 and 'upscale' in opts):
        im = im.resize((int(x*r), int(y*r)), resample=Image.ANTIALIAS)

    if 'crop' in opts:
        x, y   = [float(v) for v in im.size]
        ex, ey = (x-min(x, xr))/2, (y-min(y, yr))/2
        if ex or ey:
            im = im.crop((int(ex), int(ey), int(x-ex), int(y-ey)))
    return im

    # if 'crop' in opts:
    #     if 'top_left' in opts:
    #         #draw cropping box from upper left corner of image
    #         box = (0, 0, int(min(x, xr)), int(min(y, yr)))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'top_right' in opts:
    #         #draw cropping box from upper right corner of image
    #         box = (int(x-min(x, xr)), 0, int(x), int(min(y, yr)))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'bottom_left' in opts:
    #         #draw cropping box from lower left corner of image
    #         box = (0, int(y-min(y, yr)), int(xr), int(y))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     elif 'bottom_right' in opts:
    #         #draw cropping box from lower right corner of image
    #         (int(x-min(x, xr)), int(y-min(y, yr)), int(x), int(y))
    #         im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    #     else:
    #         ex, ey = (x-min(x, xr))/2, (y-min(y, yr))/2
    #         if ex or ey:
    #             box = (int(ex), int(ey), int(x-ex), int(y-ey))
    #             im = im.resize((int(x), int(y)), resample=Image.ANTIALIAS).crop(box)
    # return im

scale_and_crop.valid_options = ('crop', 'upscale')

def convert_filename(value):
    """
    Convert Filename.
    """

    if CONVERT_FILENAME:
        return value.replace(" ", "_").lower()
    else:
        return value

<|endoftext|>


<commit_before>def duration(value):
    ONE_SECOND = 1000
    ONE_MINUTE = ONE_SECOND * 60

    if not value:
        return '0 s'

    if value < 3 * ONE_SECOND:
        return '%d ms' % (value,)
    elif value < 12 * ONE_MINUTE:
        return '%d s' % (value / ONE_SECOND,)
    else:
        return '%d m' % (value / ONE_MINUTE,)
<commit_msg>Show timing in minutes if its more than 3 minutes<commit_after>def duration(value):
    ONE_SECOND = 1000
    ONE_MINUTE = ONE_SECOND * 60

    if not value:
        return '0 s'

    if value < 3 * ONE_SECOND:
        return '%d ms' % (value,)
    elif value < 5 * ONE_MINUTE:
        return '%d s' % (value / ONE_SECOND,)
    else:
        return '%d m' % (value / ONE_MINUTE,)
<|endoftext|>


<commit_before>import copy
import operator
import random
from .evaluator import Evaluator

class LazyValue:
	pass

class LazyNum(LazyValue):
	def __init__(self):
		self._neg = False

	def evaluate(self, source) -> int:
		raise NotImplementedError

	def _cmp(op):
		def func(self, other):
			if isinstance(other, (int, LazyNum)):
				# When comparing a LazyNum with an int, turn it into an
				# Evaluator that compares the int to the result of the LazyNum
				return LazyNumEvaluator(self, other, getattr(operator, op))
			return getattr(super(), "__%s__" % (op))(other)
		return func

	__eq__ = _cmp("eq")
	__ge__ = _cmp("ge")
	__gt__ = _cmp("gt")
	__le__ = _cmp("le")
	__lt__ = _cmp("lt")

	def __neg__(self):
		ret = copy.copy(self)
		ret._neg = not self._neg
		return ret

	def num(self, n):
		return -n if self._neg else n

	def get_entities(self, source):
		from .selector import Selector
		if isinstance(self.selector, Selector):
			entities = self.selector.eval(source.game, source)
		elif isinstance(self.selector, LazyValue):
			entities = [self.selector.evaluate(source)]
		else:
			# TODO assert that self.selector is a TargetedAction
			entities = sum(self.selector.trigger(source), [])
		return entities

class LazyNumEvaluator(Evaluator):
	def __init__(self, num, other, cmp):
		super().__init__()
		self.num = num
		self.other = other
		self.cmp = cmp

	def check(self, source):
		num = self.num.evaluate(source)
		other = self.other
		if isinstance(other, LazyNum):
			other = other.evaluate(source)
		return self.cmp(num, other)

class Count(LazyNum):
	"""
	Lazily count the matches in a selector
	"""
	def __init__(self, selector):
		super().__init__()
		self.selector = selector

	def __repr__(self):
		return "%s(%r)" % (self.__class__.__name__, self.selector)

	def evaluate(self, source):
		return self.num(len(self.get_entities(source)))

class Attr(LazyNum):
	"""
	Lazily evaluate the sum of all tags in a selector
	"""
	def __init__(self, selector, tag):
		super().__init__()
		self.selector = selector
		self.tag = tag

	def __repr__(self):
		return "%s(%r, %r)" % (self.__class__.__name__, self.selector, self.tag)

	def evaluate(self, source):
		entities = self.get_entities(source)
		if isinstance(self.tag, str):
			ret = sum(getattr(e, self.tag) for e in entities)
		else:
			# XXX: int() because of CardList counter tags
			ret = sum(int(e.tags[self.tag]) for e in entities)
		return self.num(ret)

class RandomNumber(LazyNum):
	def __init__(self, *args):
		super().__init__()
		self.choices = args

	def __repr__(self):
		return "%s(%r)" % (self.__class__.__name__, self.choices)

	def evaluate(self, source):
		return self.num(random.choice(self.choices))
<commit_msg>Add a __repr__ for LazyNumEvaluator<commit_after>import copy
import operator
import random
from .evaluator import Evaluator

class LazyValue:
	pass

class LazyNum(LazyValue):
	def __init__(self):
		self._neg = False

	def evaluate(self, source) -> int:
		raise NotImplementedError

	def _cmp(op):
		def func(self, other):
			if isinstance(other, (int, LazyNum)):
				# When comparing a LazyNum with an int, turn it into an
				# Evaluator that compares the int to the result of the LazyNum
				return LazyNumEvaluator(self, other, getattr(operator, op))
			return getattr(super(), "__%s__" % (op))(other)
		return func

	__eq__ = _cmp("eq")
	__ge__ = _cmp("ge")
	__gt__ = _cmp("gt")
	__le__ = _cmp("le")
	__lt__ = _cmp("lt")

	def __neg__(self):
		ret = copy.copy(self)
		ret._neg = not self._neg
		return ret

	def num(self, n):
		return -n if self._neg else n

	def get_entities(self, source):
		from .selector import Selector
		if isinstance(self.selector, Selector):
			entities = self.selector.eval(source.game, source)
		elif isinstance(self.selector, LazyValue):
			entities = [self.selector.evaluate(source)]
		else:
			# TODO assert that self.selector is a TargetedAction
			entities = sum(self.selector.trigger(source), [])
		return entities

class LazyNumEvaluator(Evaluator):
	def __init__(self, num, other, cmp):
		super().__init__()
		self.num = num
		self.other = other
		self.cmp = cmp

	def __repr__(self):
		return "%s(%r, %r)" % (self.cmp.__name__, self.num, self.other)

	def check(self, source):
		num = self.num.evaluate(source)
		other = self.other
		if isinstance(other, LazyNum):
			other = other.evaluate(source)
		return self.cmp(num, other)

class Count(LazyNum):
	"""
	Lazily count the matches in a selector
	"""
	def __init__(self, selector):
		super().__init__()
		self.selector = selector

	def __repr__(self):
		return "%s(%r)" % (self.__class__.__name__, self.selector)

	def evaluate(self, source):
		return self.num(len(self.get_entities(source)))

class Attr(LazyNum):
	"""
	Lazily evaluate the sum of all tags in a selector
	"""
	def __init__(self, selector, tag):
		super().__init__()
		self.selector = selector
		self.tag = tag

	def __repr__(self):
		return "%s(%r, %r)" % (self.__class__.__name__, self.selector, self.tag)

	def evaluate(self, source):
		entities = self.get_entities(source)
		if isinstance(self.tag, str):
			ret = sum(getattr(e, self.tag) for e in entities)
		else:
			# XXX: int() because of CardList counter tags
			ret = sum(int(e.tags[self.tag]) for e in entities)
		return self.num(ret)

class RandomNumber(LazyNum):
	def __init__(self, *args):
		super().__init__()
		self.choices = args

	def __repr__(self):
		return "%s(%r)" % (self.__class__.__name__, self.choices)

	def evaluate(self, source):
		return self.num(random.choice(self.choices))
<|endoftext|>