<commit_before>import os
from numpy import pi, seterr, linspace

from skyfield.api import load
from skyfield.data.mpc import load_comets_dataframe
from skyfield.elementslib import OsculatingElements
from skyfield.keplerlib import KeplerOrbit, propagate
from skyfield.tests.test_elementslib import compare, ele_to_vec
from skyfield.units import Angle, Distance, Velocity

try:
    from io import BytesIO
except:
    from StringIO import StringIO as BytesIO

seterr(all='raise')

# Test against HORIZONS.

def test_against_horizons():
    # See the following files in the Skyfield repository:
    #
    # horizons/ceres-orbital-elements
    # horizons/ceres-position

    ts = load.timescale(builtin=True)
    t = ts.tdb_jd(2458886.500000000)

    a = 2.768873850275102E+00 # A
    e = 7.705857791518426E-02 # EC
    p_au = a * (1 - e*e)   # Wikipedia

    k = KeplerOrbit.from_mean_anomaly(
        p=Distance(au=p_au),
        e=e,
        i=Angle(degrees=2.718528770987308E+01),
        Om=Angle(degrees=2.336112629072238E+01),
        w=Angle(degrees=1.328964361683606E+02),
        M=Angle(degrees=1.382501360489816E+02),
        epoch=t,
        mu_au3_d2=2.9591220828559093E-04,
        center=None,
        target=None,
    )
    r, v = k._at(t)[:2]
    sun_au = [
        -0.004105894975783999, 0.006739680703224941, 0.002956344702049446,
    ]
    horizons_au = [
        1.334875927366032E+00, -2.239607658161781E+00, -1.328895183461897E+00,
    ]
    epsilon = Distance(m=0.001).au
    assert abs(r + sun_au - horizons_au).max() < epsilon

def test_comet():
    text = (b'    CJ95O010  1997 03 29.6333  0.916241  0.994928  130.6448'
            b'  283.3593   88.9908  20200224  -2.0  4.0  C/1995 O1 (Hale-Bopp)'
            b'                                    MPC106342\n')
    df = load_comets_dataframe(BytesIO(text))
    row = df.ix[0]

    ts = load.timescale(builtin=True)
    eph = load('de421.bsp')
    k = KeplerOrbit.from_comet_row(ts, row)

    t = ts.utc(2020, 5, 31)
    p = eph['earth'].at(t).observe(eph['sun'] + k)
    ra, dec, distance = p.radec()

    # The file authorities/mpc-hale-bopp in the repository is the source
    # of these angles.  TODO: can we tighten this bound and drive it to
    # fractions of an arcsecond?

    ra_want = Angle(hours=(23, 59, 16.6))
    dec_want = Angle(degrees=(-84, 46, 58))
    assert abs(ra_want.arcseconds() - ra.arcseconds()) < 2.0
    assert abs(dec_want.arcseconds() - dec.arcseconds()) < 0.2

# Test various round-trips through the kepler orbit object.

def _data_path(filename):
    return os.path.join(os.path.dirname(__file__), 'data', filename)

def check_orbit(p, e, i, Om, w, v,
                p_eps=None, e_eps=None, i_eps=None, Om_eps=None, w_eps=None, v_eps=None):
    pos0, vel0 = ele_to_vec(p, e, i, Om, w, v, mu)

    pos1, vel1 = propagate(pos0, vel0, 0, times, mu)
    ele = OsculatingElements(Distance(km=pos1), Velocity(km_per_s=vel1), dummy_time, mu)

    if p_eps: compare(p, ele.semi_latus_rectum.km, p_eps)
    if e_eps: compare(e, ele.eccentricity, e_eps)
    if i_eps: compare(i, ele.inclination.radians, i_eps, mod=True)
    if Om_eps: compare(Om, ele.longitude_of_ascending_node.radians, Om_eps, mod=True)
    if w_eps: compare(w, ele.argument_of_periapsis.radians, w_eps, mod=True)
    if v_eps: compare(v, ele.true_anomaly.radians, v_eps, mod=True)

times = linspace(-1e11, 1e11, 1001) # -3170 years to +3170 years, including 0
mu = 403503.2355022598
dummy_time = load.timescale().utc(2018)

def test_circular():
    check_orbit(p=300000, e=0, i=.5, Om=1, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15)

def test_circular_equatorial():
    check_orbit(p=300000, e=0, i=0, Om=0, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15)

def test_circular_polar():
    check_orbit(p=300000, e=0, i=pi/2, Om=1, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15)

def test_elliptical():
    check_orbit(p=300000, e=.3, i=1, Om=0, w=4, v=5,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-7)

def test_elliptical_equatorial():
    check_orbit(p=300000, e=.3, i=0, Om=0, w=1, v=5,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-7)

def test_elliptical_polar():
    check_orbit(p=300000, e=.2, i=pi/2, Om=1, w=2, v=3,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-8)

def test_parabolic():
    check_orbit(p=300000, e=1, i=1, Om=0, w=4, v=3,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-13, Om_eps=1e-13, w_eps=1e-13)

def test_parabolic_equatorial():
    check_orbit(p=300000, e=1, i=0, Om=0, w=1, v=2,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-13)

def test_parabolic_polar():
    check_orbit(p=300000, e=1, i=pi/2, Om=1, w=2, v=3,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-14, Om_eps=1e-13, w_eps=1e-13)

def test_hyperbolic():
    check_orbit(p=300000, e=1.3, i=1, Om=0, w=4, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-10, Om_eps=1e-10, w_eps=1e-6)

def test_hyperbolic_equatorial():
    check_orbit(p=300000, e=1.3, i=0, Om=0, w=1, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-6)

def test_hyperbolic_polar():
    check_orbit(p=300000, e=1.3, i=pi/2, Om=1, w=2, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-10, Om_eps=1e-10, w_eps=1e-6)
<commit_msg>Switch test away from deprecated ".ix", to fix CI<commit_after>import os
from numpy import pi, seterr, linspace

from skyfield.api import load
from skyfield.data.mpc import load_comets_dataframe
from skyfield.elementslib import OsculatingElements
from skyfield.keplerlib import KeplerOrbit, propagate
from skyfield.tests.test_elementslib import compare, ele_to_vec
from skyfield.units import Angle, Distance, Velocity

try:
    from io import BytesIO
except:
    from StringIO import StringIO as BytesIO

seterr(all='raise')

# Test against HORIZONS.

def test_against_horizons():
    # See the following files in the Skyfield repository:
    #
    # horizons/ceres-orbital-elements
    # horizons/ceres-position

    ts = load.timescale(builtin=True)
    t = ts.tdb_jd(2458886.500000000)

    a = 2.768873850275102E+00 # A
    e = 7.705857791518426E-02 # EC
    p_au = a * (1 - e*e)   # Wikipedia

    k = KeplerOrbit.from_mean_anomaly(
        p=Distance(au=p_au),
        e=e,
        i=Angle(degrees=2.718528770987308E+01),
        Om=Angle(degrees=2.336112629072238E+01),
        w=Angle(degrees=1.328964361683606E+02),
        M=Angle(degrees=1.382501360489816E+02),
        epoch=t,
        mu_au3_d2=2.9591220828559093E-04,
        center=None,
        target=None,
    )
    r, v = k._at(t)[:2]
    sun_au = [
        -0.004105894975783999, 0.006739680703224941, 0.002956344702049446,
    ]
    horizons_au = [
        1.334875927366032E+00, -2.239607658161781E+00, -1.328895183461897E+00,
    ]
    epsilon = Distance(m=0.001).au
    assert abs(r + sun_au - horizons_au).max() < epsilon

def test_comet():
    text = (b'    CJ95O010  1997 03 29.6333  0.916241  0.994928  130.6448'
            b'  283.3593   88.9908  20200224  -2.0  4.0  C/1995 O1 (Hale-Bopp)'
            b'                                    MPC106342\n')
    df = load_comets_dataframe(BytesIO(text))
    row = df.iloc[0]

    ts = load.timescale(builtin=True)
    eph = load('de421.bsp')
    k = KeplerOrbit.from_comet_row(ts, row)

    t = ts.utc(2020, 5, 31)
    p = eph['earth'].at(t).observe(eph['sun'] + k)
    ra, dec, distance = p.radec()

    # The file authorities/mpc-hale-bopp in the repository is the source
    # of these angles.  TODO: can we tighten this bound and drive it to
    # fractions of an arcsecond?

    ra_want = Angle(hours=(23, 59, 16.6))
    dec_want = Angle(degrees=(-84, 46, 58))
    assert abs(ra_want.arcseconds() - ra.arcseconds()) < 2.0
    assert abs(dec_want.arcseconds() - dec.arcseconds()) < 0.2

# Test various round-trips through the kepler orbit object.

def _data_path(filename):
    return os.path.join(os.path.dirname(__file__), 'data', filename)

def check_orbit(p, e, i, Om, w, v,
                p_eps=None, e_eps=None, i_eps=None, Om_eps=None, w_eps=None, v_eps=None):
    pos0, vel0 = ele_to_vec(p, e, i, Om, w, v, mu)

    pos1, vel1 = propagate(pos0, vel0, 0, times, mu)
    ele = OsculatingElements(Distance(km=pos1), Velocity(km_per_s=vel1), dummy_time, mu)

    if p_eps: compare(p, ele.semi_latus_rectum.km, p_eps)
    if e_eps: compare(e, ele.eccentricity, e_eps)
    if i_eps: compare(i, ele.inclination.radians, i_eps, mod=True)
    if Om_eps: compare(Om, ele.longitude_of_ascending_node.radians, Om_eps, mod=True)
    if w_eps: compare(w, ele.argument_of_periapsis.radians, w_eps, mod=True)
    if v_eps: compare(v, ele.true_anomaly.radians, v_eps, mod=True)

times = linspace(-1e11, 1e11, 1001) # -3170 years to +3170 years, including 0
mu = 403503.2355022598
dummy_time = load.timescale().utc(2018)

def test_circular():
    check_orbit(p=300000, e=0, i=.5, Om=1, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15)

def test_circular_equatorial():
    check_orbit(p=300000, e=0, i=0, Om=0, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15)

def test_circular_polar():
    check_orbit(p=300000, e=0, i=pi/2, Om=1, w=0, v=1,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15)

def test_elliptical():
    check_orbit(p=300000, e=.3, i=1, Om=0, w=4, v=5,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-7)

def test_elliptical_equatorial():
    check_orbit(p=300000, e=.3, i=0, Om=0, w=1, v=5,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-7)

def test_elliptical_polar():
    check_orbit(p=300000, e=.2, i=pi/2, Om=1, w=2, v=3,
                p_eps=1e-2, e_eps=1e-8, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-8)

def test_parabolic():
    check_orbit(p=300000, e=1, i=1, Om=0, w=4, v=3,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-13, Om_eps=1e-13, w_eps=1e-13)

def test_parabolic_equatorial():
    check_orbit(p=300000, e=1, i=0, Om=0, w=1, v=2,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-13)

def test_parabolic_polar():
    check_orbit(p=300000, e=1, i=pi/2, Om=1, w=2, v=3,
                p_eps=1e-5, e_eps=1e-14, i_eps=1e-14, Om_eps=1e-13, w_eps=1e-13)

def test_hyperbolic():
    check_orbit(p=300000, e=1.3, i=1, Om=0, w=4, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-10, Om_eps=1e-10, w_eps=1e-6)

def test_hyperbolic_equatorial():
    check_orbit(p=300000, e=1.3, i=0, Om=0, w=1, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-15, Om_eps=1e-15, w_eps=1e-6)

def test_hyperbolic_polar():
    check_orbit(p=300000, e=1.3, i=pi/2, Om=1, w=2, v=.5,
                p_eps=1e0, e_eps=1e-6, i_eps=1e-10, Om_eps=1e-10, w_eps=1e-6)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',

    url(r'^$', 'homepage.views.home', name='home'),
    url(r'(?P<coin_symbol>[-\w]+)/latest-block$', 'blocks.views.latest_block', name='latest_block'),
    url(r'(?P<coin_symbol>[-\w]+)/address/(?P<address>\w+)$', 'addresses.views.address_overview', name='address_overview'),
    url(r'(?P<coin_symbol>[-\w]+)/tx/(?P<tx_hash>\w+)$', 'transactions.views.transaction_overview', name='transaction_overview'),
    url(r'(?P<coin_symbol>[-\w]+)/block/(?P<block_representation>\w+)$', 'blocks.views.block_overview', name='block_overview'),

    url(r'^admin/', include(admin.site.urls)),
)
<commit_msg>update urls.py for coin overview<commit_after>from django.conf.urls import patterns, include, url
from django.contrib import admin

urlpatterns = patterns('',

    url(r'^$', 'homepage.views.home', name='home'),
    url(r'(?P<coin_symbol>[-\w]+)/latest-block/$', 'blocks.views.latest_block', name='latest_block'),
    url(r'(?P<coin_symbol>[-\w]+)/address/(?P<address>\w+)/$', 'addresses.views.address_overview', name='address_overview'),
    url(r'(?P<coin_symbol>[-\w]+)/tx/(?P<tx_hash>\w+)/$', 'transactions.views.transaction_overview', name='transaction_overview'),
    url(r'(?P<coin_symbol>[-\w]+)/block/(?P<block_representation>\w+)/$', 'blocks.views.block_overview', name='block_overview'),
    url(r'(?P<coin_symbol>[-\w]+)/$', 'homepage.views.coin_overview', name='coin_overview'),

    url(r'^admin/', include(admin.site.urls)),
)
<|endoftext|>


<commit_before>"""
Initializes the database of cea
"""

# HISTORY:
# J. A. Fonseca  script development          03.02.20

from __future__ import absolute_import
from __future__ import division

import cea.config
import cea.inputlocator
import os
from distutils.dir_util import copy_tree

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2015, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas", "Martin Mosteiro"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def data_initializer(locator,
                     databases_path,
                     initialize_archetypes_database=True,
                     initialize_feedstocks_database=True,
                     initialize_systems_database=True,
                     ):

    output_directory = locator.get_databases_folder()
    print("Copying databases from {source}".format(source=databases_path))
    print("Copying databases to {path}".format(path=output_directory))

    if initialize_archetypes_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'archetypes')
            complete_output_directory = locator.get_databases_archetypes_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'archetypes' database in the path you indicated, please check the spelling")

    if initialize_feedstocks_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'feedstocks')
            complete_output_directory = locator.get_databases_feedstocks_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'lifecycle' database in the path you indicated, please check the spelling")

    if initialize_systems_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'systems')
            complete_output_directory = locator.get_databases_systems_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'systems' database in the path you indicated, please check the spelling")

def main(config):
    """
    Run the properties script with input from the reference case and compare the results. This ensures that changes
    made to this script (e.g. refactorings) do not stop the script from working and also that the results stay the same.
    """

    print('Running data-intializer with scenario = %s' % config.scenario)
    print('Running data-intializer with databases located in = %s' % config.data_initializer.databases_path)
    locator = cea.inputlocator.InputLocator(config.scenario)

    initialize_archetypes_database = 'archetypes' in config.data_initializer.databases
    initialize_feedstocks_database = 'feedstocks' in config.data_initializer.databases
    initialize_systems_database = 'systems' in config.data_initializer.databases

    data_initializer(locator=locator,
                     databases_path=config.data_initializer.databases_path,
                     initialize_archetypes_database=initialize_archetypes_database,
                     initialize_feedstocks_database=initialize_feedstocks_database,
                     initialize_systems_database=initialize_systems_database
                     )

if __name__ == '__main__':
    main(cea.config.Configuration())
<commit_msg>Update data_initializer.py<commit_after>"""
Initializes the database of cea
"""

# HISTORY:
# J. A. Fonseca  script development          03.02.20

from __future__ import absolute_import
from __future__ import division

import cea.config
import cea.inputlocator
import os
from distutils.dir_util import copy_tree

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2015, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca", "Daren Thomas", "Martin Mosteiro"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

def data_initializer(locator,
                     databases_path,
                     initialize_archetypes_database=True,
                     initialize_feedstocks_database=True,
                     initialize_components_database=True,
                     initialize_assemblies_database=True,
                     ):

    output_directory = locator.get_databases_folder()
    print("Copying databases from {source}".format(source=databases_path))
    print("Copying databases to {path}".format(path=output_directory))

    if initialize_archetypes_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'archetypes')
            complete_output_directory = locator.get_databases_archetypes_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'archetypes' database in the path you indicated, please check the spelling")

    if initialize_feedstocks_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'feedstocks')
            complete_output_directory = locator.get_databases_feedstocks_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'lifecycle' database in the path you indicated, please check the spelling")

    if initialize_components_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'components')
            complete_output_directory = locator.get_databases_systems_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'components' database in the path you indicated, please check the spelling")

    if initialize_assemblies_database:
        try:
            complete_databases_path = os.path.join(databases_path, 'assemblies')
            complete_output_directory = locator.get_databases_assemblies_folder()
            copy_tree(complete_databases_path, complete_output_directory)
        except:
            raise Exception("we could not find the 'assemblies' database in the path you indicated, please check the spelling")

def main(config):
    """
    Run the properties script with input from the reference case and compare the results. This ensures that changes
    made to this script (e.g. refactorings) do not stop the script from working and also that the results stay the same.
    """

    print('Running data-intializer with scenario = %s' % config.scenario)
    print('Running data-intializer with databases located in = %s' % config.data_initializer.databases_path)
    locator = cea.inputlocator.InputLocator(config.scenario)

    initialize_archetypes_database = 'archetypes' in config.data_initializer.databases
    initialize_feedstocks_database = 'feedstocks' in config.data_initializer.databases
    initialize_components_database = 'components' in config.data_initializer.databases
    initialize_assemblies_database = 'assemblies' in config.data_initializer.databases

    data_initializer(locator=locator,
                     databases_path=config.data_initializer.databases_path,
                     initialize_archetypes_database=initialize_archetypes_database,
                     initialize_feedstocks_database=initialize_feedstocks_database,
                     initialize_components_database=initialize_components_database,
                     initialize_assemblies_database=initialize_assemblies_database
                     )

if __name__ == '__main__':
    main(cea.config.Configuration())
<|endoftext|>


<commit_before>import sys
import os.path
import argparse

from . import crawler
from .morss import FeedFetch, FeedGather, FeedFormat
from .morss import Options
from .morss import log

def cli_app():
    parser = argparse.ArgumentParser(
        prog='morss',
        description='Get full-text RSS feeds',
        epilog='GNU AGPLv3 code'
        )

    parser.add_argument('url', help='feed url')

    group = parser.add_argument_group('output')
    group.add_argument('--format', default='rss', choices=('rss', 'json', 'html', 'csv'), help='output format')
    group.add_argument('--search', action='store', type=str, metavar='STRING', help='does a basic case-sensitive search in the feed')
    group.add_argument('--clip', action='store_true', help='stick the full article content under the original feed content (useful for twitter)')
    group.add_argument('--indent', action='store_true', help='returns indented XML or JSON, takes more place, but human-readable')

    group = parser.add_argument_group('action')
    group.add_argument('--cache', action='store_true', help='only take articles from the cache (ie. don\'t grab new articles\' content), so as to save time')
    group.add_argument('--force', action='store_true', help='force refetch the rss feed and articles')
    group.add_argument('--proxy', action='store_true', help='doesn\'t fill the articles')
    group.add_argument('--newest', action='store_true', help='return the feed items in chronological order (morss ohterwise shows the items by appearing order)')
    group.add_argument('--firstlink', action='store_true', help='pull the first article mentioned in the description instead of the default link')

    group = parser.add_argument_group('custom feeds')
    group.add_argument('--items', action='store', type=str, metavar='XPATH', help='(mandatory to activate the custom feeds function) xpath rule to match all the RSS entries')
    group.add_argument('--item_link', action='store', type=str, metavar='XPATH', help='xpath rule relative to items to point to the entry\'s link')
    group.add_argument('--item_title', action='store', type=str, metavar='XPATH', help='entry\'s title')
    group.add_argument('--item_content', action='store', type=str, metavar='XPATH', help='entry\'s content')
    group.add_argument('--item_time', action='store', type=str, metavar='XPATH', help='entry\'s date & time (accepts a wide range of time formats)')

    group = parser.add_argument_group('misc')
    group.add_argument('--nolink', action='store_true', help='drop links, but keeps links\' inner text')
    group.add_argument('--noref', action='store_true', help='drop items\' link')
    group.add_argument('--silent', action='store_true', help='don\'t output the final RSS (useless on its own, but can be nice when debugging)')

    options = Options(parser.parse_args())
    url = options.url

    global DEBUG
    DEBUG = options.debug

    crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db'))

    url, rss = FeedFetch(url, options)
    rss = FeedGather(rss, url, options)
    out = FeedFormat(rss, options, 'unicode')

    if not options.silent:
        print(out)

    log('done')
<commit_msg>cli.py: remove log<commit_after>import sys
import os.path
import argparse

from . import crawler
from .morss import FeedFetch, FeedGather, FeedFormat
from .morss import Options

def cli_app():
    parser = argparse.ArgumentParser(
        prog='morss',
        description='Get full-text RSS feeds',
        epilog='GNU AGPLv3 code'
        )

    parser.add_argument('url', help='feed url')

    group = parser.add_argument_group('output')
    group.add_argument('--format', default='rss', choices=('rss', 'json', 'html', 'csv'), help='output format')
    group.add_argument('--search', action='store', type=str, metavar='STRING', help='does a basic case-sensitive search in the feed')
    group.add_argument('--clip', action='store_true', help='stick the full article content under the original feed content (useful for twitter)')
    group.add_argument('--indent', action='store_true', help='returns indented XML or JSON, takes more place, but human-readable')

    group = parser.add_argument_group('action')
    group.add_argument('--cache', action='store_true', help='only take articles from the cache (ie. don\'t grab new articles\' content), so as to save time')
    group.add_argument('--force', action='store_true', help='force refetch the rss feed and articles')
    group.add_argument('--proxy', action='store_true', help='doesn\'t fill the articles')
    group.add_argument('--newest', action='store_true', help='return the feed items in chronological order (morss ohterwise shows the items by appearing order)')
    group.add_argument('--firstlink', action='store_true', help='pull the first article mentioned in the description instead of the default link')

    group = parser.add_argument_group('custom feeds')
    group.add_argument('--items', action='store', type=str, metavar='XPATH', help='(mandatory to activate the custom feeds function) xpath rule to match all the RSS entries')
    group.add_argument('--item_link', action='store', type=str, metavar='XPATH', help='xpath rule relative to items to point to the entry\'s link')
    group.add_argument('--item_title', action='store', type=str, metavar='XPATH', help='entry\'s title')
    group.add_argument('--item_content', action='store', type=str, metavar='XPATH', help='entry\'s content')
    group.add_argument('--item_time', action='store', type=str, metavar='XPATH', help='entry\'s date & time (accepts a wide range of time formats)')

    group = parser.add_argument_group('misc')
    group.add_argument('--nolink', action='store_true', help='drop links, but keeps links\' inner text')
    group.add_argument('--noref', action='store_true', help='drop items\' link')
    group.add_argument('--silent', action='store_true', help='don\'t output the final RSS (useless on its own, but can be nice when debugging)')

    options = Options(parser.parse_args())
    url = options.url

    global DEBUG
    DEBUG = options.debug

    crawler.default_cache = crawler.SQLiteCache(os.path.expanduser('~/.cache/morss-cache.db'))

    url, rss = FeedFetch(url, options)
    rss = FeedGather(rss, url, options)
    out = FeedFormat(rss, options, 'unicode')

    if not options.silent:
        print(out)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import os
import subprocess
import sys
import time

import yaml

import screenshotter
import uploader

class ElectionsScreenshotter(object):
    """
    Implements all screenshot-related logic.
    """

    def __init__(self):
        if len(sys.argv) != 2:
            self.usage()
            sys.exit(1)

        config_filepath = sys.argv[1]
        self.config = self.load_config(config_filepath)
        self.log_file = self.init_logging()
        self.screenshotter = screenshotter.Screenshotter(self.config)
        if not self.config.get('skip_upload'):
            self.uploader = uploader.Uploader(self.config)

    def main(self):
        """
        Runs through the full screenshot process.
        """

        print """
Screenshotter is running. To view its log file:

    tail -f %s

To quit, press ^C (ctrl-C).""" % (self.log_file)

        while True:
            images = self.screenshotter.take_screenshots()
            try:
                filepaths = self.uploader.upload_screenshots(images)
                if (self.config.get('delete_local_images')):
                    [os.remove(path.get('local_filepath')) for path in images]
            except AttributeError:
                pass
            time.sleep(self.config.get('time_between_screenshots'))

    def init_logging(self):
        """
        Create a log file, and attach a basic logger to it.
        """
        log_file = self.config.get('log_file')
        # Create the log file if it does not yet exist
        with open(log_file, 'a+'):
            pass
        logging.basicConfig(filename=log_file,
                format='%(levelname)s:%(asctime)s %(message)s',
                level=logging.INFO)
        return log_file

    def load_config(self, config_filepath):
        """
        Load configuration from config.yaml.

        This file includes the urls, css selectors and slugs for the images to
        screenshot.
        """
        config = {
            'skip_upload': False,
            'aws_bucket': '',
            'aws_subpath': '',
            'aws_access_key': None,
            'aws_secret_key': None,
            'log_file': os.path.dirname(os.path.abspath(__file__)) +
                   '/../screenshotter.log',
            'delete_local_images': False,
            'time_between_screenshots': 60,
            'override_css_file': None,
            'page_load_delay': 2,
            'wait_for_js_signal': False,
            'failure_timeout': 30,
        }

        required = (
            'images',
            'local_image_directory',
        )

        raw_config = None

        with open(config_filepath) as f:
            raw_config = yaml.load(f)

        for option_name, option_value in raw_config.iteritems():
            config[option_name] = option_value

        for option in required:
            try:
                config[option] = raw_config[option]
            except KeyError:
                raise RuntimeError('Config does not have required attribute: %s' % option)

        return config

    def usage(self):
        """
        Print usage information.
        """
        print """
        USAGE: elections_screenshotter CONFIG_FILEPATH
        """

def launch_new_instance():
    """
    Launch an instance of ElectionsScreenshotter.

    This is the entry function of the command-line tool
    `elections_screenshotter`.
    """
    try:
        s = ElectionsScreenshotter()
        s.main()
    except KeyboardInterrupt:
        # Print a blank line
        print

if __name__ == '__main__':
    launch_new_instance()
<commit_msg>Improve load config docstring<commit_after>#!/usr/bin/env python

import logging
import os
import subprocess
import sys
import time

import yaml

import screenshotter
import uploader

class ElectionsScreenshotter(object):
    """
    Implements all screenshot-related logic.
    """

    def __init__(self):
        if len(sys.argv) != 2:
            self.usage()
            sys.exit(1)

        config_filepath = sys.argv[1]
        self.config = self.load_config(config_filepath)
        self.log_file = self.init_logging()
        self.screenshotter = screenshotter.Screenshotter(self.config)
        if not self.config.get('skip_upload'):
            self.uploader = uploader.Uploader(self.config)

    def main(self):
        """
        Runs through the full screenshot process.
        """

        print """
Screenshotter is running. To view its log file:

    tail -f %s

To quit, press ^C (ctrl-C).""" % (self.log_file)

        while True:
            images = self.screenshotter.take_screenshots()
            try:
                filepaths = self.uploader.upload_screenshots(images)
                if (self.config.get('delete_local_images')):
                    [os.remove(path.get('local_filepath')) for path in images]
            except AttributeError:
                pass
            time.sleep(self.config.get('time_between_screenshots'))

    def init_logging(self):
        """
        Create a log file, and attach a basic logger to it.
        """
        log_file = self.config.get('log_file')
        # Create the log file if it does not yet exist
        with open(log_file, 'a+'):
            pass
        logging.basicConfig(filename=log_file,
                format='%(levelname)s:%(asctime)s %(message)s',
                level=logging.INFO)
        return log_file

    def load_config(self, config_filepath):
        """
        Load configuration from config.yaml.

        Many options have defaults; use these unless they are overwritten in
        config.yaml. This file includes the urls, css selectors and slugs for
        the images to screenshot.
        """
        config = {
            'skip_upload': False,
            'aws_bucket': '',
            'aws_subpath': '',
            'aws_access_key': None,
            'aws_secret_key': None,
            'log_file': os.path.dirname(os.path.abspath(__file__)) +
                   '/../screenshotter.log',
            'delete_local_images': False,
            'time_between_screenshots': 60,
            'override_css_file': None,
            'page_load_delay': 2,
            'wait_for_js_signal': False,
            'failure_timeout': 30,
        }

        required = (
            'images',
            'local_image_directory',
        )

        raw_config = None

        with open(config_filepath) as f:
            raw_config = yaml.load(f)

        for option_name, option_value in raw_config.iteritems():
            config[option_name] = option_value

        for option in required:
            try:
                config[option] = raw_config[option]
            except KeyError:
                raise RuntimeError('Config does not have required attribute: %s' % option)

        return config

    def usage(self):
        """
        Print usage information.
        """
        print """
        USAGE: elections_screenshotter CONFIG_FILEPATH
        """

def launch_new_instance():
    """
    Launch an instance of ElectionsScreenshotter.

    This is the entry function of the command-line tool
    `elections_screenshotter`.
    """
    try:
        s = ElectionsScreenshotter()
        s.main()
    except KeyboardInterrupt:
        # Print a blank line
        print

if __name__ == '__main__':
    launch_new_instance()
<|endoftext|>


<commit_before>import datetime

import xcsoar
from flask import current_app
from skylines.model import db
from skylines.lib import files
from skylines.lib.datetime import from_seconds_of_day
from skylines.lib.xcsoar_.flightpath import flight_path
from skylines.model import (
    Airport, Trace, FlightPhase, TimeZone, Location
)

def read_location(node):
    """
    Reads the latitude and longitude attributes of the node
    and returns a Location instance if the node was parsed correctly.
    """

    if node is None:
        return None

    if 'latitude' not in node or 'longitude' not in node:
        return None

    try:
        latitude = float(node['latitude'])
        longitude = float(node['longitude'])
        return Location(latitude=latitude, longitude=longitude)
    except ValueError:
        return None

def import_location_attribute(node, name):
    """
    Reads a Location instance from an attribute of the node
    with the given name.
    """

    if node is None or name not in node:
        return None

    location = node[name]
    return read_location(location)

def import_datetime_attribute(node, name):
    if node is None or name not in node:
        return None
    if isinstance(node[name], datetime.datetime):
        return node[name]
    else:
        return None

def find_contest(root, name):
    if 'contests' not in root:
        return None

    if name not in root['contests']:
        return None

    return root['contests'][name]

def find_trace(contest, name):
    if name not in contest:
        return None

    return contest[name]

def read_time_of_day(turnpoint, flight):
    if 'time' not in turnpoint:
        return None

    return from_seconds_of_day(flight.takeoff_time.date(),
                               int(turnpoint['time']))

def delete_trace(contest_name, trace_name, flight):
    q = Trace.query(
        flight=flight, contest_type=contest_name, trace_type=trace_name)
    q.delete()

def save_trace(contest_name, trace_name, node, flight):
    delete_trace(contest_name, trace_name, flight)

    if 'turnpoints' not in node:
        return

    locations = []
    times = []
    for turnpoint in node['turnpoints']:
        location = read_location(turnpoint['location'])
        time = read_time_of_day(turnpoint, flight)

        if location is None or time is None:
            continue

        locations.append(location)
        times.append(time)

    if len(locations) < 2 or len(times) < 2:
        return

    trace = Trace()
    trace.contest_type = contest_name
    trace.trace_type = trace_name
    trace.locations = locations
    trace.times = times

    if 'duration' in node:
        trace.duration = datetime.timedelta(seconds=int(node['duration']))

    if 'distance' in node:
        trace.distance = int(node['distance'])

    flight.traces.append(trace)

def save_contest(contest_name, traces, flight):
    for trace_name, trace in traces.iteritems():
        save_trace(contest_name, trace_name, trace, flight)

def save_contests(root, flight):
    if 'contests' not in root or flight.takeoff_time is None:
        # The takeoff_time is needed to convert the
        # time integer to a DateTime instance
        return

    for contest_name, traces in root['contests'].iteritems():
        save_contest(contest_name, traces, flight)

def get_takeoff_date(flight):
    if flight.takeoff_location is None:
        return flight.takeoff_time

    timezone = TimeZone.by_location(flight.takeoff_location)
    if timezone is None:
        return flight.takeoff_time

    return timezone.fromutc(flight.takeoff_time).date()

def save_takeoff(event, flight):
    flight.takeoff_time = import_datetime_attribute(event, 'time')
    flight.takeoff_location = read_location(event['location'])
    if flight.takeoff_location is not None:
        flight.takeoff_airport = Airport.by_location(flight.takeoff_location,
                                                     date=flight.takeoff_time)

    flight.date_local = get_takeoff_date(flight)

def save_landing(event, flight):
    flight.landing_time = import_datetime_attribute(event, 'time')
    flight.landing_location = read_location(event['location'])
    if flight.landing_location is not None:
        flight.landing_airport = Airport.by_location(flight.landing_location,
                                                     date=flight.landing_time)

def save_events(events, flight):
    if 'takeoff' in events:
        save_takeoff(events['takeoff'], flight)
    if 'scoring_start' in events:
        flight.scoring_start_time = import_datetime_attribute(events['scoring_start'], 'time')
    if 'scoring_end' in events:
        flight.scoring_end_time = import_datetime_attribute(events['scoring_end'], 'time')
    if 'landing' in events:
        save_landing(events['landing'], flight)

def save_phases(root, flight):
    flight.delete_phases()

    if 'phases' not in root or 'performance' not in root:
        return

    PT_IDX = {'': None,
              'powered': FlightPhase.PT_POWERED,
              'cruise': FlightPhase.PT_CRUISE,
              'circling': FlightPhase.PT_CIRCLING}

    CD_IDX = {'': None,
              'left': FlightPhase.CD_LEFT,
              'mixed': FlightPhase.CD_MIXED,
              'right': FlightPhase.CD_RIGHT,
              'total': FlightPhase.CD_TOTAL}

    for phdata in root['phases']:
        ph = FlightPhase()
        ph.flight = flight
        ph.aggregate = False
        ph.start_time = import_datetime_attribute(phdata, 'start_time')
        ph.end_time = import_datetime_attribute(phdata, 'end_time')
        ph.phase_type = PT_IDX[phdata['type']]
        ph.circling_direction = CD_IDX[phdata['circling_direction']]
        ph.alt_diff = phdata['alt_diff']
        ph.duration = datetime.timedelta(seconds=phdata['duration'])
        ph.distance = phdata['distance']
        ph.speed = phdata['speed']
        ph.vario = phdata['vario']
        ph.glide_rate = phdata['glide_rate']
        ph.count = 1

        db.session.add(ph)

    for statname in ["total", "left", "right", "mixed"]:
        phdata = root['performance']["circling_%s" % statname]
        ph = FlightPhase(flight=flight)
        ph.aggregate = True
        ph.phase_type = FlightPhase.PT_CIRCLING
        ph.fraction = round(phdata['fraction'] * 100)
        ph.circling_direction = CD_IDX[statname]
        ph.alt_diff = phdata['alt_diff']
        ph.duration = datetime.timedelta(seconds=phdata['duration'])
        ph.vario = phdata['vario']
        ph.count = phdata['count']

        db.session.add(ph)

    phdata = root['performance']['cruise_total']
    ph = FlightPhase(flight=flight)
    ph.aggregate = True
    ph.phase_type = FlightPhase.PT_CRUISE
    ph.circling_direction = FlightPhase.CD_TOTAL
    ph.alt_diff = phdata['alt_diff']
    ph.duration = datetime.timedelta(seconds=phdata['duration'])
    ph.fraction = round(phdata['fraction'] * 100)
    ph.distance = phdata['distance']
    ph.speed = phdata['speed']
    ph.vario = phdata['vario']
    ph.glide_rate = phdata['glide_rate']
    ph.count = phdata['count']

    db.session.add(ph)

def get_limits():
    iter_limit = int(current_app.config.get('SKYLINES_ANALYSIS_ITER', 10e6))
    # Each node of the triangle solver has a size of 92 bytes...
    tree_size_limit = int(current_app.config.get('SKYLINES_ANALYSIS_MEMORY', 256)) \
        * 1024 * 1024 / 92

    return dict(iter_limit=iter_limit, tree_size_limit=tree_size_limit)

def get_analysis_times(times):
    chosen_period_seconds = 0
    chosen_period = None

    has_release = [key for key, value in enumerate(times) if 'release' in value]

    if has_release:
        for i in has_release:
            scoring_periods = []

            # gather all possible scoring periods of this flight
            if times[i]['power_states'] and \
                [not state['powered'] for state in times[i]['power_states']
                 if not state['powered']]:
                scoring_start = min(next(state for state in times[i]['power_states']
                                         if not state['powered']),
                                    times[i]['release'],
                                    key=lambda x: x['time'])
            else:
                scoring_start = times[i]['release']

            if times[i]['power_states']:
                for state in times[i]['power_states']:
                    if state['powered'] and state['time'] > scoring_start['time']:
                        scoring_periods.append(
                            dict(takeoff=times[i]['takeoff'],
                                 scoring_start=scoring_start,
                                 scoring_end=min(state,
                                                 times[i]['landing'].copy(),
                                                 key=lambda x: x['time']),
                                 landing=times[i]['landing']))

                    elif not state['powered'] and state['time'] > scoring_start['time']:
                        scoring_start = state

            if scoring_start['time'] < times[i]['landing']['time']:
                scoring_periods.append(dict(takeoff=times[i]['takeoff'],
                                            scoring_start=scoring_start,
                                            scoring_end=times[i]['landing'].copy(),
                                            landing=times[i]['landing']))

            for period in scoring_periods:
                total_seconds = (period['scoring_end']['time']
                                 - period['scoring_start']['time']).total_seconds()

                if total_seconds > chosen_period_seconds:
                    chosen_period_seconds = total_seconds
                    chosen_period = period

    else:
        for i in range(len(times)):
            total_seconds = (times[i]['landing']['time']
                             - times[i]['takeoff']['time']).total_seconds()

            if total_seconds > chosen_period_seconds:
                chosen_period_seconds = total_seconds
                chosen_period = dict(takeoff=times[i]['takeoff'],
                                     scoring_start=None,
                                     scoring_end=None,
                                     landing=times[i]['landing'])

    return chosen_period

def run_analyse_flight(flight,
                       full=None, triangle=None, sprint=None,
                       fp=None):
    limits = get_limits()

    if not fp:
        filename = files.filename_to_path(flight.igc_file.filename)
        fp = flight_path(filename, add_elevation=True, max_points=None)

    if len(fp) < 2:
        return None

    xcsoar_flight = xcsoar.Flight(fp)

    analysis_times = get_analysis_times(xcsoar_flight.times())

    # Fallback if automated flight detection has failed - check if first and last
    # fix could be ok and use both of them for takeoff and landing
    if not analysis_times and fp[0].datetime < fp[-1].datetime:
        analysis_times = dict(takeoff=dict(time=fp[0].datetime,
                                           location=fp[0].location),
                              scoring_start=None,
                              scoring_end=None,
                              landing=dict(time=fp[-1].datetime,
                                           location=fp[-1].location))

    # Give priority to already stored takeoff and landing times
    if flight.takeoff_time:
        analysis_times['takeoff']['time'] = flight.takeoff_time
        analysis_times['takeoff']['location'] = dict(latitude=flight.takeoff_location.latitude,
                                                     longitude=flight.takeoff_location.longitude)

    if flight.landing_time:
        analysis_times['landing']['time'] = flight.landing_time
        analysis_times['landing']['location'] = dict(latitude=flight.landing_location.latitude,
                                                     longitude=flight.landing_location.longitude)

    # If no scoring window was found fallback to takeoff - landing
    if not analysis_times['scoring_start']:
        analysis_times['scoring_start'] = analysis_times['takeoff'].copy()

    if not analysis_times['scoring_end']:
        analysis_times['scoring_end'] = analysis_times['landing'].copy()

    # And give priority to already stored scoring times
    if flight.scoring_start_time:
        analysis_times['scoring_start']['time'] = flight.scoring_start_time

    if flight.scoring_end_time:
        analysis_times['scoring_end']['time'] = flight.scoring_end_time

    if analysis_times:
        analysis = xcsoar_flight.analyse(analysis_times['takeoff']['time'],
                                         analysis_times['scoring_start']['time'],
                                         analysis_times['scoring_end']['time'],
                                         analysis_times['landing']['time'],
                                         full=full,
                                         triangle=triangle,
                                         sprint=sprint,
                                         max_iterations=limits['iter_limit'],
                                         max_tree_size=limits['tree_size_limit'])
        analysis['events'] = analysis_times

        return analysis

    else:
        return None

def analyse_flight(flight, full=512, triangle=1024, sprint=64, fp=None):
    path = files.filename_to_path(flight.igc_file.filename)
    current_app.logger.info('Analyzing ' + path)

    root = run_analyse_flight(
        flight, full=full, triangle=triangle, sprint=sprint,
        fp=fp)

    if root is None:
        current_app.logger.warning('Analyze flight failed.')
        return False

    if 'events' in root:
        save_events(root['events'], flight)

    if flight.takeoff_time is None \
       or flight.landing_time is None:
        return False

    contest = find_contest(root, 'olc_plus')
    if contest is not None:
        trace = find_trace(contest, 'classic')
        if trace is not None and 'distance' in trace:
            flight.olc_classic_distance = int(trace['distance'])
        else:
            flight.olc_classic_distance = None

        trace = find_trace(contest, 'triangle')
        if trace is not None and 'distance' in trace:
            flight.olc_triangle_distance = int(trace['distance'])
        else:
            flight.olc_triangle_distance = None

        trace = find_trace(contest, 'plus')
        if trace is not None and 'score' in trace:
            flight.olc_plus_score = trace['score']
        else:
            flight.olc_plus_score = None

    save_contests(root, flight)
    save_phases(root, flight)

    flight.needs_analysis = False
    return True
<commit_msg>lib.xcsoar.analysis: store QNH from xcsoar analysis<commit_after>import datetime

import xcsoar
from flask import current_app
from skylines.model import db
from skylines.lib import files
from skylines.lib.datetime import from_seconds_of_day
from skylines.lib.xcsoar_.flightpath import flight_path
from skylines.model import (
    Airport, Trace, FlightPhase, TimeZone, Location
)

def read_location(node):
    """
    Reads the latitude and longitude attributes of the node
    and returns a Location instance if the node was parsed correctly.
    """

    if node is None:
        return None

    if 'latitude' not in node or 'longitude' not in node:
        return None

    try:
        latitude = float(node['latitude'])
        longitude = float(node['longitude'])
        return Location(latitude=latitude, longitude=longitude)
    except ValueError:
        return None

def import_location_attribute(node, name):
    """
    Reads a Location instance from an attribute of the node
    with the given name.
    """

    if node is None or name not in node:
        return None

    location = node[name]
    return read_location(location)

def import_datetime_attribute(node, name):
    if node is None or name not in node:
        return None
    if isinstance(node[name], datetime.datetime):
        return node[name]
    else:
        return None

def find_contest(root, name):
    if 'contests' not in root:
        return None

    if name not in root['contests']:
        return None

    return root['contests'][name]

def find_trace(contest, name):
    if name not in contest:
        return None

    return contest[name]

def read_time_of_day(turnpoint, flight):
    if 'time' not in turnpoint:
        return None

    return from_seconds_of_day(flight.takeoff_time.date(),
                               int(turnpoint['time']))

def delete_trace(contest_name, trace_name, flight):
    q = Trace.query(
        flight=flight, contest_type=contest_name, trace_type=trace_name)
    q.delete()

def save_trace(contest_name, trace_name, node, flight):
    delete_trace(contest_name, trace_name, flight)

    if 'turnpoints' not in node:
        return

    locations = []
    times = []
    for turnpoint in node['turnpoints']:
        location = read_location(turnpoint['location'])
        time = read_time_of_day(turnpoint, flight)

        if location is None or time is None:
            continue

        locations.append(location)
        times.append(time)

    if len(locations) < 2 or len(times) < 2:
        return

    trace = Trace()
    trace.contest_type = contest_name
    trace.trace_type = trace_name
    trace.locations = locations
    trace.times = times

    if 'duration' in node:
        trace.duration = datetime.timedelta(seconds=int(node['duration']))

    if 'distance' in node:
        trace.distance = int(node['distance'])

    flight.traces.append(trace)

def save_contest(contest_name, traces, flight):
    for trace_name, trace in traces.iteritems():
        save_trace(contest_name, trace_name, trace, flight)

def save_contests(root, flight):
    if 'contests' not in root or flight.takeoff_time is None:
        # The takeoff_time is needed to convert the
        # time integer to a DateTime instance
        return

    for contest_name, traces in root['contests'].iteritems():
        save_contest(contest_name, traces, flight)

def get_takeoff_date(flight):
    if flight.takeoff_location is None:
        return flight.takeoff_time

    timezone = TimeZone.by_location(flight.takeoff_location)
    if timezone is None:
        return flight.takeoff_time

    return timezone.fromutc(flight.takeoff_time).date()

def save_takeoff(event, flight):
    flight.takeoff_time = import_datetime_attribute(event, 'time')
    flight.takeoff_location = read_location(event['location'])
    if flight.takeoff_location is not None:
        flight.takeoff_airport = Airport.by_location(flight.takeoff_location,
                                                     date=flight.takeoff_time)

    flight.date_local = get_takeoff_date(flight)

def save_landing(event, flight):
    flight.landing_time = import_datetime_attribute(event, 'time')
    flight.landing_location = read_location(event['location'])
    if flight.landing_location is not None:
        flight.landing_airport = Airport.by_location(flight.landing_location,
                                                     date=flight.landing_time)

def save_events(events, flight):
    if 'takeoff' in events:
        save_takeoff(events['takeoff'], flight)
    if 'scoring_start' in events:
        flight.scoring_start_time = import_datetime_attribute(events['scoring_start'], 'time')
    if 'scoring_end' in events:
        flight.scoring_end_time = import_datetime_attribute(events['scoring_end'], 'time')
    if 'landing' in events:
        save_landing(events['landing'], flight)

def save_phases(root, flight):
    flight.delete_phases()

    if 'phases' not in root or 'performance' not in root:
        return

    PT_IDX = {'': None,
              'powered': FlightPhase.PT_POWERED,
              'cruise': FlightPhase.PT_CRUISE,
              'circling': FlightPhase.PT_CIRCLING}

    CD_IDX = {'': None,
              'left': FlightPhase.CD_LEFT,
              'mixed': FlightPhase.CD_MIXED,
              'right': FlightPhase.CD_RIGHT,
              'total': FlightPhase.CD_TOTAL}

    for phdata in root['phases']:
        ph = FlightPhase()
        ph.flight = flight
        ph.aggregate = False
        ph.start_time = import_datetime_attribute(phdata, 'start_time')
        ph.end_time = import_datetime_attribute(phdata, 'end_time')
        ph.phase_type = PT_IDX[phdata['type']]
        ph.circling_direction = CD_IDX[phdata['circling_direction']]
        ph.alt_diff = phdata['alt_diff']
        ph.duration = datetime.timedelta(seconds=phdata['duration'])
        ph.distance = phdata['distance']
        ph.speed = phdata['speed']
        ph.vario = phdata['vario']
        ph.glide_rate = phdata['glide_rate']
        ph.count = 1

        db.session.add(ph)

    for statname in ["total", "left", "right", "mixed"]:
        phdata = root['performance']["circling_%s" % statname]
        ph = FlightPhase(flight=flight)
        ph.aggregate = True
        ph.phase_type = FlightPhase.PT_CIRCLING
        ph.fraction = round(phdata['fraction'] * 100)
        ph.circling_direction = CD_IDX[statname]
        ph.alt_diff = phdata['alt_diff']
        ph.duration = datetime.timedelta(seconds=phdata['duration'])
        ph.vario = phdata['vario']
        ph.count = phdata['count']

        db.session.add(ph)

    phdata = root['performance']['cruise_total']
    ph = FlightPhase(flight=flight)
    ph.aggregate = True
    ph.phase_type = FlightPhase.PT_CRUISE
    ph.circling_direction = FlightPhase.CD_TOTAL
    ph.alt_diff = phdata['alt_diff']
    ph.duration = datetime.timedelta(seconds=phdata['duration'])
    ph.fraction = round(phdata['fraction'] * 100)
    ph.distance = phdata['distance']
    ph.speed = phdata['speed']
    ph.vario = phdata['vario']
    ph.glide_rate = phdata['glide_rate']
    ph.count = phdata['count']

    db.session.add(ph)

def get_limits():
    iter_limit = int(current_app.config.get('SKYLINES_ANALYSIS_ITER', 10e6))
    # Each node of the triangle solver has a size of 92 bytes...
    tree_size_limit = int(current_app.config.get('SKYLINES_ANALYSIS_MEMORY', 256)) \
        * 1024 * 1024 / 92

    return dict(iter_limit=iter_limit, tree_size_limit=tree_size_limit)

def get_analysis_times(times):
    chosen_period_seconds = 0
    chosen_period = None

    has_release = [key for key, value in enumerate(times) if 'release' in value]

    if has_release:
        for i in has_release:
            scoring_periods = []

            # gather all possible scoring periods of this flight
            if times[i]['power_states'] and \
                [not state['powered'] for state in times[i]['power_states']
                 if not state['powered']]:
                scoring_start = min(next(state for state in times[i]['power_states']
                                         if not state['powered']),
                                    times[i]['release'],
                                    key=lambda x: x['time'])
            else:
                scoring_start = times[i]['release']

            if times[i]['power_states']:
                for state in times[i]['power_states']:
                    if state['powered'] and state['time'] > scoring_start['time']:
                        scoring_periods.append(
                            dict(takeoff=times[i]['takeoff'],
                                 scoring_start=scoring_start,
                                 scoring_end=min(state,
                                                 times[i]['landing'].copy(),
                                                 key=lambda x: x['time']),
                                 landing=times[i]['landing']))

                    elif not state['powered'] and state['time'] > scoring_start['time']:
                        scoring_start = state

            if scoring_start['time'] < times[i]['landing']['time']:
                scoring_periods.append(dict(takeoff=times[i]['takeoff'],
                                            scoring_start=scoring_start,
                                            scoring_end=times[i]['landing'].copy(),
                                            landing=times[i]['landing']))

            for period in scoring_periods:
                total_seconds = (period['scoring_end']['time']
                                 - period['scoring_start']['time']).total_seconds()

                if total_seconds > chosen_period_seconds:
                    chosen_period_seconds = total_seconds
                    chosen_period = period

    else:
        for i in range(len(times)):
            total_seconds = (times[i]['landing']['time']
                             - times[i]['takeoff']['time']).total_seconds()

            if total_seconds > chosen_period_seconds:
                chosen_period_seconds = total_seconds
                chosen_period = dict(takeoff=times[i]['takeoff'],
                                     scoring_start=None,
                                     scoring_end=None,
                                     landing=times[i]['landing'])

    return chosen_period

def run_analyse_flight(flight,
                       full=None, triangle=None, sprint=None,
                       fp=None):
    limits = get_limits()

    if not fp:
        filename = files.filename_to_path(flight.igc_file.filename)
        fp = flight_path(filename, add_elevation=True, max_points=None)

    if len(fp) < 2:
        return None

    xcsoar_flight = xcsoar.Flight(fp)

    analysis_times = get_analysis_times(xcsoar_flight.times())

    # Fallback if automated flight detection has failed - check if first and last
    # fix could be ok and use both of them for takeoff and landing
    if not analysis_times and fp[0].datetime < fp[-1].datetime:
        analysis_times = dict(takeoff=dict(time=fp[0].datetime,
                                           location=fp[0].location),
                              scoring_start=None,
                              scoring_end=None,
                              landing=dict(time=fp[-1].datetime,
                                           location=fp[-1].location))

    # Give priority to already stored takeoff and landing times
    if flight.takeoff_time:
        analysis_times['takeoff']['time'] = flight.takeoff_time
        analysis_times['takeoff']['location'] = dict(latitude=flight.takeoff_location.latitude,
                                                     longitude=flight.takeoff_location.longitude)

    if flight.landing_time:
        analysis_times['landing']['time'] = flight.landing_time
        analysis_times['landing']['location'] = dict(latitude=flight.landing_location.latitude,
                                                     longitude=flight.landing_location.longitude)

    # If no scoring window was found fallback to takeoff - landing
    if not analysis_times['scoring_start']:
        analysis_times['scoring_start'] = analysis_times['takeoff'].copy()

    if not analysis_times['scoring_end']:
        analysis_times['scoring_end'] = analysis_times['landing'].copy()

    # And give priority to already stored scoring times
    if flight.scoring_start_time:
        analysis_times['scoring_start']['time'] = flight.scoring_start_time

    if flight.scoring_end_time:
        analysis_times['scoring_end']['time'] = flight.scoring_end_time

    if analysis_times:
        analysis = xcsoar_flight.analyse(analysis_times['takeoff']['time'] - datetime.timedelta(seconds=300),
                                         analysis_times['scoring_start']['time'],
                                         analysis_times['scoring_end']['time'],
                                         analysis_times['landing']['time'] + datetime.timedelta(seconds=300),
                                         full=full,
                                         triangle=triangle,
                                         sprint=sprint,
                                         max_iterations=limits['iter_limit'],
                                         max_tree_size=limits['tree_size_limit'])
        analysis['events'] = analysis_times

        return analysis

    else:
        return None

def analyse_flight(flight, full=512, triangle=1024, sprint=64, fp=None):
    path = files.filename_to_path(flight.igc_file.filename)
    current_app.logger.info('Analyzing ' + path)

    root = run_analyse_flight(
        flight, full=full, triangle=triangle, sprint=sprint,
        fp=fp)

    if root is None:
        current_app.logger.warning('Analyze flight failed.')
        return False

    if 'qnh' in root and root['qnh'] is not None:
        flight.qnh = root['qnh']

    if 'events' in root:
        save_events(root['events'], flight)

    if flight.takeoff_time is None \
       or flight.landing_time is None:
        return False

    contest = find_contest(root, 'olc_plus')
    if contest is not None:
        trace = find_trace(contest, 'classic')
        if trace is not None and 'distance' in trace:
            flight.olc_classic_distance = int(trace['distance'])
        else:
            flight.olc_classic_distance = None

        trace = find_trace(contest, 'triangle')
        if trace is not None and 'distance' in trace:
            flight.olc_triangle_distance = int(trace['distance'])
        else:
            flight.olc_triangle_distance = None

        trace = find_trace(contest, 'plus')
        if trace is not None and 'score' in trace:
            flight.olc_plus_score = trace['score']
        else:
            flight.olc_plus_score = None

    save_contests(root, flight)
    save_phases(root, flight)

    flight.needs_analysis = False
    return True
<|endoftext|>


<commit_before>import json
import os
from collections import OrderedDict

import geopandas
import pandas
import yaml
from flask import current_app
from flask_restplus import Namespace, Resource, abort

import cea.inputlocator
import cea.utilities.dbf
from cea.utilities.schedule_reader import read_cea_schedule, save_cea_schedule
from cea.plots.supply_system.supply_system_map import get_building_connectivity
from cea.plots.variable_naming import get_color_array
from cea.technologies.network_layout.main import layout_network, NetworkLayout
from cea.utilities.standardize_coordinates import get_geographic_coordinate_system

api = Namespace('Inputs', description='Input data for CEA')

COLORS = {
    'surroundings': get_color_array('white'),
    'dh': get_color_array('red'),
    'dc': get_color_array('blue'),
    'disconnected': get_color_array('grey')
}

def read_inputs_field_types():
    """Parse the inputs.yaml file and create the dictionary of column types"""
    inputs = yaml.load(
        open(os.path.join(os.path.dirname(__file__), '../inputs/inputs.yml')).read())

    for db in inputs.keys():
        inputs[db]['fieldnames'] = [field['name']
                                    for field in inputs[db]['fields']]
    return inputs

INPUTS = read_inputs_field_types()
INPUT_KEYS = INPUTS.keys()
GEOJSON_KEYS = ['zone', 'surroundings', 'streets', 'dc', 'dh']
NETWORK_KEYS = ['dc', 'dh']

# INPUT_MODEL = api.model('Input', {
#     'fields': fields.List(fields.String, description='Column names')
# })

# GEOJSON_MODEL = api.model('GeoJSON',{
#     'test': fields.String()
# })

# BUILDING_PROPS_MODEL = api.model('Building Properties', {
#     'geojsons': fields.List(fields.Nested(GEOJSON_MODEL)),
#     'tables': fields.List(fields.String)
# })

@api.route('/')
class InputList(Resource):
    def get(self):
        return {'buildingProperties': INPUT_KEYS, 'geoJSONs': GEOJSON_KEYS}

@api.route('/building-properties/<string:db>')
class InputBuildingProperties(Resource):
    def get(self, db):
        if db not in INPUTS:
            abort(400, 'Input file not found: %s' % db, choices=INPUT_KEYS)
        db_info = INPUTS[db]
        columns = OrderedDict()
        for field in db_info['fields']:
            columns[field['name']] = field['type']
        return columns

@api.route('/geojson/<string:kind>')
class InputGeojson(Resource):
    def get(self, kind):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        if kind not in GEOJSON_KEYS:
            abort(400, 'Input file not found: %s' % kind, choices=GEOJSON_KEYS)
        # Building geojsons
        elif kind in INPUT_KEYS and kind in GEOJSON_KEYS:
            db_info = INPUTS[kind]
            config = current_app.cea_config
            locator = cea.inputlocator.InputLocator(config.scenario)
            location = getattr(locator, db_info['location'])()
            if db_info['type'] != 'shp':
                abort(400, 'Invalid database for geojson: %s' % location)
            return df_to_json(location, bbox=True)[0]
        elif kind in NETWORK_KEYS:
            return get_network(config, kind)[0]
        elif kind == 'streets':
            return df_to_json(locator.get_street_network())[0]

@api.route('/building-properties')
class BuildingProperties(Resource):
    def get(self):
        return get_building_properties()

@api.route('/all-inputs')
class AllInputs(Resource):
    def get(self):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        # FIXME: Find a better way, current used to test for Input Editor
        store = get_building_properties()
        store['geojsons'] = {}
        store['connected_buildings'] = {}
        store['crs'] = {}
        store['geojsons']['zone'], store['crs']['zone'] = df_to_json(
            locator.get_zone_geometry(), bbox=True, trigger_abort=False)
        store['geojsons']['surroundings'], store['crs']['surroundings'] = df_to_json(
            locator.get_surroundings_geometry(), bbox=True, trigger_abort=False)
        store['geojsons']['streets'], store['crs']['streets'] = df_to_json(
            locator.get_street_network(), trigger_abort=False)
        store['geojsons']['dc'], store['connected_buildings']['dc'], store['crs']['dc'] = get_network(
            config, 'dc', trigger_abort=False)
        store['geojsons']['dh'], store['connected_buildings']['dh'],  store['crs']['dh'] = get_network(
            config, 'dh', trigger_abort=False)
        store['colors'] = COLORS
        store['schedules'] = {}

        return store
    def put(self):
        form = api.payload
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        tables = form['tables']
        geojsons = form['geojsons']
        crs = form['crs']
        schedules = form['schedules']

        out = {'tables': {}, 'geojsons': {}}

        # TODO: Maybe save the files to temp location in case something fails
        for db in INPUTS:
            db_info = INPUTS[db]
            location = getattr(locator, db_info['location'])()

            if len(tables[db]) != 0:
                if db_info['type'] == 'shp':
                    from cea.utilities.standardize_coordinates import get_geographic_coordinate_system
                    table_df = geopandas.GeoDataFrame.from_features(geojsons[db]['features'],
                                                                    crs=get_geographic_coordinate_system())
                    out['geojsons'][db] = json.loads(table_df.to_json(show_bbox=True))
                    table_df = table_df.to_crs(crs[db])
                    table_df.to_file(location, driver='ESRI Shapefile', encoding='ISO-8859-1')

                    table_df = pandas.DataFrame(table_df.drop(columns='geometry'))
                    out['tables'][db] = json.loads(table_df.set_index('Name').to_json(orient='index'))
                elif db_info['type'] == 'dbf':
                    table_df = pandas.read_json(json.dumps(tables[db]), orient='index')

                    # Make sure index name is 'Name;
                    table_df.index.name = 'Name'
                    table_df = table_df.reset_index()

                    cea.utilities.dbf.dataframe_to_dbf(table_df, location)
                    out['tables'][db] = json.loads(table_df.set_index('Name').to_json(orient='index'))

            else:  # delete file if empty
                out['tables'][db] = {}
                if os.path.isfile(location):
                    if db_info['type'] == 'shp':
                        import glob
                        for filepath in glob.glob(os.path.join(locator.get_building_geometry_folder(), '%s.*' % db)):
                            os.remove(filepath)
                    elif db_info['type'] == 'dbf':
                        os.remove(location)
                if db_info['type'] == 'shp':
                    out['geojsons'][db] = {}

        if schedules:
            for building in schedules:
                dict_to_schedule(locator, building, schedules[building])

        return out

def get_building_properties():
    import cea.glossary
    # FIXME: Find a better way to ensure order of tabs
    tabs = ['zone', 'age', 'occupancy', 'architecture', 'internal-loads', 'indoor-comfort', 'air-conditioning-systems',
            'supply-systems', 'surroundings']

    config = current_app.cea_config
    locator = cea.inputlocator.InputLocator(config.scenario)
    store = {'tables': {}, 'columns': {}, 'order': tabs}
    glossary = cea.glossary.read_glossary_df()
    filenames = glossary['FILE_NAME'].str.split(pat='/').str[-1]
    for db in INPUTS:
        db_info = INPUTS[db]
        location = getattr(locator, db_info['location'])()
        try:
            if db_info['type'] == 'shp':
                table_df = geopandas.GeoDataFrame.from_file(location)
                table_df = pandas.DataFrame(
                    table_df.drop(columns='geometry'))
                if 'REFERENCE' in db_info['fieldnames'] and 'REFERENCE' not in table_df.columns:
                    table_df['REFERENCE'] = None
                store['tables'][db] = json.loads(
                    table_df.set_index('Name').to_json(orient='index'))
            else:
                assert db_info['type'] == 'dbf', 'Unexpected database type: %s' % db_info['type']
                table_df = cea.utilities.dbf.dbf_to_dataframe(location)
                if 'REFERENCE' in db_info['fieldnames'] and 'REFERENCE' not in table_df.columns:
                    table_df['REFERENCE'] = None
                store['tables'][db] = json.loads(
                    table_df.set_index('Name').to_json(orient='index'))

            columns = OrderedDict()
            db_glossary = json.loads(glossary[filenames == '%s.%s' % (db.replace('-', '_'), db_info['type'])]
                                     [['VARIABLE', 'UNIT', 'DESCRIPTION']].set_index('VARIABLE').to_json(orient='index'))

            for field in db_info['fields']:
                column = field['name']
                columns[column] = {}
                if column == 'REFERENCE':
                    continue
                columns[column]['type'] = field['type']
                if field['type'] == 'choice':
                    path = getattr(locator, field['location']['path'])()
                    columns[column]['path'] = path
                    # TODO: Try to optimize this step to decrease the number of file reading
                    columns[column]['choices'] = get_choices(field['location'], path)
                if 'constraints' in field:
                    columns[column]['constraints'] = field['constraints']
                columns[column]['description'] = db_glossary[column]['DESCRIPTION']
                columns[column]['unit'] = db_glossary[column]['UNIT']
            store['columns'][db] = columns

        except IOError as e:
            print(e)
            store['tables'][db] = {}
            store['columns'][db] = {}

    return store

def get_network(config, network_type, trigger_abort=True):
    # TODO: Get a list of names and send all in the json
    try:
        locator = cea.inputlocator.InputLocator(config.scenario)
        building_connectivity = get_building_connectivity(locator)
        network_type = network_type.upper()
        connected_buildings = building_connectivity[building_connectivity['{}_connectivity'.format(
            network_type)] == 1]['Name'].values.tolist()
        network_name = 'base'

        # Do not calculate if no connected buildings
        if len(connected_buildings) < 2:
            return None, [], None

        edges = locator.get_network_layout_edges_shapefile(
            network_type, network_name)
        nodes = locator.get_network_layout_nodes_shapefile(
            network_type, network_name)
        supply_system = locator.get_building_supply()

        no_network_file = not os.path.isfile(edges) or not os.path.isfile(nodes)
        supply_system_modified = os.path.getmtime(supply_system)

        # Generate network files
        if no_network_file or supply_system_modified > os.path.getmtime(edges) or supply_system_modified > os.path.getmtime(nodes):
            config.network_layout.network_type = network_type
            config.network_layout.connected_buildings = connected_buildings
            network_layout = NetworkLayout(network_layout=config.network_layout)
            layout_network(network_layout, locator, output_name_network=network_name)

        network_json, crs = df_to_json(edges, trigger_abort=trigger_abort)
        nodes_json, _ = df_to_json(nodes, trigger_abort=trigger_abort)
        network_json['features'].extend(nodes_json['features'])
        network_json['properties'] = {'connected_buildings': connected_buildings}
        return network_json, connected_buildings, crs
    except IOError as e:
        print(e)
        return None, [], None

def df_to_json(file_location, bbox=False, trigger_abort=True):
    from cea.utilities.standardize_coordinates import get_lat_lon_projected_shapefile, get_projected_coordinate_system
    try:
        table_df = geopandas.GeoDataFrame.from_file(file_location)
        # Save coordinate system
        lat, lon = get_lat_lon_projected_shapefile(table_df)
        crs = get_projected_coordinate_system(lat, lon)
        # make sure that the geojson is coded in latitude / longitude
        out = table_df.to_crs(get_geographic_coordinate_system())
        out = json.loads(out.to_json(show_bbox=bbox))
        return out, crs
    except IOError as e:
        print(e)
        if trigger_abort:
            abort(400, 'Input file not found: %s' % file_location)
        return None, None
    except RuntimeError as e:
        print(e)
        if trigger_abort:
            abort(400, e.message)

@api.route('/building-schedule/<string:building>')
class BuildingSchedule(Resource):
    def get(self, building):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)
        try:
            return schedule_to_dict(locator, building)
        except IOError as e:
            print(e)
            abort(500, 'File not found')

def get_choices(location, path):
    df = pandas.read_excel(path, location['sheet'])
    if 'filter' in location:
        choices = df[df.eval(location['filter'])][location['column']].tolist()
    else:
        choices = df[location['column']].tolist()
    return [{'value': choice, 'label': df.loc[df[location['column']] == choice, 'Description'].values[0]} for choice in choices]

def schedule_to_dict(locator, building):
    schedule_path = locator.get_building_weekly_schedules(building)
    schedule_data, schedule_complementary_data = read_cea_schedule(schedule_path)
    df = pandas.DataFrame(schedule_data).set_index(['DAY', 'HOUR'])
    out = {'SCHEDULES': {schedule_type: {day: df.loc[day][schedule_type].values.tolist() for day in df.index.levels[0]}
                         for schedule_type in df.columns}}
    out.update(schedule_complementary_data)
    return out

def dict_to_schedule(locator, building, schedule_dict):
    schedule_path = locator.get_building_weekly_schedules(building)
    schedule_data = schedule_dict['SCHEDULES']
    schedule_complementary_data = {'MONTHLY_MULTIPLIER': schedule_dict['MONTHLY_MULTIPLIER'],
                                   'METADATA': schedule_dict['METADATA']}

    data = pandas.DataFrame()
    for day in ['WEEKDAY', 'SATURDAY', 'SUNDAY']:
        df = pandas.DataFrame({'HOUR': range(1, 25), 'DAY': [day] * 24})
        for schedule_type, schedule in schedule_data.items():
            df[schedule_type] = schedule[day]
        data = data.append(df, ignore_index=True)
    save_cea_schedule(data.to_dict('list'), schedule_complementary_data, schedule_path)<commit_msg>Add endpoint to read input databases<commit_after>import json
import os
from collections import OrderedDict

import geopandas
import pandas
import yaml
from flask import current_app
from flask_restplus import Namespace, Resource, abort
from .databases import SCHEMA_KEY, get_database_dict

import cea.inputlocator
import cea.utilities.dbf
from cea.utilities.schedule_reader import read_cea_schedule, save_cea_schedule
from cea.plots.supply_system.supply_system_map import get_building_connectivity
from cea.plots.variable_naming import get_color_array
from cea.technologies.network_layout.main import layout_network, NetworkLayout
from cea.utilities.standardize_coordinates import get_geographic_coordinate_system

api = Namespace('Inputs', description='Input data for CEA')

COLORS = {
    'surroundings': get_color_array('white'),
    'dh': get_color_array('red'),
    'dc': get_color_array('blue'),
    'disconnected': get_color_array('grey')
}

def read_inputs_field_types():
    """Parse the inputs.yaml file and create the dictionary of column types"""
    inputs = yaml.load(
        open(os.path.join(os.path.dirname(__file__), '../inputs/inputs.yml')).read())

    for db in inputs.keys():
        inputs[db]['fieldnames'] = [field['name']
                                    for field in inputs[db]['fields']]
    return inputs

INPUTS = read_inputs_field_types()
INPUT_KEYS = INPUTS.keys()
GEOJSON_KEYS = ['zone', 'surroundings', 'streets', 'dc', 'dh']
NETWORK_KEYS = ['dc', 'dh']

# INPUT_MODEL = api.model('Input', {
#     'fields': fields.List(fields.String, description='Column names')
# })

# GEOJSON_MODEL = api.model('GeoJSON',{
#     'test': fields.String()
# })

# BUILDING_PROPS_MODEL = api.model('Building Properties', {
#     'geojsons': fields.List(fields.Nested(GEOJSON_MODEL)),
#     'tables': fields.List(fields.String)
# })

@api.route('/')
class InputList(Resource):
    def get(self):
        return {'buildingProperties': INPUT_KEYS, 'geoJSONs': GEOJSON_KEYS}

@api.route('/building-properties/<string:db>')
class InputBuildingProperties(Resource):
    def get(self, db):
        if db not in INPUTS:
            abort(400, 'Input file not found: %s' % db, choices=INPUT_KEYS)
        db_info = INPUTS[db]
        columns = OrderedDict()
        for field in db_info['fields']:
            columns[field['name']] = field['type']
        return columns

@api.route('/geojson/<string:kind>')
class InputGeojson(Resource):
    def get(self, kind):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        if kind not in GEOJSON_KEYS:
            abort(400, 'Input file not found: %s' % kind, choices=GEOJSON_KEYS)
        # Building geojsons
        elif kind in INPUT_KEYS and kind in GEOJSON_KEYS:
            db_info = INPUTS[kind]
            config = current_app.cea_config
            locator = cea.inputlocator.InputLocator(config.scenario)
            location = getattr(locator, db_info['location'])()
            if db_info['type'] != 'shp':
                abort(400, 'Invalid database for geojson: %s' % location)
            return df_to_json(location, bbox=True)[0]
        elif kind in NETWORK_KEYS:
            return get_network(config, kind)[0]
        elif kind == 'streets':
            return df_to_json(locator.get_street_network())[0]

@api.route('/building-properties')
class BuildingProperties(Resource):
    def get(self):
        return get_building_properties()

@api.route('/all-inputs')
class AllInputs(Resource):
    def get(self):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        # FIXME: Find a better way, current used to test for Input Editor
        store = get_building_properties()
        store['geojsons'] = {}
        store['connected_buildings'] = {}
        store['crs'] = {}
        store['geojsons']['zone'], store['crs']['zone'] = df_to_json(
            locator.get_zone_geometry(), bbox=True, trigger_abort=False)
        store['geojsons']['surroundings'], store['crs']['surroundings'] = df_to_json(
            locator.get_surroundings_geometry(), bbox=True, trigger_abort=False)
        store['geojsons']['streets'], store['crs']['streets'] = df_to_json(
            locator.get_street_network(), trigger_abort=False)
        store['geojsons']['dc'], store['connected_buildings']['dc'], store['crs']['dc'] = get_network(
            config, 'dc', trigger_abort=False)
        store['geojsons']['dh'], store['connected_buildings']['dh'],  store['crs']['dh'] = get_network(
            config, 'dh', trigger_abort=False)
        store['colors'] = COLORS
        store['schedules'] = {}

        return store
    def put(self):
        form = api.payload
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)

        tables = form['tables']
        geojsons = form['geojsons']
        crs = form['crs']
        schedules = form['schedules']

        out = {'tables': {}, 'geojsons': {}}

        # TODO: Maybe save the files to temp location in case something fails
        for db in INPUTS:
            db_info = INPUTS[db]
            location = getattr(locator, db_info['location'])()

            if len(tables[db]) != 0:
                if db_info['type'] == 'shp':
                    from cea.utilities.standardize_coordinates import get_geographic_coordinate_system
                    table_df = geopandas.GeoDataFrame.from_features(geojsons[db]['features'],
                                                                    crs=get_geographic_coordinate_system())
                    out['geojsons'][db] = json.loads(table_df.to_json(show_bbox=True))
                    table_df = table_df.to_crs(crs[db])
                    table_df.to_file(location, driver='ESRI Shapefile', encoding='ISO-8859-1')

                    table_df = pandas.DataFrame(table_df.drop(columns='geometry'))
                    out['tables'][db] = json.loads(table_df.set_index('Name').to_json(orient='index'))
                elif db_info['type'] == 'dbf':
                    table_df = pandas.read_json(json.dumps(tables[db]), orient='index')

                    # Make sure index name is 'Name;
                    table_df.index.name = 'Name'
                    table_df = table_df.reset_index()

                    cea.utilities.dbf.dataframe_to_dbf(table_df, location)
                    out['tables'][db] = json.loads(table_df.set_index('Name').to_json(orient='index'))

            else:  # delete file if empty
                out['tables'][db] = {}
                if os.path.isfile(location):
                    if db_info['type'] == 'shp':
                        import glob
                        for filepath in glob.glob(os.path.join(locator.get_building_geometry_folder(), '%s.*' % db)):
                            os.remove(filepath)
                    elif db_info['type'] == 'dbf':
                        os.remove(location)
                if db_info['type'] == 'shp':
                    out['geojsons'][db] = {}

        if schedules:
            for building in schedules:
                dict_to_schedule(locator, building, schedules[building])

        return out

def get_building_properties():
    import cea.glossary
    # FIXME: Find a better way to ensure order of tabs
    tabs = ['zone', 'age', 'occupancy', 'architecture', 'internal-loads', 'indoor-comfort', 'air-conditioning-systems',
            'supply-systems', 'surroundings']

    config = current_app.cea_config
    locator = cea.inputlocator.InputLocator(config.scenario)
    store = {'tables': {}, 'columns': {}, 'order': tabs}
    glossary = cea.glossary.read_glossary_df()
    filenames = glossary['FILE_NAME'].str.split(pat='/').str[-1]
    for db in INPUTS:
        db_info = INPUTS[db]
        location = getattr(locator, db_info['location'])()
        try:
            if db_info['type'] == 'shp':
                table_df = geopandas.GeoDataFrame.from_file(location)
                table_df = pandas.DataFrame(
                    table_df.drop(columns='geometry'))
                if 'REFERENCE' in db_info['fieldnames'] and 'REFERENCE' not in table_df.columns:
                    table_df['REFERENCE'] = None
                store['tables'][db] = json.loads(
                    table_df.set_index('Name').to_json(orient='index'))
            else:
                assert db_info['type'] == 'dbf', 'Unexpected database type: %s' % db_info['type']
                table_df = cea.utilities.dbf.dbf_to_dataframe(location)
                if 'REFERENCE' in db_info['fieldnames'] and 'REFERENCE' not in table_df.columns:
                    table_df['REFERENCE'] = None
                store['tables'][db] = json.loads(
                    table_df.set_index('Name').to_json(orient='index'))

            columns = OrderedDict()
            db_glossary = json.loads(glossary[filenames == '%s.%s' % (db.replace('-', '_'), db_info['type'])]
                                     [['VARIABLE', 'UNIT', 'DESCRIPTION']].set_index('VARIABLE').to_json(orient='index'))

            for field in db_info['fields']:
                column = field['name']
                columns[column] = {}
                if column == 'REFERENCE':
                    continue
                columns[column]['type'] = field['type']
                if field['type'] == 'choice':
                    path = getattr(locator, field['location']['path'])()
                    columns[column]['path'] = path
                    # TODO: Try to optimize this step to decrease the number of file reading
                    columns[column]['choices'] = get_choices(field['location'], path)
                if 'constraints' in field:
                    columns[column]['constraints'] = field['constraints']
                columns[column]['description'] = db_glossary[column]['DESCRIPTION']
                columns[column]['unit'] = db_glossary[column]['UNIT']
            store['columns'][db] = columns

        except IOError as e:
            print(e)
            store['tables'][db] = {}
            store['columns'][db] = {}

    return store

def get_network(config, network_type, trigger_abort=True):
    # TODO: Get a list of names and send all in the json
    try:
        locator = cea.inputlocator.InputLocator(config.scenario)
        building_connectivity = get_building_connectivity(locator)
        network_type = network_type.upper()
        connected_buildings = building_connectivity[building_connectivity['{}_connectivity'.format(
            network_type)] == 1]['Name'].values.tolist()
        network_name = 'base'

        # Do not calculate if no connected buildings
        if len(connected_buildings) < 2:
            return None, [], None

        edges = locator.get_network_layout_edges_shapefile(
            network_type, network_name)
        nodes = locator.get_network_layout_nodes_shapefile(
            network_type, network_name)
        supply_system = locator.get_building_supply()

        no_network_file = not os.path.isfile(edges) or not os.path.isfile(nodes)
        supply_system_modified = os.path.getmtime(supply_system)

        # Generate network files
        if no_network_file or supply_system_modified > os.path.getmtime(edges) or supply_system_modified > os.path.getmtime(nodes):
            config.network_layout.network_type = network_type
            config.network_layout.connected_buildings = connected_buildings
            network_layout = NetworkLayout(network_layout=config.network_layout)
            layout_network(network_layout, locator, output_name_network=network_name)

        network_json, crs = df_to_json(edges, trigger_abort=trigger_abort)
        nodes_json, _ = df_to_json(nodes, trigger_abort=trigger_abort)
        network_json['features'].extend(nodes_json['features'])
        network_json['properties'] = {'connected_buildings': connected_buildings}
        return network_json, connected_buildings, crs
    except IOError as e:
        print(e)
        return None, [], None

def df_to_json(file_location, bbox=False, trigger_abort=True):
    from cea.utilities.standardize_coordinates import get_lat_lon_projected_shapefile, get_projected_coordinate_system
    try:
        table_df = geopandas.GeoDataFrame.from_file(file_location)
        # Save coordinate system
        lat, lon = get_lat_lon_projected_shapefile(table_df)
        crs = get_projected_coordinate_system(lat, lon)
        # make sure that the geojson is coded in latitude / longitude
        out = table_df.to_crs(get_geographic_coordinate_system())
        out = json.loads(out.to_json(show_bbox=bbox))
        return out, crs
    except IOError as e:
        print(e)
        if trigger_abort:
            abort(400, 'Input file not found: %s' % file_location)
        return None, None
    except RuntimeError as e:
        print(e)
        if trigger_abort:
            abort(400, e.message)

@api.route('/building-schedule/<string:building>')
class BuildingSchedule(Resource):
    def get(self, building):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)
        try:
            return schedule_to_dict(locator, building)
        except IOError as e:
            print(e)
            abort(500, 'File not found')

@api.route('/databases/<string:db>')
class InputDatabase(Resource):
    def get(self, db):
        config = current_app.cea_config
        locator = cea.inputlocator.InputLocator(config.scenario)
        # Need to change locator method for schedules since `schema.yml` uses
        # `get_database_standard_schedules_use` instead of `get_database_standard_schedules`
        locator_methods = dict(SCHEMA_KEY)
        locator_methods['schedules'] = 'get_database_standard_schedules'
        db_names = locator_methods.keys()
        try:
            # All or nothing. Abort reading databases if encounter error
            if db == 'all':
                out = {}
                for db_name in db_names:
                    out[db_name] = get_database_dict(locator.__getattribute__(locator_methods[db_name])(), db_name)
                return out
            elif db in db_names:
                return get_database_dict(locator.__getattribute__(locator_methods[db])(), db)
            else:
                abort(400, "Could not find '{}' database. Try instead {}".format(db, ", ".join(db_names)))
        except IOError as e:
            print(e)
            abort(500, e.message)

def get_choices(location, path):
    df = pandas.read_excel(path, location['sheet'])
    if 'filter' in location:
        choices = df[df.eval(location['filter'])][location['column']].tolist()
    else:
        choices = df[location['column']].tolist()
    return [{'value': choice, 'label': df.loc[df[location['column']] == choice, 'Description'].values[0]} for choice in choices]

def schedule_to_dict(locator, building):
    schedule_path = locator.get_building_weekly_schedules(building)
    schedule_data, schedule_complementary_data = read_cea_schedule(schedule_path)
    df = pandas.DataFrame(schedule_data).set_index(['DAY', 'HOUR'])
    out = {'SCHEDULES': {schedule_type: {day: df.loc[day][schedule_type].values.tolist() for day in df.index.levels[0]}
                         for schedule_type in df.columns}}
    out.update(schedule_complementary_data)
    return out

def dict_to_schedule(locator, building, schedule_dict):
    schedule_path = locator.get_building_weekly_schedules(building)
    schedule_data = schedule_dict['SCHEDULES']
    schedule_complementary_data = {'MONTHLY_MULTIPLIER': schedule_dict['MONTHLY_MULTIPLIER'],
                                   'METADATA': schedule_dict['METADATA']}

    data = pandas.DataFrame()
    for day in ['WEEKDAY', 'SATURDAY', 'SUNDAY']:
        df = pandas.DataFrame({'HOUR': range(1, 25), 'DAY': [day] * 24})
        for schedule_type, schedule in schedule_data.items():
            df[schedule_type] = schedule[day]
        data = data.append(df, ignore_index=True)
    save_cea_schedule(data.to_dict('list'), schedule_complementary_data, schedule_path)<|endoftext|>