<commit_before>from __future__ import print_function, division
import collections
from sympy import Matrix, Integer, sympify

class NDimArray(object):
    """

    Examples
    ========

    Create an N-dim array of zeros:

    >>> from sympy.tensor.array import MutableDenseNDimArray
    >>> a = MutableDenseNDimArray.zeros(2, 3, 4)
    >>> a
    [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]

    Create an N-dim array from a list;

    >>> a = MutableDenseNDimArray([[2, 3], [4, 5]])
    >>> a
    [[2, 3], [4, 5]]

    >>> b = MutableDenseNDimArray([[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]])
    >>> b
    [[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]]

    Create an N-dim array from a flat list with dimension shape:

    >>> a = MutableDenseNDimArray(2, 3, [1, 2, 3, 4, 5, 6])
    >>> a
    [[1, 2, 3], [4, 5, 6]]

    Create an N-dim array from a matrix:

    >>> from sympy import Matrix
    >>> a = Matrix([[1,2],[3,4]])
    >>> a
    Matrix([
    [1, 2],
    [3, 4]])
    >>> b = MutableDenseNDimArray(a)
    >>> b
    [[1, 2], [3, 4]]

    Arithmetic operations on N-dim arrays

    >>> a = MutableDenseNDimArray(2, 2, [1, 1, 1, 1])
    >>> b = MutableDenseNDimArray(2, 2, [4, 4, 4, 4])
    >>> c = a + b
    >>> c
    [[5, 5], [5, 5]]
    >>> a - b
    [[-3, -3], [-3, -3]]

    """
    def __new__(cls, *args, **kwargs):
        from sympy.tensor.array import MutableDenseNDimArray
        return MutableDenseNDimArray(*args, **kwargs)

    def _parse_index(self, index):

        if isinstance(index, (int, Integer)):
            if index >= self._loop_size:
                raise ValueError("index out of range")
            return index

        if len(index) != self._rank:
            raise ValueError('Wrong number of array axes')

        real_index = 0
        # check if input index can exist in current indexing
        for i in range(self._rank):
            if index[i] > self.shape[i]:
                raise ValueError('Index ' + str(index) + ' out of border')
            real_index = real_index*self.shape[i] + index[i]

        return real_index

    def _get_tuple_index(self, integer_index):
        index = []
        for i, sh in enumerate(reversed(self.shape)):
            index.append(integer_index % sh)
            integer_index //= sh
        index.reverse()
        return tuple(index)

    def _setter_iterable_check(self, value):
        if isinstance(value, (collections.Iterable, Matrix, NDimArray)):
            raise NotImplementedError

    def __len__(self):
        """Overload common function len(). Returns number of elements in array.

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,3)
        >>> a
        [[0, 0, 0], [0, 0, 0], [0, 0, 0]]
        >>> len(a)
        9

        """
        return self._loop_size

    @property
    def shape(self):
        """
        Returns array shape (dimension).

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,3)
        >>> a.shape
        (3, 3)

        """
        return self._shape

    def rank(self):
        """
        Returns rank of array.

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,4,5,6,3)
        >>> a.rank()
        5

        """
        return self._rank

    def __str__(self):
        """Returns string, allows to use standart functions print() and str().

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(2, 2)
        >>> a
        [[0, 0], [0, 0]]

        """
        def f(sh, shape_left, i, j):
            if len(shape_left) == 1:
                return "["+", ".join([str(self[e]) for e in range(i, j)])+"]"

            sh //= shape_left[0]
            return "[" + ", ".join([f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh) for e in range(shape_left[0])]) + "]" # + "\n"*len(shape_left)

        return f(self._loop_size, self.shape, 0, self._loop_size)

        out_str = ''

        # forming output string
        for i, el in enumerate(self):

            out_str += str(el) + '  '
            chidx = i+1
            for sh in reversed(self.shape):
                if chidx % sh == 0:
                    out_str += '\n'
                    chidx //= sh

        return out_str

    def __repr__(self):
        return self.__str__()

    def tolist(self):
        """
        Conveting MutableDenseNDimArray to one-dim list

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray(2, 2, [1, 2, 3, 4])
        >>> a
        [[1, 2], [3, 4]]
        >>> b = a.tolist()
        >>> b
        [[1, 2], [3, 4]]
        """

        def f(sh, shape_left, i, j):
            if len(shape_left) == 1:
                return [self[e] for e in range(i, j)]
            result = []
            sh //= shape_left[0]
            for e in range(shape_left[0]):
                result.append(f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh))
            return result

        return f(self._loop_size, self.shape, 0, self._loop_size)

    def __add__(self, other):
        if not isinstance(other, NDimArray):
            raise TypeError(str(other))

        if self.shape != other.shape:
            raise ValueError("array shape mismatch")
        result_list = [i+j for i,j in zip(self, other)]

        return type(self)(*(self.shape + (result_list,)))

    def __sub__(self, other):
        if not isinstance(other, NDimArray):
            raise TypeError(str(other))

        if self.shape != other.shape:
            raise ValueError("array shape mismatch")
        result_list = [i-j for i,j in zip(self, other)]

        return type(self)(*(self.shape + (result_list,)))

    def __mul__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [i*other for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __rmul__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [other*i for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __div__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [i/other for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __rdiv__(self, other):
        raise TypeError('unsupported operation on NDimArray')

    def __eq__(self, other):
        """
        NDimArray instances can be compared to each other.
        Instances equal if they have same shape and data.

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(2, 3)
        >>> b = MutableDenseNDimArray.zeros(2, 3)
        >>> a == b
        True
        >>> c = a.reshape(3, 2)
        >>> c == b
        False
        >>> a[0,0] = 1
        >>> b[0,0] = 2
        >>> a == b
        False
        """
        if not isinstance(other, NDimArray):
            return False
        return (self.shape == other.shape) and (list(self) == list(other))

    def __ne__(self, other):
        return not self.__eq__(other)

    __truediv__ = __div__
    __rtruediv__ = __rdiv__
<commit_msg>typo fix<commit_after>from __future__ import print_function, division
import collections
from sympy import Matrix, Integer, sympify

class NDimArray(object):
    """

    Examples
    ========

    Create an N-dim array of zeros:

    >>> from sympy.tensor.array import MutableDenseNDimArray
    >>> a = MutableDenseNDimArray.zeros(2, 3, 4)
    >>> a
    [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]

    Create an N-dim array from a list;

    >>> a = MutableDenseNDimArray([[2, 3], [4, 5]])
    >>> a
    [[2, 3], [4, 5]]

    >>> b = MutableDenseNDimArray([[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]])
    >>> b
    [[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]]

    Create an N-dim array from a flat list with dimension shape:

    >>> a = MutableDenseNDimArray(2, 3, [1, 2, 3, 4, 5, 6])
    >>> a
    [[1, 2, 3], [4, 5, 6]]

    Create an N-dim array from a matrix:

    >>> from sympy import Matrix
    >>> a = Matrix([[1,2],[3,4]])
    >>> a
    Matrix([
    [1, 2],
    [3, 4]])
    >>> b = MutableDenseNDimArray(a)
    >>> b
    [[1, 2], [3, 4]]

    Arithmetic operations on N-dim arrays

    >>> a = MutableDenseNDimArray(2, 2, [1, 1, 1, 1])
    >>> b = MutableDenseNDimArray(2, 2, [4, 4, 4, 4])
    >>> c = a + b
    >>> c
    [[5, 5], [5, 5]]
    >>> a - b
    [[-3, -3], [-3, -3]]

    """
    def __new__(cls, *args, **kwargs):
        from sympy.tensor.array import MutableDenseNDimArray
        return MutableDenseNDimArray(*args, **kwargs)

    def _parse_index(self, index):

        if isinstance(index, (int, Integer)):
            if index >= self._loop_size:
                raise ValueError("index out of range")
            return index

        if len(index) != self._rank:
            raise ValueError('Wrong number of array axes')

        real_index = 0
        # check if input index can exist in current indexing
        for i in range(self._rank):
            if index[i] > self.shape[i]:
                raise ValueError('Index ' + str(index) + ' out of border')
            real_index = real_index*self.shape[i] + index[i]

        return real_index

    def _get_tuple_index(self, integer_index):
        index = []
        for i, sh in enumerate(reversed(self.shape)):
            index.append(integer_index % sh)
            integer_index //= sh
        index.reverse()
        return tuple(index)

    def _setter_iterable_check(self, value):
        if isinstance(value, (collections.Iterable, Matrix, NDimArray)):
            raise NotImplementedError

    def __len__(self):
        """Overload common function len(). Returns number of elements in array.

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,3)
        >>> a
        [[0, 0, 0], [0, 0, 0], [0, 0, 0]]
        >>> len(a)
        9

        """
        return self._loop_size

    @property
    def shape(self):
        """
        Returns array shape (dimension).

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,3)
        >>> a.shape
        (3, 3)

        """
        return self._shape

    def rank(self):
        """
        Returns rank of array.

        Examples
        ========

        >>> from sympy.tensor.array.dense_ndim_array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(3,4,5,6,3)
        >>> a.rank()
        5

        """
        return self._rank

    def __str__(self):
        """Returns string, allows to use standard functions print() and str().

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(2, 2)
        >>> a
        [[0, 0], [0, 0]]

        """
        def f(sh, shape_left, i, j):
            if len(shape_left) == 1:
                return "["+", ".join([str(self[e]) for e in range(i, j)])+"]"

            sh //= shape_left[0]
            return "[" + ", ".join([f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh) for e in range(shape_left[0])]) + "]" # + "\n"*len(shape_left)

        return f(self._loop_size, self.shape, 0, self._loop_size)

        out_str = ''

        # forming output string
        for i, el in enumerate(self):

            out_str += str(el) + '  '
            chidx = i+1
            for sh in reversed(self.shape):
                if chidx % sh == 0:
                    out_str += '\n'
                    chidx //= sh

        return out_str

    def __repr__(self):
        return self.__str__()

    def tolist(self):
        """
        Conveting MutableDenseNDimArray to one-dim list

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray(2, 2, [1, 2, 3, 4])
        >>> a
        [[1, 2], [3, 4]]
        >>> b = a.tolist()
        >>> b
        [[1, 2], [3, 4]]
        """

        def f(sh, shape_left, i, j):
            if len(shape_left) == 1:
                return [self[e] for e in range(i, j)]
            result = []
            sh //= shape_left[0]
            for e in range(shape_left[0]):
                result.append(f(sh, shape_left[1:], i+e*sh, i+(e+1)*sh))
            return result

        return f(self._loop_size, self.shape, 0, self._loop_size)

    def __add__(self, other):
        if not isinstance(other, NDimArray):
            raise TypeError(str(other))

        if self.shape != other.shape:
            raise ValueError("array shape mismatch")
        result_list = [i+j for i,j in zip(self, other)]

        return type(self)(*(self.shape + (result_list,)))

    def __sub__(self, other):
        if not isinstance(other, NDimArray):
            raise TypeError(str(other))

        if self.shape != other.shape:
            raise ValueError("array shape mismatch")
        result_list = [i-j for i,j in zip(self, other)]

        return type(self)(*(self.shape + (result_list,)))

    def __mul__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [i*other for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __rmul__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [other*i for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __div__(self, other):
        if isinstance(other, (collections.Iterable,NDimArray, Matrix)):
            raise ValueError("scalar expected")
        other = sympify(other)
        result_list = [i/other for i in self]
        return type(self)(*(self.shape + (result_list,)))

    def __rdiv__(self, other):
        raise TypeError('unsupported operation on NDimArray')

    def __eq__(self, other):
        """
        NDimArray instances can be compared to each other.
        Instances equal if they have same shape and data.

        Examples
        ========

        >>> from sympy.tensor.array import MutableDenseNDimArray
        >>> a = MutableDenseNDimArray.zeros(2, 3)
        >>> b = MutableDenseNDimArray.zeros(2, 3)
        >>> a == b
        True
        >>> c = a.reshape(3, 2)
        >>> c == b
        False
        >>> a[0,0] = 1
        >>> b[0,0] = 2
        >>> a == b
        False
        """
        if not isinstance(other, NDimArray):
            return False
        return (self.shape == other.shape) and (list(self) == list(other))

    def __ne__(self, other):
        return not self.__eq__(other)

    __truediv__ = __div__
    __rtruediv__ = __rdiv__
<|endoftext|>


<commit_before><commit_msg>added force start from admin controls<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##  app.py
##  streamflow_prediction_tool
##
##  Created by Alan D. Snow.
##  Copyright © 2015-2016 Alan D. Snow. All rights reserved.
##  License: BSD 3-Clause
from uuid import uuid5, NAMESPACE_DNS
from datetime import datetime

from tethys_sdk.base import TethysAppBase, url_map_maker
from tethys_sdk.app_settings import CustomSetting, PersistentStoreDatabaseSetting

class StreamflowPredictionTool(TethysAppBase):
    """
    Tethys app class for streamflow_prediction_tool.
    """

    name = 'Streamflow Prediction Tool'
    index = 'streamflow_prediction_tool:home'
    icon = 'streamflow_prediction_tool/images/logo.png'
    package = 'streamflow_prediction_tool'
    root_url = 'streamflow-prediction-tool'
    color = '#34495e'
    description = 'Provides 15-day streamflow predicted estimates by using ' \
                  'ECMWF (ecmwf.int) runoff predictions routed with the RAPID ' \
                  '(rapid-hub.org) program. Return period estimates ' \
                  'and warning flags aid in determining the severity.'
    enable_feedback = False
    feedback_emails = []

    def url_maps(self):
        """
        Add controllers
        """
        UrlMap = url_map_maker(self.root_url)

        url_maps = (UrlMap(name='home',
                           url='streamflow-prediction-tool',
                           controller='streamflow_prediction_tool.controllers.home'),
                    UrlMap(name='map',
                           url='streamflow-prediction-tool/map',
                           controller='streamflow_prediction_tool.controllers.map'),
                    UrlMap(name='get_ecmwf_reach_statistical_hydrograph_ajax',
                           url='streamflow-prediction-tool/map/ecmwf-get-hydrograph',
                           controller='streamflow_prediction_tool.controllers_ajax.ecmwf_get_hydrograph'),
                    UrlMap(name='get_era_interim_reach_hydrograph_ajax',
                           url='streamflow-prediction-tool/map/era-interim-get-hydrograph',
                           controller='streamflow_prediction_tool.controllers_ajax.era_interim_get_hydrograph'),
                    UrlMap(name='get_era_interim_csv_ajax',
                           url='streamflow-prediction-tool/map/era-interim-get-csv',
                           controller='streamflow_prediction_tool.controllers_ajax.era_interim_get_csv'),
                    UrlMap(name='get_warning_points_ajax',
                           url='streamflow-prediction-tool/map/get-warning-points',
                           controller='streamflow_prediction_tool.controllers_ajax.generate_warning_points'),
                    UrlMap(name='ecmf_get_avaialable_dates_ajax',
                           url='streamflow-prediction-tool/map/ecmwf-get-avaialable-dates',
                           controller='streamflow_prediction_tool.controllers_ajax.ecmwf_get_avaialable_dates'),
                    UrlMap(name='get_seasonal_streamflow',
                           url='streamflow-prediction-tool/map/get_seasonal_streamflow',
                           controller='streamflow_prediction_tool.controllers_ajax.get_seasonal_streamflow'),
                    UrlMap(name='add-watershed',
                           url='streamflow-prediction-tool/add-watershed',
                           controller='streamflow_prediction_tool.controllers.add_watershed'),
                    UrlMap(name='add-watershed-ajax',
                           url='streamflow-prediction-tool/add-watershed/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_add'),
                    UrlMap(name='add-watershed-ecmwf-rapid-files-ajax',
                           url='streamflow-prediction-tool/add-watershed/upload_ecmwf_rapid',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_ecmwf_rapid_file_upload'),
                    UrlMap(name='add-watershed-update-ajax',
                           url='streamflow-prediction-tool/add-watershed/update',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_update'),
                    UrlMap(name='add-watershed-delete-ajax',
                           url='streamflow-prediction-tool/add-watershed/delete',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_delete'),
                    UrlMap(name='manage-watersheds',
                           url='streamflow-prediction-tool/manage-watersheds',
                           controller='streamflow_prediction_tool.controllers.manage_watersheds'),
                    UrlMap(name='manage-watersheds-table',
                           url='streamflow-prediction-tool/manage-watersheds/table',
                           controller='streamflow_prediction_tool.controllers.manage_watersheds_table'),
                    UrlMap(name='manage-watersheds-edit',
                           url='streamflow-prediction-tool/manage-watersheds/edit',
                           controller='streamflow_prediction_tool.controllers.edit_watershed'),
                    UrlMap(name='manage-watershed-ecmwf-rapid-files-ajax',
                           url='streamflow-prediction-tool/manage-watersheds/upload_ecmwf_rapid',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_ecmwf_rapid_file_upload'),
                    UrlMap(name='delete-watershed',
                           url='streamflow-prediction-tool/manage-watersheds/delete',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_delete'),
                    UrlMap(name='update-watershed',
                           url='streamflow-prediction-tool/manage-watersheds/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_update'),
                    UrlMap(name='add-geoserver',
                           url='streamflow-prediction-tool/add-geoserver',
                           controller='streamflow_prediction_tool.controllers.add_geoserver'),
                    UrlMap(name='add-geoserver-ajax',
                           url='streamflow-prediction-tool/add-geoserver/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.geoserver_add'),
                    UrlMap(name='manage-geoservers',
                           url='streamflow-prediction-tool/manage-geoservers',
                           controller='streamflow_prediction_tool.controllers.manage_geoservers'),
                    UrlMap(name='manage-geoservers-table',
                           url='streamflow-prediction-tool/manage-geoservers/table',
                           controller='streamflow_prediction_tool.controllers.manage_geoservers_table'),
                    UrlMap(name='update-geoservers-ajax',
                           url='streamflow-prediction-tool/manage-geoservers/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.geoserver_update'),
                    UrlMap(name='delete-geoserver-ajax',
                           url='streamflow-prediction-tool/manage-geoservers/delete',
                           controller='streamflow_prediction_tool.controllers_ajax.geoserver_delete'),
                    UrlMap(name='add-data-store',
                           url='streamflow-prediction-tool/add-data-store',
                           controller='streamflow_prediction_tool.controllers.add_data_store'),
                    UrlMap(name='add-data-store-ajax',
                           url='streamflow-prediction-tool/add-data-store/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.data_store_add'),
                    UrlMap(name='manage-data-stores',
                           url='streamflow-prediction-tool/manage-data-stores',
                           controller='streamflow_prediction_tool.controllers.manage_data_stores'),
                    UrlMap(name='manage-data-stores-table',
                           url='streamflow-prediction-tool/manage-data-stores/table',
                           controller='streamflow_prediction_tool.controllers.manage_data_stores_table'),
                    UrlMap(name='update-data-store-ajax',
                           url='streamflow-prediction-tool/manage-data-stores/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.data_store_update'),
                    UrlMap(name='delete-data-store-ajax',
                           url='streamflow-prediction-tool/manage-data-stores/delete',
                           controller='streamflow_prediction_tool.controllers_ajax.data_store_delete'),
                    UrlMap(name='add-watershed-group',
                           url='streamflow-prediction-tool/add-watershed-group',
                           controller='streamflow_prediction_tool.controllers.add_watershed_group'),
                    UrlMap(name='add-watershed-group-ajax',
                           url='streamflow-prediction-tool/add-watershed-group/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_group_add'),
                    UrlMap(name='manage-watershed-groups',
                           url='streamflow-prediction-tool/manage-watershed-groups',
                           controller='streamflow_prediction_tool.controllers.manage_watershed_groups'),
                    UrlMap(name='manage-watershed-groups-table',
                           url='streamflow-prediction-tool/manage-watershed-groups/table',
                           controller='streamflow_prediction_tool.controllers.manage_watershed_groups_table'),
                    UrlMap(name='update-watershed-group-ajax',
                           url='streamflow-prediction-tool/manage-watershed-groups/submit',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_group_update'),
                    UrlMap(name='delete-watershed-group-ajax',
                           url='streamflow-prediction-tool/manage-watershed-groups/delete',
                           controller='streamflow_prediction_tool.controllers_ajax.watershed_group_delete'),
                    UrlMap(name='getting-started',
                           url='streamflow-prediction-tool/getting-started',
                           controller='streamflow_prediction_tool.controllers.getting_started'),
                    UrlMap(name='publications',
                           url='streamflow-prediction-tool/publications',
                           controller='streamflow_prediction_tool.controllers.publications'),
                    UrlMap(name='waterml',
                           url='streamflow-prediction-tool/api/GetWaterML',
                           controller='streamflow_prediction_tool.api.get_waterml'),
                    UrlMap(name='era_interim',
                           url='streamflow-prediction-tool/api/GetHistoricData',
                           controller='streamflow_prediction_tool.api.get_historic_data'),
                    UrlMap(name='return_periods',
                           url='streamflow-prediction-tool/api/GetReturnPeriods',
                           controller='streamflow_prediction_tool.api.get_return_periods'),
                    UrlMap(name='return_periods',
                           url='streamflow-prediction-tool/api/GetAvailableDates',
                           controller='streamflow_prediction_tool.api.get_available_dates'),
                    UrlMap(name='watershed_list',
                           url='streamflow-prediction-tool/api/GetWatersheds',
                           controller='streamflow_prediction_tool.api.get_watershed_list'),
                    UrlMap(name='warning_points',
                           url='streamflow-prediction-tool/api/GetWarningPoints',
                           controller='streamflow_prediction_tool.api.get_warning_points'),
        )
        return url_maps

    def persistent_store_settings(self):
        """
        Define Persistent Store Settings.
        """
        ps_settings = (
            PersistentStoreDatabaseSetting(
                name='main_db',
                description='primary database',
                initializer='streamflow_prediction_tool.init_stores.init_main_db',
                required=True
            ),
        )

        return ps_settings

    def custom_settings(self):
        """
        Custom app settings.
        """
        custom_settings = (
            CustomSetting(
                name='app_instance_id',
                type=CustomSetting.TYPE_STRING,
                description='Unique identifier for this instance of the app.',
                value=uuid5(NAMESPACE_DNS, '%s%s' % ("spt", datetime.now())),
                required=True
            ),
            CustomSetting(
                name='ecmwf_forecast_folder',
                type=CustomSetting.TYPE_STRING,
                description='Server directory path to ECMWF forecasts.',
                required=True
            ),
            CustomSetting(
                name='historical_folder',
                type=CustomSetting.TYPE_STRING,
                description=('Server directory path to historical streamflow, '
                             'return period, and seasonal files.'),
                required=True
            ),
        )

        return custom_settings<commit_msg>updated to pass flake8 & pylint<commit_after># -*- coding: utf-8 -*-
"""app.py

    Created by Alan D. Snow.
    License: BSD 3-Clause
"""
from uuid import uuid5, NAMESPACE_DNS
from datetime import datetime

from tethys_sdk.base import TethysAppBase, url_map_maker
from tethys_sdk.app_settings import (CustomSetting,
                                     PersistentStoreDatabaseSetting)

class StreamflowPredictionTool(TethysAppBase):
    """
    Tethys app class for streamflow_prediction_tool.
    """

    name = 'Streamflow Prediction Tool'
    index = 'streamflow_prediction_tool:home'
    icon = 'streamflow_prediction_tool/images/logo.png'
    package = 'streamflow_prediction_tool'
    root_url = 'streamflow-prediction-tool'
    color = '#34495e'
    description = ('Provides 15-day streamflow predicted estimates by using '
                   'ECMWF (ecmwf.int) runoff predictions routed with the RAPID'
                   ' (rapid-hub.org) program. Return period estimates '
                   'and warning flags aid in determining the severity.')
    enable_feedback = False
    feedback_emails = []

    def url_maps(self):
        """
        Add controllers
        """
        url_map = url_map_maker(self.root_url)

        return (
            url_map(name='home',
                    url='streamflow-prediction-tool',
                    controller='streamflow_prediction_tool.controllers.home'),
            url_map(name='map',
                    url='streamflow-prediction-tool/map',
                    controller='streamflow_prediction_tool.controllers'
                               '.app_map'),
            url_map(name='get_ecmwf_reach_statistical_hydrograph_ajax',
                    url='streamflow-prediction-tool/map/ecmwf-get-hydrograph',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.ecmwf_get_hydrograph'),
            url_map(name='get_era_interim_reach_hydrograph_ajax',
                    url='streamflow-prediction-tool/map/'
                        'era-interim-get-hydrograph',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.era_interim_get_hydrograph'),
            url_map(name='get_era_interim_csv_ajax',
                    url='streamflow-prediction-tool/map/era-interim-get-csv',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.era_interim_get_csv'),
            url_map(name='get_warning_points_ajax',
                    url='streamflow-prediction-tool/map/get-warning-points',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.generate_warning_points'),
            url_map(name='ecmf_get_avaialable_dates_ajax',
                    url='streamflow-prediction-tool/map/'
                        'ecmwf-get-avaialable-dates',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.ecmwf_get_avaialable_dates'),
            url_map(name='get_seasonal_streamflow',
                    url='streamflow-prediction-tool/map/'
                        'get_seasonal_streamflow',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.get_seasonal_streamflow'),
            url_map(name='add-watershed',
                    url='streamflow-prediction-tool/add-watershed',
                    controller='streamflow_prediction_tool.controllers'
                               '.add_watershed'),
            url_map(name='add-watershed-ajax',
                    url='streamflow-prediction-tool/add-watershed/submit',
                    controller='streamflow_prediction_tool'
                               '.controllers_ajax.watershed_add'),
            url_map(name='add-watershed-ecmwf-rapid-files-ajax',
                    url='streamflow-prediction-tool/add-watershed/'
                        'upload_ecmwf_rapid',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_ecmwf_rapid_file_upload'),
            url_map(name='add-watershed-update-ajax',
                    url='streamflow-prediction-tool/add-watershed/update',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_update'),
            url_map(name='add-watershed-delete-ajax',
                    url='streamflow-prediction-tool/add-watershed/delete',
                    controller='streamflow_prediction_tool'
                               '.controllers_ajax.watershed_delete'),
            url_map(name='manage-watersheds',
                    url='streamflow-prediction-tool/manage-watersheds',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_watersheds'),
            url_map(name='manage-watersheds-table',
                    url='streamflow-prediction-tool/manage-watersheds/table',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_watersheds_table'),
            url_map(name='manage-watersheds-edit',
                    url='streamflow-prediction-tool/manage-watersheds/edit',
                    controller='streamflow_prediction_tool.controllers'
                               '.edit_watershed'),
            url_map(name='manage-watershed-ecmwf-rapid-files-ajax',
                    url='streamflow-prediction-tool/manage-watersheds/'
                        'upload_ecmwf_rapid',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_ecmwf_rapid_file_upload'),
            url_map(name='delete-watershed',
                    url='streamflow-prediction-tool/manage-watersheds/delete',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_delete'),
            url_map(name='update-watershed',
                    url='streamflow-prediction-tool/manage-watersheds/submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_update'),
            url_map(name='add-geoserver',
                    url='streamflow-prediction-tool/add-geoserver',
                    controller='streamflow_prediction_tool.controllers'
                               '.add_geoserver'),
            url_map(name='add-geoserver-ajax',
                    url='streamflow-prediction-tool/add-geoserver/submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.geoserver_add'),
            url_map(name='manage-geoservers',
                    url='streamflow-prediction-tool/manage-geoservers',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_geoservers'),
            url_map(name='manage-geoservers-table',
                    url='streamflow-prediction-tool/manage-geoservers/table',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_geoservers_table'),
            url_map(name='update-geoservers-ajax',
                    url='streamflow-prediction-tool/manage-geoservers/submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.geoserver_update'),
            url_map(name='delete-geoserver-ajax',
                    url='streamflow-prediction-tool/manage-geoservers/delete',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.geoserver_delete'),
            url_map(name='add-data-store',
                    url='streamflow-prediction-tool/add-data-store',
                    controller='streamflow_prediction_tool.controllers'
                               '.add_data_store'),
            url_map(name='add-data-store-ajax',
                    url='streamflow-prediction-tool/add-data-store/submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.data_store_add'),
            url_map(name='manage-data-stores',
                    url='streamflow-prediction-tool/manage-data-stores',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_data_stores'),
            url_map(name='manage-data-stores-table',
                    url='streamflow-prediction-tool/manage-data-stores/table',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_data_stores_table'),
            url_map(name='update-data-store-ajax',
                    url='streamflow-prediction-tool/manage-data-stores/submit',
                    controller='streamflow_prediction_tool'
                               '.controllers_ajax.data_store_update'),
            url_map(name='delete-data-store-ajax',
                    url='streamflow-prediction-tool/manage-data-stores/delete',
                    controller='streamflow_prediction_tool'
                               '.controllers_ajax.data_store_delete'),
            url_map(name='add-watershed-group',
                    url='streamflow-prediction-tool/add-watershed-group',
                    controller='streamflow_prediction_tool.controllers'
                               '.add_watershed_group'),
            url_map(name='add-watershed-group-ajax',
                    url='streamflow-prediction-tool/add-watershed-group/'
                        'submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_group_add'),
            url_map(name='manage-watershed-groups',
                    url='streamflow-prediction-tool/manage-watershed-groups',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_watershed_groups'),
            url_map(name='manage-watershed-groups-table',
                    url='streamflow-prediction-tool/manage-watershed-groups/'
                        'table',
                    controller='streamflow_prediction_tool.controllers'
                               '.manage_watershed_groups_table'),
            url_map(name='update-watershed-group-ajax',
                    url='streamflow-prediction-tool/manage-watershed-groups/'
                        'submit',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_group_update'),
            url_map(name='delete-watershed-group-ajax',
                    url='streamflow-prediction-tool/manage-watershed-groups/'
                        'delete',
                    controller='streamflow_prediction_tool.controllers_ajax'
                               '.watershed_group_delete'),
            url_map(name='getting-started',
                    url='streamflow-prediction-tool/getting-started',
                    controller='streamflow_prediction_tool.controllers'
                               '.getting_started'),
            url_map(name='publications',
                    url='streamflow-prediction-tool/publications',
                    controller='streamflow_prediction_tool.controllers'
                               '.publications'),
            url_map(name='waterml',
                    url='streamflow-prediction-tool/api/GetWaterML',
                    controller='streamflow_prediction_tool.api'
                               '.get_waterml'),
            url_map(name='era_interim',
                    url='streamflow-prediction-tool/api/GetHistoricData',
                    controller='streamflow_prediction_tool.api'
                               '.get_historic_data'),
            url_map(name='return_periods',
                    url='streamflow-prediction-tool/api/GetReturnPeriods',
                    controller='streamflow_prediction_tool.api'
                               '.get_return_periods'),
            url_map(name='return_periods',
                    url='streamflow-prediction-tool/api/GetAvailableDates',
                    controller='streamflow_prediction_tool.api'
                               '.get_available_dates'),
            url_map(name='watershed_list',
                    url='streamflow-prediction-tool/api/GetWatersheds',
                    controller='streamflow_prediction_tool.api'
                               '.get_watershed_list'),
            url_map(name='warning_points',
                    url='streamflow-prediction-tool/api/GetWarningPoints',
                    controller='streamflow_prediction_tool.api'
                               '.get_warning_points'),
        )

    def persistent_store_settings(self):
        """
        Define Persistent Store Settings.
        """
        return (
            PersistentStoreDatabaseSetting(
                name='main_db',
                description='primary database',
                initializer='streamflow_prediction_tool'
                            '.init_stores.init_main_db',
                required=True
            ),
        )

    def custom_settings(self):
        """
        Custom app settings.
        """
        return (
            CustomSetting(
                name='app_instance_id',
                type=CustomSetting.TYPE_STRING,
                description='Unique identifier for this instance of the app.',
                value=uuid5(NAMESPACE_DNS, '%s%s' % ("spt", datetime.now())),
                required=True
            ),
            CustomSetting(
                name='ecmwf_forecast_folder',
                type=CustomSetting.TYPE_STRING,
                description='Server directory path to ECMWF forecasts.',
                required=True
            ),
            CustomSetting(
                name='historical_folder',
                type=CustomSetting.TYPE_STRING,
                description=('Server directory path to historical streamflow, '
                             'return period, and seasonal files.'),
                required=True
            ),
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    __package__ = "keras_retinanet.bin"
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))

import keras
import keras.preprocessing.image
from keras.utils import multi_gpu_model

import tensorflow as tf

from .. import losses
from .. import layers
from ..callbacks import RedirectModel
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..models.resnet import ResNet50RetinaNet
from ..utils.keras_version import check_keras_version

def get_session():
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    return tf.Session(config=config)

def create_models(num_classes, weights='imagenet', multi_gpu=0):
    # create "base" model (no NMS)
    image = keras.layers.Input((None, None, 3))

    # Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
    # optionally wrap in a parallel model
    if multi_gpu > 1:
        with tf.device('/cpu:0'):
            model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
        training_model = multi_gpu_model(model, gpus=multi_gpu)
    else:
        model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
        training_model = model

    # append NMS for prediction only
    classification   = model.outputs[1]
    detections       = model.outputs[2]
    boxes            = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
    detections       = layers.NonMaximumSuppression(name='nms')([boxes, classification, detections])
    prediction_model = keras.models.Model(inputs=model.inputs, outputs=model.outputs[:2] + [detections])

    # compile model
    training_model.compile(
        loss={
            'regression'    : losses.smooth_l1(),
            'classification': losses.focal()
        },
        optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001)
    )

    return model, training_model, prediction_model

def create_callbacks(model, training_model, prediction_model, validation_generator, dataset_type, snapshot_path, evaluation, snapshots):
    callbacks = []

    # save the prediction model
    if snapshots:
        checkpoint = keras.callbacks.ModelCheckpoint(
            os.path.join(
                snapshot_path,
                'resnet50_{dataset_type}_{{epoch:02d}}.h5'.format(dataset_type=dataset_type)
            ),
            verbose=1
        )
        checkpoint = RedirectModel(checkpoint, prediction_model)
        callbacks.append(checkpoint)

    if evaluation and dataset_type == 'coco':
        from ..callbacks.coco import CocoEval

        # use prediction model for evaluation
        evaluation = CocoEval(validation_generator)
        evaluation = RedirectModel(evaluation, prediction_model)
        callbacks.append(evaluation)

    lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)
    callbacks.append(lr_scheduler)

    return callbacks

def create_generators(args):
    # create image data generator objects
    train_image_data_generator = keras.preprocessing.image.ImageDataGenerator(
        horizontal_flip=True,
    )
    val_image_data_generator = keras.preprocessing.image.ImageDataGenerator()

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        train_generator = CocoGenerator(
            args.coco_path,
            'train2017',
            train_image_data_generator,
            batch_size=args.batch_size
        )

        validation_generator = CocoGenerator(
            args.coco_path,
            'val2017',
            val_image_data_generator,
            batch_size=args.batch_size
        )
    elif args.dataset_type == 'pascal':
        train_generator = PascalVocGenerator(
            args.pascal_path,
            'trainval',
            train_image_data_generator,
            batch_size=args.batch_size
        )

        validation_generator = PascalVocGenerator(
            args.pascal_path,
            'test',
            val_image_data_generator,
            batch_size=args.batch_size
        )
    elif args.dataset_type == 'csv':
        train_generator = CSVGenerator(
            args.annotations,
            args.classes,
            train_image_data_generator,
            batch_size=args.batch_size
        )

        if args.val_annotations:
            validation_generator = CSVGenerator(
                args.val_annotations,
                args.classes,
                val_image_data_generator,
                batch_size=args.batch_size
            )
        else:
            validation_generator = None
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return train_generator, validation_generator

def check_args(parsed_args):
    """
    Function to check for inherent contradictions within parsed arguments.
    For example, batch_size < num_gpus
    Intended to raise errors prior to backend initialisation.

    :param parsed_args: parser.parse_args()
    :return: parsed_args
    """

    if parsed_args.multi_gpu > 1 and parsed_args.batch_size < parsed_args.multi_gpu:
        raise ValueError(
            "Batch size ({}) must be equal to or higher than the number of GPUs ({})".format(parsed_args.batch_size,
                                                                                parsed_args.multi_gpu))

    return parsed_args

def parse_args():
    parser     = argparse.ArgumentParser(description='Simple training script for training a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path', help='Path to dataset directory (ie. /tmp/COCO).')

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for training.')
    csv_parser.add_argument('classes', help='Path to a CSV file containing class label mapping.')
    csv_parser.add_argument('--val-annotations', help='Path to CSV file containing annotations for validation (optional).')

    parser.add_argument('--weights',       help='Weights to use for initialization (defaults to ImageNet).', default='imagenet')
    parser.add_argument('--batch-size',    help='Size of the batches.', default=1, type=int)
    parser.add_argument('--gpu',           help='Id of the GPU to use (as reported by nvidia-smi).')
    parser.add_argument('--multi-gpu',     help='Number of GPUs to use for parallel processing.', type=int, default=0)
    parser.add_argument('--epochs',        help='Number of epochs to train.', type=int, default=50)
    parser.add_argument('--steps',         help='Number of steps per epoch.', type=int, default=10000)
    parser.add_argument('--snapshot-path', help='Path to store snapshots of models during training (defaults to \'./snapshots\')', default='./snapshots')
    parser.add_argument('--no-evaluation', help='Disable per epoch evaluation.', dest='evaluation', action='store_false')
    parser.add_argument('--no-snapshots',  help='Disable saving snapshots.', dest='snapshots', action='store_false')
    parser.set_defaults(evaluation=True, snapshots=True)

    return check_args(parser.parse_args())

def main():
    # parse arguments
    args = parse_args()

    # make sure keras is the minimum required version
    check_keras_version()

    # optionally choose specific GPU
    if args.gpu:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    keras.backend.tensorflow_backend.set_session(get_session())

    # create the generators
    train_generator, validation_generator = create_generators(args)

    # create the model
    print('Creating model, this may take a second...')
    model, training_model, prediction_model = create_models(num_classes=train_generator.num_classes(), weights=args.weights, multi_gpu=args.multi_gpu)

    # print model summary
    print(model.summary())

    # create the callbacks
    callbacks = create_callbacks(
        model,
        training_model,
        prediction_model,
        validation_generator,
        dataset_type=args.dataset_type,
        snapshot_path=args.snapshot_path,
        evaluation=args.evaluation,
        snapshots=args.snapshots,
    )

    # start training
    training_model.fit_generator(
        generator=train_generator,
        steps_per_epoch=args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,
    )

if __name__ == '__main__':
    main()
<commit_msg>Move evaluation argument to COCO subparser.<commit_after>#!/usr/bin/env python

"""
Copyright 2017-2018 Fizyr (https://fizyr.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
import os
import sys

# Allow relative imports when being executed as script.
if __name__ == "__main__" and __package__ is None:
    __package__ = "keras_retinanet.bin"
    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))

import keras
import keras.preprocessing.image
from keras.utils import multi_gpu_model

import tensorflow as tf

from .. import losses
from .. import layers
from ..callbacks import RedirectModel
from ..preprocessing.pascal_voc import PascalVocGenerator
from ..preprocessing.csv_generator import CSVGenerator
from ..models.resnet import ResNet50RetinaNet
from ..utils.keras_version import check_keras_version

def get_session():
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    return tf.Session(config=config)

def create_models(num_classes, weights='imagenet', multi_gpu=0):
    # create "base" model (no NMS)
    image = keras.layers.Input((None, None, 3))

    # Keras recommends initialising a multi-gpu model on the CPU to ease weight sharing, and to prevent OOM errors.
    # optionally wrap in a parallel model
    if multi_gpu > 1:
        with tf.device('/cpu:0'):
            model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
        training_model = multi_gpu_model(model, gpus=multi_gpu)
    else:
        model = ResNet50RetinaNet(image, num_classes=num_classes, weights=weights, nms=False)
        training_model = model

    # append NMS for prediction only
    classification   = model.outputs[1]
    detections       = model.outputs[2]
    boxes            = keras.layers.Lambda(lambda x: x[:, :, :4])(detections)
    detections       = layers.NonMaximumSuppression(name='nms')([boxes, classification, detections])
    prediction_model = keras.models.Model(inputs=model.inputs, outputs=model.outputs[:2] + [detections])

    # compile model
    training_model.compile(
        loss={
            'regression'    : losses.smooth_l1(),
            'classification': losses.focal()
        },
        optimizer=keras.optimizers.adam(lr=1e-5, clipnorm=0.001)
    )

    return model, training_model, prediction_model

def create_callbacks(model, training_model, prediction_model, validation_generator, args):
    callbacks = []

    # save the prediction model
    if args.snapshots:
        checkpoint = keras.callbacks.ModelCheckpoint(
            os.path.join(
                args.snapshot_path,
                'resnet50_{dataset_type}_{{epoch:02d}}.h5'.format(dataset_type=args.dataset_type)
            ),
            verbose=1
        )
        checkpoint = RedirectModel(checkpoint, prediction_model)
        callbacks.append(checkpoint)

    if args.dataset_type == 'coco' and args.evaluation:
        from ..callbacks.coco import CocoEval

        # use prediction model for evaluation
        evaluation = CocoEval(validation_generator)
        evaluation = RedirectModel(evaluation, prediction_model)
        callbacks.append(evaluation)

    lr_scheduler = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience=2, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)
    callbacks.append(lr_scheduler)

    return callbacks

def create_generators(args):
    # create image data generator objects
    train_image_data_generator = keras.preprocessing.image.ImageDataGenerator(
        horizontal_flip=True,
    )
    val_image_data_generator = keras.preprocessing.image.ImageDataGenerator()

    if args.dataset_type == 'coco':
        # import here to prevent unnecessary dependency on cocoapi
        from ..preprocessing.coco import CocoGenerator

        train_generator = CocoGenerator(
            args.coco_path,
            'train2017',
            train_image_data_generator,
            batch_size=args.batch_size
        )

        validation_generator = CocoGenerator(
            args.coco_path,
            'val2017',
            val_image_data_generator,
            batch_size=args.batch_size
        )
    elif args.dataset_type == 'pascal':
        train_generator = PascalVocGenerator(
            args.pascal_path,
            'trainval',
            train_image_data_generator,
            batch_size=args.batch_size
        )

        validation_generator = PascalVocGenerator(
            args.pascal_path,
            'test',
            val_image_data_generator,
            batch_size=args.batch_size
        )
    elif args.dataset_type == 'csv':
        train_generator = CSVGenerator(
            args.annotations,
            args.classes,
            train_image_data_generator,
            batch_size=args.batch_size
        )

        if args.val_annotations:
            validation_generator = CSVGenerator(
                args.val_annotations,
                args.classes,
                val_image_data_generator,
                batch_size=args.batch_size
            )
        else:
            validation_generator = None
    else:
        raise ValueError('Invalid data type received: {}'.format(args.dataset_type))

    return train_generator, validation_generator

def check_args(parsed_args):
    """
    Function to check for inherent contradictions within parsed arguments.
    For example, batch_size < num_gpus
    Intended to raise errors prior to backend initialisation.

    :param parsed_args: parser.parse_args()
    :return: parsed_args
    """

    if parsed_args.multi_gpu > 1 and parsed_args.batch_size < parsed_args.multi_gpu:
        raise ValueError(
            "Batch size ({}) must be equal to or higher than the number of GPUs ({})".format(parsed_args.batch_size,
                                                                                parsed_args.multi_gpu))

    return parsed_args

def parse_args():
    parser     = argparse.ArgumentParser(description='Simple training script for training a RetinaNet network.')
    subparsers = parser.add_subparsers(help='Arguments for specific dataset types.', dest='dataset_type')
    subparsers.required = True

    coco_parser = subparsers.add_parser('coco')
    coco_parser.add_argument('coco_path', help='Path to dataset directory (ie. /tmp/COCO).')
    coco_parser.add_argument('--no-evaluation', help='Disable per epoch evaluation.', dest='evaluation', action='store_false')
    coco_parser.set_defaults(evaluation=True)

    pascal_parser = subparsers.add_parser('pascal')
    pascal_parser.add_argument('pascal_path', help='Path to dataset directory (ie. /tmp/VOCdevkit).')

    csv_parser = subparsers.add_parser('csv')
    csv_parser.add_argument('annotations', help='Path to CSV file containing annotations for training.')
    csv_parser.add_argument('classes', help='Path to a CSV file containing class label mapping.')
    csv_parser.add_argument('--val-annotations', help='Path to CSV file containing annotations for validation (optional).')

    parser.add_argument('--weights',       help='Weights to use for initialization (defaults to ImageNet).', default='imagenet')
    parser.add_argument('--batch-size',    help='Size of the batches.', default=1, type=int)
    parser.add_argument('--gpu',           help='Id of the GPU to use (as reported by nvidia-smi).')
    parser.add_argument('--multi-gpu',     help='Number of GPUs to use for parallel processing.', type=int, default=0)
    parser.add_argument('--epochs',        help='Number of epochs to train.', type=int, default=50)
    parser.add_argument('--steps',         help='Number of steps per epoch.', type=int, default=10000)
    parser.add_argument('--snapshot-path', help='Path to store snapshots of models during training (defaults to \'./snapshots\')', default='./snapshots')
    parser.add_argument('--no-snapshots',  help='Disable saving snapshots.', dest='snapshots', action='store_false')
    parser.set_defaults(snapshots=True)

    return check_args(parser.parse_args())

def main():
    # parse arguments
    args = parse_args()

    # make sure keras is the minimum required version
    check_keras_version()

    # optionally choose specific GPU
    if args.gpu:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
    keras.backend.tensorflow_backend.set_session(get_session())

    # create the generators
    train_generator, validation_generator = create_generators(args)

    # create the model
    print('Creating model, this may take a second...')
    model, training_model, prediction_model = create_models(num_classes=train_generator.num_classes(), weights=args.weights, multi_gpu=args.multi_gpu)

    # print model summary
    print(model.summary())

    # create the callbacks
    callbacks = create_callbacks(
        model,
        training_model,
        prediction_model,
        validation_generator,
        args,
    )

    # start training
    training_model.fit_generator(
        generator=train_generator,
        steps_per_epoch=args.steps,
        epochs=args.epochs,
        verbose=1,
        callbacks=callbacks,
    )

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>'''
Installation of Python packages using pip.
==========================================

A state module to manage system installed python packages

.. code-block:: yaml

    virtualenvwrapper:
      pip.installed:
        - version: 3.0.1
'''

# Import Salt libs
from salt.exceptions import CommandExecutionError, CommandNotFoundError

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              log=None,
              proxy=None,
              timeout=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              cwd=None):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    pip_bin :  None
        Deprecated, use bin_env
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}
    try:
        pip_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
    except (CommandNotFoundError, CommandExecutionError) as err:
        ret['result'] = False
        ret['comment'] = 'Error installing \'{0}\': {1}'.format(name, err)
        return ret

    if ignore_installed == False and name in pip_list:
        if force_reinstall == False and upgrade == False:
            ret['result'] = True
            ret['comment'] = 'Package already installed'
            return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Python package {0} is set to be installed'.format(
                name)
        return ret

    pip_install_call = __salt__['pip.install'](
        pkgs=name,
        requirements=requirements,
        bin_env=bin_env,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        runas=user,
        cwd=cwd
    )

    if pip_install_call and pip_install_call['retcode']==0:
        ret['result'] = True

        pkg_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
        if not pkg_list:
            ret['comment'] = (
                'There was no error installing package \'{0}\' although '
                'it does not show when calling \'pip.freeze\'.'.format(name)
            )
            ret['changes']["{0}==???".format(name)] = 'Installed'
            return ret

        version = list(pkg_list.values())[0]
        pkg_name = next(iter(pkg_list))
        ret['changes']["{0}=={1}".format(pkg_name, version)] = 'Installed'
        ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        ret['comment'] = 'Failed to install package {0}. Error: {1}'.format(
            name, pip_install_call['stderr']
        )
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            packages=None,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            cwd=None):
    """
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    bin_env : None
        the pip executable or virtualenenv to use
    """

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    try:
        pip_list = __salt__["pip.list"](bin_env=bin_env, runas=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError), err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret["result"] = True
        ret["comment"] = "Package is not installed."
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__["pip.uninstall"](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 runas=user,
                                 cwd=cwd):
        ret["result"] = True
        ret["changes"][name] = "Removed"
        ret["comment"] = "Package was successfully removed."
    else:
        ret["result"] = False
        ret["comment"] = "Could not remove package."
    return ret
<commit_msg>fix another exception syntax in the pip state<commit_after>'''
Installation of Python packages using pip.
==========================================

A state module to manage system installed python packages

.. code-block:: yaml

    virtualenvwrapper:
      pip.installed:
        - version: 3.0.1
'''

# Import Salt libs
from salt.exceptions import CommandExecutionError, CommandNotFoundError

def installed(name,
              pip_bin=None,
              requirements=None,
              env=None,
              bin_env=None,
              log=None,
              proxy=None,
              timeout=None,
              editable=None,
              find_links=None,
              index_url=None,
              extra_index_url=None,
              no_index=False,
              mirrors=None,
              build=None,
              target=None,
              download=None,
              download_cache=None,
              source=None,
              upgrade=False,
              force_reinstall=False,
              ignore_installed=False,
              no_deps=False,
              no_install=False,
              no_download=False,
              install_options=None,
              user=None,
              cwd=None):
    '''
    Make sure the package is installed

    name
        The name of the python package to install
    pip_bin :  None
        Deprecated, use bin_env
    env : None
        Deprecated, use bin_env
    bin_env : None
        the pip executable or virtualenv to use

    '''
    if pip_bin and not bin_env:
        bin_env = pip_bin
    elif env and not bin_env:
        bin_env = env

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}
    try:
        pip_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
    except (CommandNotFoundError, CommandExecutionError) as err:
        ret['result'] = False
        ret['comment'] = 'Error installing \'{0}\': {1}'.format(name, err)
        return ret

    if ignore_installed == False and name in pip_list:
        if force_reinstall == False and upgrade == False:
            ret['result'] = True
            ret['comment'] = 'Package already installed'
            return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Python package {0} is set to be installed'.format(
                name)
        return ret

    pip_install_call = __salt__['pip.install'](
        pkgs=name,
        requirements=requirements,
        bin_env=bin_env,
        log=log,
        proxy=proxy,
        timeout=timeout,
        editable=editable,
        find_links=find_links,
        index_url=index_url,
        extra_index_url=extra_index_url,
        no_index=no_index,
        mirrors=mirrors,
        build=build,
        target=target,
        download=download,
        download_cache=download_cache,
        source=source,
        upgrade=upgrade,
        force_reinstall=force_reinstall,
        ignore_installed=ignore_installed,
        no_deps=no_deps,
        no_install=no_install,
        no_download=no_download,
        install_options=install_options,
        runas=user,
        cwd=cwd
    )

    if pip_install_call and pip_install_call['retcode']==0:
        ret['result'] = True

        pkg_list = __salt__['pip.list'](name, bin_env, runas=user, cwd=cwd)
        if not pkg_list:
            ret['comment'] = (
                'There was no error installing package \'{0}\' although '
                'it does not show when calling \'pip.freeze\'.'.format(name)
            )
            ret['changes']["{0}==???".format(name)] = 'Installed'
            return ret

        version = list(pkg_list.values())[0]
        pkg_name = next(iter(pkg_list))
        ret['changes']["{0}=={1}".format(pkg_name, version)] = 'Installed'
        ret['comment'] = 'Package was successfully installed'
    elif pip_install_call:
        ret['result'] = False
        ret['comment'] = 'Failed to install package {0}. Error: {1}'.format(
            name, pip_install_call['stderr']
        )
    else:
        ret['result'] = False
        ret['comment'] = 'Could not install package'

    return ret

def removed(name,
            packages=None,
            requirements=None,
            bin_env=None,
            log=None,
            proxy=None,
            timeout=None,
            user=None,
            cwd=None):
    """
    Make sure that a package is not installed.

    name
        The name of the package to uninstall
    bin_env : None
        the pip executable or virtualenenv to use
    """

    ret = {'name': name, 'result': None, 'comment': '', 'changes': {}}

    try:
        pip_list = __salt__["pip.list"](bin_env=bin_env, runas=user, cwd=cwd)
    except (CommandExecutionError, CommandNotFoundError) as err:
        ret['result'] = False
        ret['comment'] = 'Error uninstalling \'{0}\': {1}'.format(name, err)
        return ret

    if name not in pip_list:
        ret["result"] = True
        ret["comment"] = "Package is not installed."
        return ret

    if __opts__['test']:
        ret['result'] = None
        ret['comment'] = 'Package {0} is set to be removed'.format(name)
        return ret

    if __salt__["pip.uninstall"](pkgs=name,
                                 requirements=requirements,
                                 bin_env=bin_env,
                                 log=log,
                                 proxy=proxy,
                                 timeout=timeout,
                                 runas=user,
                                 cwd=cwd):
        ret["result"] = True
        ret["changes"][name] = "Removed"
        ret["comment"] = "Package was successfully removed."
    else:
        ret["result"] = False
        ret["comment"] = "Could not remove package."
    return ret
<|endoftext|>