<commit_before># Copyright (C) 2016 Antoine Carme <Antoine.Carme@Laposte.net>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license

import pandas as pd
import numpy as np

class cModelControl:

    def __init__(self):
        self.mActiveTransformations = {};
        self.mActivePeriodics = {};
        self.mActiveTrends = {};
        self.mActiveAutoRegressions = {};
        self.mKnownTransformations = ['None', 'Difference', 'RelativeDifference',
                                      'Integration', 'BoxCox',
                                      'Quantization', 'Logit',
                                      'Fisher', 'Anscombe'];
        self.mKnownTrends = ['ConstantTrend',
                             'Lag1Trend', 'LinearTrend', 'PolyTrend',
                             'MovingAverage', 'MovingMedian'];
        self.mKnownPeriodics = ['NoCycle', 'BestCycle',
                                'Seasonal_MonthOfYear' ,
                                'Seasonal_Second' ,
                                'Seasonal_Minute' ,
                                'Seasonal_Hour' ,
                                'Seasonal_DayOfWeek' ,
                                'Seasonal_DayOfMonth',
                                'Seasonal_WeekOfYear'];
        self.mKnownAutoRegressions = ['NoAR' , 'AR' , 'ARX' , 'SVR', 'MLP' , 'LSTM' , 'XGB' , 'XGBX'];
        # now , set he default models
        self.set_active_transformations(self.mKnownTransformations[0:4]);
        self.set_active_trends(self.mKnownTrends[0:4]);
        self.set_active_periodics(self.mKnownPeriodics);
        self.set_active_autoregressions(self.mKnownAutoRegressions[0:3]);

    def set_active_transformations(self, transformations):
        self.mActiveTransformations = {};
        for transformation in self.mKnownTransformations:
            if(transformation in transformations):
                self.mActiveTransformations[transformation] = True;
            else:
                self.mActiveTransformations[transformation] = False;
        if(True not in self.mActiveTransformations.values()):
            # default
            self.mActiveTransformations['None'] = True;

    def set_active_trends(self, trends):
        self.mActiveTrends = {};
        for trend in self.mKnownTrends:
            if(trend in trends):
                self.mActiveTrends[trend] = True;
            else:
                self.mActiveTrends[trend] = False;
        if(True not in self.mActiveTrends.values()):
            # default
            self.mActiveTrends['ConstantTrend'] = True;

    def set_active_periodics(self, periodics):
        self.mActivePeriodics = {};
        for period in self.mKnownPeriodics:
            if(period in periodics):
                self.mActivePeriodics[period] = True;
            else:
                self.mActivePeriodics[period] = False;
        if(True not in self.mActivePeriodics.values()):
            # default
            self.mActivePeriodics['NoCycle'] = True;

    def set_active_autoregressions(self, autoregs):
        self.mActiveAutoRegressions = {};
        for autoreg in self.mKnownAutoRegressions:
            if(autoreg in autoregs):
                self.mActiveAutoRegressions[autoreg] = True;
            else:
                self.mActiveAutoRegressions[autoreg] = False;
        if(True not in self.mActiveAutoRegressions.values()):
            # default
            self.mActiveAutoRegressions['NoAR'] = True;

    def disable_all_transformations(self):
        self.set_active_transformations([]);

    def disable_all_trends(self):
        self.set_active_trends([]);

    def disable_all_periodics(self):
        self.set_active_periodics([]);

    def disable_all_autoregressions(self):
        self.set_active_autoregressions([]);

class cSignalDecomposition_Options(cModelControl):

    def __init__(self):
        super().__init__();
        self.mParallelMode = True;
        self.mNbCores = 8;
        self.mEstimRatio = 0.8;
        self.mCustomSplit = None
        self.mAddPredictionIntervals = True
        self.enable_fast_mode();
        self.mTimeDeltaComputationMethod = "AVG"; # can be "AVG", "MODE", "USER"
        self.mUserTimeDelta = None;
        self.mBusinessDaysOnly = False;
        self.mMaxExogenousCategories = 5;
        self.mNoBoxCoxOrders = [];
        self.mBoxCoxOrders = [-2.0, -1.0 , 0.0,  2.0];
        self.mExtensiveBoxCoxOrders = [-2, -1, -0.5, -0.33 , -0.25 , 0.0, 2, 0.5, 0.33 , 0.25];
        self.mMaxFeatureForAutoreg = 1000;
        self.mModelSelection_Criterion = "MAPE";
        self.mCycle_Criterion = "L2";
        self.mCycle_Criterion_Threshold = None;
        self.mHierarchicalCombinationMethod = "BU";
        self.mForecastRectifier = None # can be "relu" to force positive forecast values
        self.mXGBOptions = None
        self.disableDebuggingOptions();

    def disableDebuggingOptions(self):
        self.mDebug = False;
        self.mDebugCycles = False;
        self.mDebugProfile = False;
        self.mDebugPerformance = False;

    def enable_slow_mode(self):
        self.mQuantiles = [5, 10, 20]; # quintiles, deciles, and vingtiles;)
        self.mMovingAverageLengths = [5, 7, 12, 24 , 30, 60];
        self.mMovingMedianLengths = [5, 7, 12, 24 , 30, 60];
        # PyAF does not detect complex seasonal patterns #73.
        # use unlimited cycle lengths in slow mode
        self.mCycleLengths = None;

        self.set_active_transformations(self.mKnownTransformations);
        self.set_active_trends(self.mKnownTrends);
        self.set_active_periodics(self.mKnownPeriodics);
        self.set_active_autoregressions(self.mKnownAutoRegressions);

        self.mMaxAROrder = 64;
        self.mFilterSeasonals = False

    def enable_fast_mode(self):
        self.mQuantiles = [5, 10, 20]; # quintiles, deciles, and vingtiles;)
        self.mMovingAverageLengths = [5, 7, 12, 24 , 30, 60];
        self.mMovingMedianLengths = [5, 7, 12, 24 , 30, 60];

        self.mCycleLengths = [5, 7, 12, 24 , 30, 60];

        self.mMaxAROrder = 64;
        self.mFilterSeasonals = True

    # Add a low-memory mode for Heroku #25
    def enable_low_memory_mode(self):
        self.mMaxAROrder = 7;
        self.set_active_transformations(['None']);
        self.mParallelMode = False;
        self.mFilterSeasonals = True

    '''
    Cannot yet build keras models in parallel/multiprocessing in some cases
    (tensorflow backend). theano seems OK.
    Possible solution : increase developer knowledge of keras !!
    '''
    def  canBuildKerasModel(self, iModel):
        try:
            import keras
            import keras
            from keras import callbacks
            from keras.models import Sequential
            from keras.layers import Dense, Dropout
            from keras.layers import LSTM
            lBackEnd = keras.backend.backend()
            if((lBackEnd == "tensorflow") and (self.mParallelMode)):
                return False;
            else:
                return True;
        except:
            return False;

<commit_msg>Add the possibility to use cross validation when training PyAF models #105<commit_after># Copyright (C) 2016 Antoine Carme <Antoine.Carme@Laposte.net>
# All rights reserved.

# This file is part of the Python Automatic Forecasting (PyAF) library and is made available under
# the terms of the 3 Clause BSD license

import pandas as pd
import numpy as np

class cModelControl:

    def __init__(self):
        self.mActiveTransformations = {};
        self.mActivePeriodics = {};
        self.mActiveTrends = {};
        self.mActiveAutoRegressions = {};
        self.mKnownTransformations = ['None', 'Difference', 'RelativeDifference',
                                      'Integration', 'BoxCox',
                                      'Quantization', 'Logit',
                                      'Fisher', 'Anscombe'];
        self.mKnownTrends = ['ConstantTrend',
                             'Lag1Trend', 'LinearTrend', 'PolyTrend',
                             'MovingAverage', 'MovingMedian'];
        self.mKnownPeriodics = ['NoCycle', 'BestCycle',
                                'Seasonal_MonthOfYear' ,
                                'Seasonal_Second' ,
                                'Seasonal_Minute' ,
                                'Seasonal_Hour' ,
                                'Seasonal_DayOfWeek' ,
                                'Seasonal_DayOfMonth',
                                'Seasonal_WeekOfYear'];
        self.mKnownAutoRegressions = ['NoAR' , 'AR' , 'ARX' , 'SVR', 'MLP' , 'LSTM' , 'XGB' , 'XGBX'];
        # now , set he default models
        self.set_active_transformations(self.mKnownTransformations[0:4]);
        self.set_active_trends(self.mKnownTrends[0:4]);
        self.set_active_periodics(self.mKnownPeriodics);
        self.set_active_autoregressions(self.mKnownAutoRegressions[0:3]);

    def set_active_transformations(self, transformations):
        self.mActiveTransformations = {};
        for transformation in self.mKnownTransformations:
            if(transformation in transformations):
                self.mActiveTransformations[transformation] = True;
            else:
                self.mActiveTransformations[transformation] = False;
        if(True not in self.mActiveTransformations.values()):
            # default
            self.mActiveTransformations['None'] = True;

    def set_active_trends(self, trends):
        self.mActiveTrends = {};
        for trend in self.mKnownTrends:
            if(trend in trends):
                self.mActiveTrends[trend] = True;
            else:
                self.mActiveTrends[trend] = False;
        if(True not in self.mActiveTrends.values()):
            # default
            self.mActiveTrends['ConstantTrend'] = True;

    def set_active_periodics(self, periodics):
        self.mActivePeriodics = {};
        for period in self.mKnownPeriodics:
            if(period in periodics):
                self.mActivePeriodics[period] = True;
            else:
                self.mActivePeriodics[period] = False;
        if(True not in self.mActivePeriodics.values()):
            # default
            self.mActivePeriodics['NoCycle'] = True;

    def set_active_autoregressions(self, autoregs):
        self.mActiveAutoRegressions = {};
        for autoreg in self.mKnownAutoRegressions:
            if(autoreg in autoregs):
                self.mActiveAutoRegressions[autoreg] = True;
            else:
                self.mActiveAutoRegressions[autoreg] = False;
        if(True not in self.mActiveAutoRegressions.values()):
            # default
            self.mActiveAutoRegressions['NoAR'] = True;

    def disable_all_transformations(self):
        self.set_active_transformations([]);

    def disable_all_trends(self):
        self.set_active_trends([]);

    def disable_all_periodics(self):
        self.set_active_periodics([]);

    def disable_all_autoregressions(self):
        self.set_active_autoregressions([]);

class cCrossValidationOptions:
    def __init__(self):
        self.mMethod = None;
        self.mNbFolds = 10

class cSignalDecomposition_Options(cModelControl):

    def __init__(self):
        super().__init__();
        self.mParallelMode = True;
        self.mNbCores = 8;
        self.mEstimRatio = 0.8; # to be deprecated when cross validation is OK.
        self.mCustomSplit = None
        self.mAddPredictionIntervals = True
        self.enable_fast_mode();
        self.mTimeDeltaComputationMethod = "AVG"; # can be "AVG", "MODE", "USER"
        self.mUserTimeDelta = None;
        self.mBusinessDaysOnly = False;
        self.mMaxExogenousCategories = 5;
        self.mNoBoxCoxOrders = [];
        self.mBoxCoxOrders = [-2.0, -1.0 , 0.0,  2.0];
        self.mExtensiveBoxCoxOrders = [-2, -1, -0.5, -0.33 , -0.25 , 0.0, 2, 0.5, 0.33 , 0.25];
        self.mMaxFeatureForAutoreg = 1000;
        self.mModelSelection_Criterion = "MAPE";
        self.mCycle_Criterion = "L2";
        self.mCycle_Criterion_Threshold = None;
        self.mHierarchicalCombinationMethod = "BU";
        self.mForecastRectifier = None # can be "relu" to force positive forecast values
        self.mXGBOptions = None
        self.mCrossValidationOptions = cCrossValidationOptions()
        self.disableDebuggingOptions();

    def disableDebuggingOptions(self):
        self.mDebug = False;
        self.mDebugCycles = False;
        self.mDebugProfile = False;
        self.mDebugPerformance = False;

    def enable_slow_mode(self):
        self.mQuantiles = [5, 10, 20]; # quintiles, deciles, and vingtiles;)
        self.mMovingAverageLengths = [5, 7, 12, 24 , 30, 60];
        self.mMovingMedianLengths = [5, 7, 12, 24 , 30, 60];
        # PyAF does not detect complex seasonal patterns #73.
        # use unlimited cycle lengths in slow mode
        self.mCycleLengths = None;

        self.set_active_transformations(self.mKnownTransformations);
        self.set_active_trends(self.mKnownTrends);
        self.set_active_periodics(self.mKnownPeriodics);
        self.set_active_autoregressions(self.mKnownAutoRegressions);

        self.mMaxAROrder = 64;
        self.mFilterSeasonals = False

    def enable_fast_mode(self):
        self.mQuantiles = [5, 10, 20]; # quintiles, deciles, and vingtiles;)
        self.mMovingAverageLengths = [5, 7, 12, 24 , 30, 60];
        self.mMovingMedianLengths = [5, 7, 12, 24 , 30, 60];

        self.mCycleLengths = [5, 7, 12, 24 , 30, 60];

        self.mMaxAROrder = 64;
        self.mFilterSeasonals = True

    # Add a low-memory mode for Heroku #25
    def enable_low_memory_mode(self):
        self.mMaxAROrder = 7;
        self.set_active_transformations(['None']);
        self.mParallelMode = False;
        self.mFilterSeasonals = True

    '''
    Cannot yet build keras models in parallel/multiprocessing in some cases
    (tensorflow backend). theano seems OK.
    Possible solution : increase developer knowledge of keras !!
    '''
    def  canBuildKerasModel(self, iModel):
        try:
            import keras
            import keras
            from keras import callbacks
            from keras.models import Sequential
            from keras.layers import Dense, Dropout
            from keras.layers import LSTM
            lBackEnd = keras.backend.backend()
            if((lBackEnd == "tensorflow") and (self.mParallelMode)):
                return False;
            else:
                return True;
        except:
            return False;

<|endoftext|>


<commit_before>import json
import re
from django.core.exceptions import ObjectDoesNotExist
from nexus import NexusReader
from rest_framework import viewsets
from serializers import *
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import *
from rest_framework.views import Request, Response
from filters import *

# Resource routes
class VariableDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableDescriptionSerializer
    filter_fields = ('label', 'name', 'index_categories', 'niche_categories',)
    queryset = VariableDescription.objects.all()
    # Override retrieve to use the detail serializer, which includes categories
    def retrieve(self, request, *args, **kwargs):
        self.object = self.get_object()
        serializer = VariableDescriptionDetailSerializer(self.object)
        return Response(serializer.data)

class VariableCategoryViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableCategorySerializer
    filter_fields = ('name', 'index_variables', 'niche_variables',)
    queryset = VariableCategory.objects.all()
    # Override retrieve to use the detail serializer, which includes variables
    def retrieve(self, request, *args, **kwargs):
        self.object = self.get_object()
        serializer = VariableCategoryDetailSerializer(self.object)
        return Response(serializer.data)

class VariableCodeDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = VariableCodeDescriptionSerializer
    filter_class = VariableCodeDescriptionFilter
    queryset = VariableCodeDescription.objects.all()

# Can filter by code, code__variable, or society
class VariableCodedValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableCodedValueSerializer
    filter_fields = ('variable','coded_value','code','society','code',)
    # Avoid additional database trips by select_related for foreign keys
    queryset = VariableCodedValue.objects.select_related('variable').select_related('code').all()

class SocietyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = SocietySerializer
    queryset = Society.objects.all()

class ISOCodeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = ISOCodeSerializer
    filter_fields = ('iso_code',)
    queryset = ISOCode.objects.all()

class EnvironmentalVariableViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalVariableSerializer
    filter_fields = ('name', 'units',)
    queryset = EnvironmentalVariable.objects.all()

class EnvironmentalValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalValueSerializer
    filter_fields = ('variable','environmental',)
    queryset = EnvironmentalValue.objects.all()

class EnvironmentalViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalSerializer
    filter_fields = ('society', 'iso_code',)
    queryset = Environmental.objects.all()

class LanguageClassViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = LanguageClassSerializer
    filter_class = LanguageClassFilter
    queryset = LanguageClass.objects.all()

# Need an API to get classifications / languages for a class

class LanguageClassificationViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = LanguageClassificationSerializer
    filter_class = LanguageClassificationFilter
    queryset = LanguageClassification.objects.all()

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageSerializer
    filter_fields = ('name', 'iso_code', 'society',)
    queryset = Language.objects.all()

class LanguageTreeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageTreeSerializer
    filter_fields = ('name',)
    queryset = LanguageTree.objects.all()

def result_set_from_query_dict(query_dict):
    result_set = SocietyResultSet()
    # Criteria keeps track of what types of data were searched on, so that we can
    # AND them together
    criteria = []

    if 'language_filters' in query_dict:
        criteria.append(SEARCH_LANGUAGE)
        language_filters = query_dict['language_filters']
        for filter in language_filters:
            language_ids = [int(x) for x in filter['language_ids']]
            languages = Language.objects.filter(pk__in=language_ids) # Returns a queryset
            languages.select_related('societies')
            for language in languages:
                for society in language.societies.all():
                    result_set.add_language(society,language)

    if 'variable_codes' in query_dict:
        criteria.append(SEARCH_VARIABLES)
        # Now get the societies from variables
        variable_code_ids = [int(x) for x in query_dict['variable_codes']]
        codes = VariableCodeDescription.objects.filter(pk__in=variable_code_ids) # returns a queryset
        coded_value_ids = []
        # Aggregate all the coded values for each selected code
        for code in codes:
            coded_value_ids += code.variablecodedvalue_set.values_list('id', flat=True)
        # Coded values have a FK to society.  Aggregate the societies from each value
        values = VariableCodedValue.objects.filter(id__in=coded_value_ids)
        values = values.select_related('society','variable')
        for value in values:
            result_set.add_cultural(value.society,value.variable,value)

    if 'environmental_filters' in query_dict:
        criteria.append(SEARCH_ENVIRONMENTAL)
        environmental_filters = query_dict['environmental_filters']
        # There can be multiple filters, so we must aggregate the results.
        for filter in environmental_filters:
            values = EnvironmentalValue.objects.filter(variable=filter['id'])
            operator = filter['operator']
            if operator == 'inrange':
                values = values.filter(value__gt=filter['params'][0]).filter(value__lt=filter['params'][1])
            elif operator == 'outrange':
                values = values.filter(value__gt=filter['params'][1]).filter(value__lt=filter['params'][0])
            elif operator == 'gt':
                values = values.filter(value__gt=filter['params'][0])
            elif operator == 'lt':
                values = values.filter(value__lt=filter['params'][0])
            values = values.select_related('variable','environmental__society')
            # get the societies from the values
            for value in values:
                result_set.add_environmental(value.society(), value.variable, value)
    if 'geographic_regions' in query_dict:
        criteria.append(SEARCH_GEOGRAPHIC)
        geographic_region_ids = [int(x) for x in query_dict['geographic_regions']]
        regions = GeographicRegion.objects.filter(pk__in=geographic_region_ids) # returns a queryset
        for region in regions:
            for society in Society.objects.filter(location__intersects=region.geom):
                result_set.add_geographic_region(society, region)
    # Filter the results to those that matched all criteria
    result_set.finalize(criteria)
    return result_set

# search/filter APIs
@api_view(['POST'])
@permission_classes((AllowAny,))
def find_societies(request):
    """
    View to find the societies that match an input request.  Currently expects
    { language_filters: [{language_ids: [1,2,3]}], variable_codes: [4,5,6...],
    environmental_filters: [{id: 1, operator: 'gt', params: [0.0]}, {id:3, operator 'inrange', params: [10.0,20.0] }] }

    Returns serialized collection of SocietyResult objects
    """
    result_set = result_set_from_query_dict(request.DATA)
    return Response(SocietyResultSetSerializer(result_set).data)

class GeographicRegionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = GeographicRegionSerializer
    model = GeographicRegion
    filter_class = GeographicRegionFilter

@api_view(['POST'])
@permission_classes((AllowAny,))
def trees_from_languages(request):
    if 'language_ids' in request.DATA:
        language_ids = [int(x) for x in request.DATA['language_ids']]
        trees = LanguageTree.objects.filter(languages__pk__in=language_ids).distinct()
    else:
        trees = None
    return Response(LanguageTreeSerializer(trees, many=True).data,)

@api_view(['GET'])
@permission_classes((AllowAny,))
def newick_tree(request, *args, **kwargs):
    # Get a newick format tree from a language tree id
    language_tree_id = kwargs['pk']
    try:
        language_tree = LanguageTree.objects.get(pk=language_tree_id)
    except ObjectDoesNotExist:
        raise Http404
    n = NexusReader(language_tree.file.path)
    # Remove '[&R]' from newick string
    tree = re.sub(r'\[.*?\]', '', n.trees.trees[0])
    # Remove data before the =
    try:
        tree = tree[tree.index('=')+1:]
    except ValueError:
        tree = tree
    return Response(NewickTreeSerializer(NewickTree(tree)).data,)

@api_view(['GET'])
@permission_classes((AllowAny,))
def csv_download(request):
    # Ideally these would be handled by serializers, but we've already got logic to parse a query object
    query_string = request.QUERY_PARAMS['query']
    # Need to parse the JSON
    query_dict = json.loads(query_string)
    result_set = result_set_from_query_dict(query_dict)
    return Response(SocietyResultSetSerializer(result_set).data)
<commit_msg>Hook up CSV Renderer to csv_download<commit_after>import json
import re
from django.core.exceptions import ObjectDoesNotExist
from nexus import NexusReader
from rest_framework import viewsets
from serializers import *
from rest_framework.decorators import api_view, permission_classes, renderer_classes
from rest_framework.permissions import *
from rest_framework.views import Request, Response
from filters import *
from renderers import DPLACECsvRenderer

# Resource routes
class VariableDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableDescriptionSerializer
    filter_fields = ('label', 'name', 'index_categories', 'niche_categories',)
    queryset = VariableDescription.objects.all()
    # Override retrieve to use the detail serializer, which includes categories
    def retrieve(self, request, *args, **kwargs):
        self.object = self.get_object()
        serializer = VariableDescriptionDetailSerializer(self.object)
        return Response(serializer.data)

class VariableCategoryViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableCategorySerializer
    filter_fields = ('name', 'index_variables', 'niche_variables',)
    queryset = VariableCategory.objects.all()
    # Override retrieve to use the detail serializer, which includes variables
    def retrieve(self, request, *args, **kwargs):
        self.object = self.get_object()
        serializer = VariableCategoryDetailSerializer(self.object)
        return Response(serializer.data)

class VariableCodeDescriptionViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = VariableCodeDescriptionSerializer
    filter_class = VariableCodeDescriptionFilter
    queryset = VariableCodeDescription.objects.all()

# Can filter by code, code__variable, or society
class VariableCodedValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = VariableCodedValueSerializer
    filter_fields = ('variable','coded_value','code','society','code',)
    # Avoid additional database trips by select_related for foreign keys
    queryset = VariableCodedValue.objects.select_related('variable').select_related('code').all()

class SocietyViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = SocietySerializer
    queryset = Society.objects.all()

class ISOCodeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = ISOCodeSerializer
    filter_fields = ('iso_code',)
    queryset = ISOCode.objects.all()

class EnvironmentalVariableViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalVariableSerializer
    filter_fields = ('name', 'units',)
    queryset = EnvironmentalVariable.objects.all()

class EnvironmentalValueViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalValueSerializer
    filter_fields = ('variable','environmental',)
    queryset = EnvironmentalValue.objects.all()

class EnvironmentalViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = EnvironmentalSerializer
    filter_fields = ('society', 'iso_code',)
    queryset = Environmental.objects.all()

class LanguageClassViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = LanguageClassSerializer
    filter_class = LanguageClassFilter
    queryset = LanguageClass.objects.all()

# Need an API to get classifications / languages for a class

class LanguageClassificationViewSet(viewsets.ReadOnlyModelViewSet):
    # Model ordering is ignored when filter_fields enabled, requires FilterSet subclass
    # see https://github.com/tomchristie/django-rest-framework/issues/1432
    serializer_class = LanguageClassificationSerializer
    filter_class = LanguageClassificationFilter
    queryset = LanguageClassification.objects.all()

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageSerializer
    filter_fields = ('name', 'iso_code', 'society',)
    queryset = Language.objects.all()

class LanguageTreeViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = LanguageTreeSerializer
    filter_fields = ('name',)
    queryset = LanguageTree.objects.all()

def result_set_from_query_dict(query_dict):
    result_set = SocietyResultSet()
    # Criteria keeps track of what types of data were searched on, so that we can
    # AND them together
    criteria = []

    if 'language_filters' in query_dict:
        criteria.append(SEARCH_LANGUAGE)
        language_filters = query_dict['language_filters']
        for filter in language_filters:
            language_ids = [int(x) for x in filter['language_ids']]
            languages = Language.objects.filter(pk__in=language_ids) # Returns a queryset
            languages.select_related('societies')
            for language in languages:
                for society in language.societies.all():
                    result_set.add_language(society,language)

    if 'variable_codes' in query_dict:
        criteria.append(SEARCH_VARIABLES)
        # Now get the societies from variables
        variable_code_ids = [int(x) for x in query_dict['variable_codes']]
        codes = VariableCodeDescription.objects.filter(pk__in=variable_code_ids) # returns a queryset
        coded_value_ids = []
        # Aggregate all the coded values for each selected code
        for code in codes:
            coded_value_ids += code.variablecodedvalue_set.values_list('id', flat=True)
        # Coded values have a FK to society.  Aggregate the societies from each value
        values = VariableCodedValue.objects.filter(id__in=coded_value_ids)
        values = values.select_related('society','variable')
        for value in values:
            result_set.add_cultural(value.society,value.variable,value)

    if 'environmental_filters' in query_dict:
        criteria.append(SEARCH_ENVIRONMENTAL)
        environmental_filters = query_dict['environmental_filters']
        # There can be multiple filters, so we must aggregate the results.
        for filter in environmental_filters:
            values = EnvironmentalValue.objects.filter(variable=filter['id'])
            operator = filter['operator']
            if operator == 'inrange':
                values = values.filter(value__gt=filter['params'][0]).filter(value__lt=filter['params'][1])
            elif operator == 'outrange':
                values = values.filter(value__gt=filter['params'][1]).filter(value__lt=filter['params'][0])
            elif operator == 'gt':
                values = values.filter(value__gt=filter['params'][0])
            elif operator == 'lt':
                values = values.filter(value__lt=filter['params'][0])
            values = values.select_related('variable','environmental__society')
            # get the societies from the values
            for value in values:
                result_set.add_environmental(value.society(), value.variable, value)
    if 'geographic_regions' in query_dict:
        criteria.append(SEARCH_GEOGRAPHIC)
        geographic_region_ids = [int(x) for x in query_dict['geographic_regions']]
        regions = GeographicRegion.objects.filter(pk__in=geographic_region_ids) # returns a queryset
        for region in regions:
            for society in Society.objects.filter(location__intersects=region.geom):
                result_set.add_geographic_region(society, region)
    # Filter the results to those that matched all criteria
    result_set.finalize(criteria)
    return result_set

# search/filter APIs
@api_view(['POST'])
@permission_classes((AllowAny,))
def find_societies(request):
    """
    View to find the societies that match an input request.  Currently expects
    { language_filters: [{language_ids: [1,2,3]}], variable_codes: [4,5,6...],
    environmental_filters: [{id: 1, operator: 'gt', params: [0.0]}, {id:3, operator 'inrange', params: [10.0,20.0] }] }

    Returns serialized collection of SocietyResult objects
    """
    result_set = result_set_from_query_dict(request.DATA)
    return Response(SocietyResultSetSerializer(result_set).data)

class GeographicRegionViewSet(viewsets.ReadOnlyModelViewSet):
    serializer_class = GeographicRegionSerializer
    model = GeographicRegion
    filter_class = GeographicRegionFilter

@api_view(['POST'])
@permission_classes((AllowAny,))
def trees_from_languages(request):
    if 'language_ids' in request.DATA:
        language_ids = [int(x) for x in request.DATA['language_ids']]
        trees = LanguageTree.objects.filter(languages__pk__in=language_ids).distinct()
    else:
        trees = None
    return Response(LanguageTreeSerializer(trees, many=True).data,)

@api_view(['GET'])
@permission_classes((AllowAny,))
def newick_tree(request, *args, **kwargs):
    # Get a newick format tree from a language tree id
    language_tree_id = kwargs['pk']
    try:
        language_tree = LanguageTree.objects.get(pk=language_tree_id)
    except ObjectDoesNotExist:
        raise Http404
    n = NexusReader(language_tree.file.path)
    # Remove '[&R]' from newick string
    tree = re.sub(r'\[.*?\]', '', n.trees.trees[0])
    # Remove data before the =
    try:
        tree = tree[tree.index('=')+1:]
    except ValueError:
        tree = tree
    return Response(NewickTreeSerializer(NewickTree(tree)).data,)

@api_view(['GET'])
@permission_classes((AllowAny,))
@renderer_classes((DPLACECsvRenderer,))
def csv_download(request):
    # Ideally these would be handled by serializers, but we've already got logic to parse a query object
    query_string = request.QUERY_PARAMS['query']
    # Need to parse the JSON
    query_dict = json.loads(query_string)
    result_set = result_set_from_query_dict(query_dict)
    return Response(SocietyResultSetSerializer(result_set).data)
<|endoftext|>


<commit_before>import cPickle as pickle
from itertools import chain

import pymongo
from ming.orm import state
from pylons import c

from flyway import Migration
from allura.model import Thread, AppConfig, ArtifactReference, ProjectRole
from forgetracker.model import Ticket, Globals

class TrackerMigration(Migration):
    def __init__(self, *args, **kwargs):
        super(TrackerMigration, self).__init__(*args, **kwargs)
        try:
            c.project
        except (TypeError, AttributeError), exc:
            class EmptyClass(): pass
            c._push_object(EmptyClass())
            c.project = EmptyClass()
            c.project._id = None
            c.app = EmptyClass()
            c.app.config = EmptyClass()
            c.app.config.options = EmptyClass()
            c.app.config.options.mount_point = None

class V0(TrackerMigration):
    '''Migrate Thread.artifact_id to Thread.artifact_reference'''
    version = 0

    def up(self):
        for pg in self.ormsession.find(Ticket):
            q1 = self.ormsession.find(Thread, dict(artifact_id=pg._id))
            q2 = self.ormsession.find(Thread, {'artifact_reference.artifact_id':pg._id})
            for t in chain(q1, q2):
                t.artifact_reference = self._dump_ref(pg)
                t.artifact_id = None
                self.ormsession.update_now(t, state(t))
        self.ormsession.flush()

    def down(self):
        for pg in self.ormsession.find(Ticket):
            for t in self.ormsession.find(Thread, dict(artifact_reference=self._dump_ref(pg))):
                t.artifact_id = pg._id
                t.artifact_reference = None
                self.ormsession.update_now(t, state(t))
        self.ormsession.flush()

    def _dump_ref(self, art):
        app_config = self.ormsession.get(AppConfig, art.app_config_id)
        return ArtifactReference(dict(
            project_id=app_config.project_id,
            mount_point=app_config.options.mount_point,
            artifact_type=pymongo.bson.Binary(pickle.dumps(art.__class__)),
            artifact_id=art._id))

class AddShowInSearchAttributeToAllCustomFields(TrackerMigration):
    version = 1

    def up(self):
        for custom_field in self.each_custom_field():
            custom_field['show_in_search'] = False
        self.ormsession.flush()

    def down(self):
        for custom_field in self.each_custom_field():
            del custom_field['show_in_search']
        self.ormsession.flush()

    def each_custom_field(self):
        for tracker_globals in self.ormsession.find(Globals):
            for custom_field in tracker_globals.custom_fields:
                yield custom_field

class AddSaveSearchesPermission(TrackerMigration):
    version = 2

    def up(self):
        first_ticket = self.ormsession.find(Ticket).first()
        if first_ticket:
            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
            app_config.acl.save_searches = []
            developer = self.ormsession.find(ProjectRole, dict(name='Developer')).first()
            admin = self.ormsession.find(ProjectRole, dict(name='Admin')).first()
            if developer:
                app_config.acl.save_searches.append(developer._id)
            if admin:
                app_config.acl.save_searches.append(admin._id)
            self.ormsession.flush()

    def down(self):
        first_ticket = self.ormsession.find(Ticket).first()
        if first_ticket:
            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
            del app_config.acl.save_searches
            self.ormsession.flush()

class SplitStatusNamesIntoOpenAndClosed(TrackerMigration):
    version = 3

    def up(self):
        for tracker_globals in self.ormsession.find(Globals):
            old_names = tracker_globals.status_names
            tracker_globals.open_status_names = ' '.join([name for name in old_names.split(' ') if name and name != 'closed'])
            tracker_globals.closed_status_names = 'closed'
            tracker_globals.status_names = ''
        self.ormsession.flush()
        self.ormsession.clear()

    def down(self):
        for tracker_globals in self.ormsession.find(Globals):
            tracker_globals.status_names = ' '.join([tracker_globals.open_status_names, tracker_globals.closed_status_names])
            tracker_globals.open_status_names = ''
            tracker_globals.closed_status_names = ''
        self.ormsession.flush()
        self.ormsession.clear()
<commit_msg>[#851] - Add migration of existing milestones to a 'milestone' custom field<commit_after>import cPickle as pickle
from itertools import chain

import pymongo
from ming.orm import state
from pylons import c

from flyway import Migration
from allura.model import Thread, AppConfig, ArtifactReference, ProjectRole
from forgetracker.model import Ticket, Globals

class TrackerMigration(Migration):
    def __init__(self, *args, **kwargs):
        super(TrackerMigration, self).__init__(*args, **kwargs)
        try:
            c.project
        except (TypeError, AttributeError), exc:
            class EmptyClass(): pass
            c._push_object(EmptyClass())
            c.project = EmptyClass()
            c.project._id = None
            c.app = EmptyClass()
            c.app.config = EmptyClass()
            c.app.config.options = EmptyClass()
            c.app.config.options.mount_point = None

class V0(TrackerMigration):
    '''Migrate Thread.artifact_id to Thread.artifact_reference'''
    version = 0

    def up(self):
        for pg in self.ormsession.find(Ticket):
            q1 = self.ormsession.find(Thread, dict(artifact_id=pg._id))
            q2 = self.ormsession.find(Thread, {'artifact_reference.artifact_id':pg._id})
            for t in chain(q1, q2):
                t.artifact_reference = self._dump_ref(pg)
                t.artifact_id = None
                self.ormsession.update_now(t, state(t))
        self.ormsession.flush()

    def down(self):
        for pg in self.ormsession.find(Ticket):
            for t in self.ormsession.find(Thread, dict(artifact_reference=self._dump_ref(pg))):
                t.artifact_id = pg._id
                t.artifact_reference = None
                self.ormsession.update_now(t, state(t))
        self.ormsession.flush()

    def _dump_ref(self, art):
        app_config = self.ormsession.get(AppConfig, art.app_config_id)
        return ArtifactReference(dict(
            project_id=app_config.project_id,
            mount_point=app_config.options.mount_point,
            artifact_type=pymongo.bson.Binary(pickle.dumps(art.__class__)),
            artifact_id=art._id))

class AddShowInSearchAttributeToAllCustomFields(TrackerMigration):
    version = 1

    def up(self):
        for custom_field in self.each_custom_field():
            custom_field['show_in_search'] = False
        self.ormsession.flush()

    def down(self):
        for custom_field in self.each_custom_field():
            del custom_field['show_in_search']
        self.ormsession.flush()

    def each_custom_field(self):
        for tracker_globals in self.ormsession.find(Globals):
            for custom_field in tracker_globals.custom_fields:
                yield custom_field

class AddSaveSearchesPermission(TrackerMigration):
    version = 2

    def up(self):
        first_ticket = self.ormsession.find(Ticket).first()
        if first_ticket:
            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
            app_config.acl.save_searches = []
            developer = self.ormsession.find(ProjectRole, dict(name='Developer')).first()
            admin = self.ormsession.find(ProjectRole, dict(name='Admin')).first()
            if developer:
                app_config.acl.save_searches.append(developer._id)
            if admin:
                app_config.acl.save_searches.append(admin._id)
            self.ormsession.flush()

    def down(self):
        first_ticket = self.ormsession.find(Ticket).first()
        if first_ticket:
            app_config = self.ormsession.get(AppConfig, first_ticket.app_config_id)
            del app_config.acl.save_searches
            self.ormsession.flush()

class SplitStatusNamesIntoOpenAndClosed(TrackerMigration):
    version = 3

    def up(self):
        for tracker_globals in self.ormsession.find(Globals):
            old_names = tracker_globals.status_names
            tracker_globals.open_status_names = ' '.join([name for name in old_names.split(' ') if name and name != 'closed'])
            tracker_globals.closed_status_names = 'closed'
            tracker_globals.status_names = ''
        self.ormsession.flush()
        self.ormsession.clear()

    def down(self):
        for tracker_globals in self.ormsession.find(Globals):
            tracker_globals.status_names = ' '.join([tracker_globals.open_status_names, tracker_globals.closed_status_names])
            tracker_globals.open_status_names = ''
            tracker_globals.closed_status_names = ''
        self.ormsession.flush()
        self.ormsession.clear()

class MoveMilestonesToCustom(TrackerMigration):
    version = 4

    def _custom_field(self, tracker_globals):
        return dict(
            type='milestone',
            label='Milestone',
            milestones=[
                dict(name=name, complete=False, due_date=None)
                for name in tracker_globals.milestone_names.split() ])

    def up(self):
        for tracker_globals in self.ormsession.find(Globals):
            fld = self._custom_field(tracker_globals)
            tracker_globals.custom_fields.append(fld)
        self.ormsession.flush()
        self.ormsession.clear()

    def down(self):
        for tracker_globals in self.ormsession.find(Globals):
            fld = self._custom_field(tracker_globals)
            if tracker_globals.custom_fields[-1] == fld:
                tracker_globals.custom_fields.pop()
        self.ormsession.flush()
        self.ormsession.clear()
<|endoftext|>


<commit_before>import numpy as np
import math
import transform
from osgeo import gdal, osr

class UTMCoord(object):
	'''
	Represents a UTM coordinate. All UTM Coordinates should be expressed in this form.
	'''
	def __init__(self, easting, northing, zone, zoneLetter = 'T'):
		self.easting = easting
		self.northing = northing
		self.zone = zone
		self.zoneLetter = zoneLetter # Note: 'T' is the zone letter of all but the northernmost point of Idaho
									 # 'Q' is the zone letter of Hawaii

	def __str__(self):
		return "UTM Coordinate "+str(self.easting)+"E,"+str(self.northing)+"N, zone"+str(self.zone)+self.zoneLetter+'\n'

class LatLongCoord(object):
	'''
	Represents a latitude and longitude pair. All Lat/Long Coordinates should be expressed in this form.
	'''
	def __init__(self, lat, long):
		self.latitude = lat
		self.longitude = long

	def __str__(self):
		return "LatLong Coordinate "+str(self.latitude)+','+str(self.longitude)+'\n'

	def latLongList(self):
		return [self.latitude, self.longitude]

	def longLatList(self):
		return [self.longitude, self.latitude]

class EnvironmentalModel(object):
	'''
	This class ultimately represents an elevation map + all of the traversable spots on it.

	Public functions:

	setMaxSlope(slope) - sets the maximum slope that can be traversed
	setObstacle(coordinates), eraseObstacle(coordinates) - set or erase an obstacle at certain coordinates
	getElevation(coordinates), getSlope(coordinates) - get Elevation or Slope at certain coordinates
	isPassable(coordinates) - determines if a certain coordinate can be traversed
	convertToRowCol(coordinates), convertToUTM(coordinates), convertToLatLong(coordinates) - converts from one coordinate system to another
	loadElevationMap(fileName) - load an elevation map from a geoTIFF or text file
	'''
	def __init__(self, elevation_map, resolution, maxSlope, NW_Coord, planet = "Earth", uuid = None):
		self.elevations = elevation_map #this is a numpy 2D array
		self.resolution = float(resolution) #this is just a float
		# [gx, gy] = np.gradient(elevation_map, resolution, resolution)
		# self.slopes = np.degrees(np.arctan(np.sqrt(np.add(np.square(gx),np.square(gy))))) # I think this syntax is correct
																						  # we want atan(sqrt(gx^2+gy^2)) in degrees
		self.slopes = np.degrees(np.arctan(np.sqrt(np.add(np.square(np.gradient(elevation_map, resolution, resolution)[0])\
					,np.square(np.gradient(elevation_map, resolution, resolution)[1]))))) # Combining for now for less RAM usage
		self.obstacles = self.slopes <= maxSlope # obstacles is basically an "isPassable" function
		self.planet = planet
		self.NW_UTM = self.convertToUTM(NW_Coord) # a UTMCoord object, default set to Boston
		self.special_obstacles = set() # a list of coordinates of obstacles are not identified by the slope
		self.UUID = uuid

	def getGravity(self):
		if self.planet == 'Earth':
			return 9.81
		elif self.planet == 'Moon':
			return 1.622

	def setMaxSlope(self, maxSlope):
		self.obstacles = self.slopes <= maxSlope
		for obstacle in self.special_obstacles:
			self.setObstacle(obstacle) # "special obstacles" still should not be traversable

	def setObstacle(self, coordinates):
		row, col = self.convertToRowCol(coordinates) # of the form (row, column)
		self.obstacles[row][col] = False
		if coordinates not in self.special_obstacles:
			self.special_obstacles.add(coordinates)

	def eraseObstacle(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		self.obstacles[row][col] = True
		if coordinates in self.special_obstacles:
			self.special_obstacles.remove(coordinates)

	def getElevation(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		return self.elevations[row][col]

	def getSlope(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		return self.slopes[row][col]

	def setNoVal(self, noValResult):
		for i, row in enumerate(self.elevations):
			for j, item in enumerate(row):
				if item == noValResult:
					self.elevations[i][j] = None
					self.slopes[i][j] = None
					self.obstacles[i][j] = True

	def _inBounds(self, coordinates):
		# determines if a state is within the boundaries of the environmental model
		# a state is a tuple of the form (row, column)
		row = coordinates[0]
		col = coordinates[1]
		return (row in range(int(np.shape(self.elevations)[0]))) and (col in range(int(np.shape(self.elevations)[1])))

	def isPassable(self, coordinates):
		# determines if coordinates can be passed through
		row, col = self.convertToRowCol(coordinates) # coordinates is a tuple
		if not self._inBounds(coordinates):
			return False
		else:
			return self.obstacles[row][col]

	def _UTMtoRowCol(self, UTM):
		'''
		Converts from UTMCoord object to a tuple representing the row and the column
		in the matrix. There may be minor rounding issues

		As of right now both UTMtoRowCol and RowColToUTM only work if we do not cross a UTM line.
		Eventually for completeness sake this should probably be changed
		'''
		if UTM.zone == self.NW_UTM.zone:
			row = round((self.NW_UTM.northing - UTM.northing)/self.resolution)
			col = round((UTM.easting - self.NW_UTM.easting)/self.resolution)
			return row, col
		else:
			print "WARNING: map is in UTM zone " + repr(self.NW_UTM.zone)
			print "Zone given: " + repr(UTM.zone)
			pass # Later I could figure out how to account for zone changes just in case a site is
				 # just on the border of a UTM zone.

	def _rowColToUTM(self, coordinates):
		'''
		The reverse of the above function.
		'''
		row, col = coordinates

		xNW, yNW = self.NW_UTM.easting, self.NW_UTM.northing
		x = xNW + col * self.resolution
		y = yNW - row * self.resolution
		return UTMCoord(x, y, self.NW_UTM.zone, self.NW_UTM.zoneLetter)

	def convertToRowCol(self, position):
		"""
		This should be called before every function in EnvironmentalModel. Converts UTMCoord or LatLongCoord
		objects into a tuple representing the row and the column of the environmental map closest to that location.
		"""
		if type(position) is UTMCoord:
			return self._UTMtoRowCol(position)
		elif type(position) is LatLongCoord:
			easting, northing, zoneNumber, zoneLetter = transform.latLongToUTM(position)
			return self._UTMtoRowCol(UTMCoord(easting, northing, zoneNumber, zoneLetter))
		elif type(position) is tuple:
			return position
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

	def convertToLatLong(self, position):
		'''
		Converts a row/column or UTM coordinate into a LatLongCoord
		'''
		if type(position) is UTMCoord:
			lat, lon = transform.UTMToLatLong(position)
			return LatLongCoord(lat, lon)
		elif type(position) is LatLongCoord:
			return position
		elif type(position) is tuple:
			UTM = self._rowColToUTM(position)
			lat, lon = transform.UTMToLatLong(UTM)
			return LatLongCoord(lat, lon)
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

	def convertToUTM(self, position):
		'''
		Converts a row/column coordinate or LatLongCoord into a UTMCoord.
		'''
		if type(position) is UTMCoord:
			return position
		elif type(position) is LatLongCoord:
			easting, northing, zoneNumber, zoneLetter = transform.latLongToUTM(position)
			return UTMCoord(easting, northing, zoneNumber, zoneLetter)
		elif type(position) is tuple:
			return self._rowColToUTM(position)
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

def loadElevationMap(filePath, maxSlope = 15, planet = 'Earth', NWCorner = None, SECorner = None, desiredRes = None, no_val = -10000, zone=None, zoneLetter=None):
	'''
	Creates a EnvironmentalModel object from either a geoTiff file or a text file.

	Issue: computer sometimes freezes whenever loading a very large geoTIFF file (1GB+)

	Current modification: trying to enable loading a square sector of a geoTIFF file.
	2 new optional inputs: NWCorner and SE corner - UTM or LatLong coordinates of the subregion
	we would like to analyze.

	Using the parameter desiredRes doesn't work very well if there are a significant number of
	"unknown" points (usually denoted by a placeholder like -10000).
	'''
	# filePath is a string representing the location of the file
	extension = filePath.split('.')[-1] # this should be the file extension

	if extension == 'txt': # This likely needs some updating though it probably won't be used anytime soon
		f = open(filePath, r)
		inputs = [] #the values that end up in inputs will become numCols, numRows, xllcorner, yllcorner, cellsize, and NODATA_value
		for i in range(6):
			inputs.append(f.readline().split(' ')[-1]) # this should work given the current format
		numCols = inputs[0]
		numRows = inputs[1]
		mapArray = np.empty([numRows, numCols]) # initializes an empty array
		for i in range(numRows):
			x = f.readline().split(' ')
			if len(x) == numCols:
				mapArray[i] = x
			else:
				print "ERROR: expected " + str(numCols) + " columns. Got " + str(len(x)) + " columns"
				return 0
		return EnvironmentalModel(mapArray, inputs[4], maxSlope, planet)
	elif extension == 'tif':
		# NOTE: Currently, SEXTANT only supports geoTIFF files that use the UTM projection and have "north up"
		gdal.UseExceptions()

		# copied from stackexchange
		dataset = gdal.Open(filePath)
		band = dataset.GetRasterBand(1)
		proj = dataset.GetProjection()

		srs = osr.SpatialReference(wkt=proj)
		projcs = srs.GetAttrValue('projcs') # This will be a string that looks something like
											# "NAD83 / UTM zone 5N"...hopefully

		if (projcs and (zone == None) and zoneLetter == None): # projcs is not None for the government Hawaii data
			zone = int(projcs.split(' ')[-1][0:-1])
			zoneLetter = projcs.split(' ')[-1][-1]

		datasetInfo = dataset.GetGeoTransform()
		# returns a list of length 6. Indices 0 and 3 are the easting and northing values of the upper left corner.
		# Indices 1 and 5 are the w-e and n-s pixel resolutions, index 5 is always negative. Indicies 2 and 4 are
		# set to zero for all maps pointing in a "North up" type projection; for now we will only be using maps where
		# North is set to up.
		NWeasting = datasetInfo[0]
		NWnorthing = datasetInfo[3]
		bufx = None
		bufy = None
		if datasetInfo[1] == -datasetInfo[5]: # SEXTANT does not support maps where the x-resolution differs from the y-resolution at the moment
			if desiredRes:
				resolution = desiredRes
				bufx = round(dataset.RasterXSize * datasetInfo[1] / resolution)
				bufy = round(dataset.RasterYSize * datasetInfo[1] / resolution)
			else:
				resolution = datasetInfo[1]
				bufx = None
				bufy = None

		NWCoord = UTMCoord(NWeasting, NWnorthing, zone, zoneLetter)

		if NWCorner == None and SECorner == None: #No NW and SE corner implies we want the entire map
			mapArray = band.ReadAsArray(buf_xsize = bufx, buf_ysize = bufy) #converts from a raster band to a numpy array
			return EnvironmentalModel(mapArray, resolution, maxSlope, planet, NWCoord)
		else:
			top = convertToUTM(NWCorner.northing)
			bot = convertToUTM(SECorner.northing)
			left = convertToUTM(NWCorner.easting)
			right = convertToUTM(SECorner.easting)

			if bot > top or left > right:
				print "ERROR with NWCorner and SECorner"
				print "NWCorner: " + str(NWCorner) + " SWCorner: " + str(SECorner)
				return 0

			if left < NWeasting:
				left = NWeasting
			if right > NWeasting + datasetInfo[1] * (dataset.RasterXSize - 1):
				right = NWeasting + datasetInfo[1] * (dataset.RasterXSize - 1)
			if top > NWnorthing:
				top = NWnorthing
			if bot < NWnorthing + datasetInfo[5] * (dataset.RasterYSize - 1):
				bot = NWnorthing + datasetInfo[5] * (dataset.RasterYSize - 1)

			x_offset = int((left - NWeasting)/resolution)
			x_size = int((right - NWeasting)/resolution) + 1 - x_offset
			y_offset = int((NWnorthing - top)/resolution)
			y_size = ((NWnorthing - top)/resolution) + 1 - y_offset

			mapArray = band.ReadAsArray(x_offset, y_offset, x_size, y_size, bufx, bufy).astype(np.float)
			return EnvironmentalModel(mapArray, resolution, maxSlope, planet, NWCoord)
	else:
		print "ERROR: expected txt or tif file. Received " + extension + " type file"
		return 0<commit_msg>call EnvironmentalModel constructor with parameters in the right order<commit_after>import numpy as np
import math
import transform
from osgeo import gdal, osr

class UTMCoord(object):
	'''
	Represents a UTM coordinate. All UTM Coordinates should be expressed in this form.
	'''
	def __init__(self, easting, northing, zone, zoneLetter = 'T'):
		self.easting = easting
		self.northing = northing
		self.zone = zone
		self.zoneLetter = zoneLetter # Note: 'T' is the zone letter of all but the northernmost point of Idaho
									 # 'Q' is the zone letter of Hawaii

	def __str__(self):
		return "UTM Coordinate "+str(self.easting)+"E,"+str(self.northing)+"N, zone"+str(self.zone)+self.zoneLetter+'\n'

class LatLongCoord(object):
	'''
	Represents a latitude and longitude pair. All Lat/Long Coordinates should be expressed in this form.
	'''
	def __init__(self, lat, long):
		self.latitude = lat
		self.longitude = long

	def __str__(self):
		return "LatLong Coordinate "+str(self.latitude)+','+str(self.longitude)+'\n'

	def latLongList(self):
		return [self.latitude, self.longitude]

	def longLatList(self):
		return [self.longitude, self.latitude]

class EnvironmentalModel(object):
	'''
	This class ultimately represents an elevation map + all of the traversable spots on it.

	Public functions:

	setMaxSlope(slope) - sets the maximum slope that can be traversed
	setObstacle(coordinates), eraseObstacle(coordinates) - set or erase an obstacle at certain coordinates
	getElevation(coordinates), getSlope(coordinates) - get Elevation or Slope at certain coordinates
	isPassable(coordinates) - determines if a certain coordinate can be traversed
	convertToRowCol(coordinates), convertToUTM(coordinates), convertToLatLong(coordinates) - converts from one coordinate system to another
	loadElevationMap(fileName) - load an elevation map from a geoTIFF or text file
	'''
	def __init__(self, elevation_map, resolution, maxSlope, NW_Coord, planet = "Earth", uuid = None):
		self.elevations = elevation_map #this is a numpy 2D array
		self.resolution = float(resolution) #this is just a float
		# [gx, gy] = np.gradient(elevation_map, resolution, resolution)
		# self.slopes = np.degrees(np.arctan(np.sqrt(np.add(np.square(gx),np.square(gy))))) # I think this syntax is correct
																						  # we want atan(sqrt(gx^2+gy^2)) in degrees
		self.slopes = np.degrees(np.arctan(np.sqrt(np.add(np.square(np.gradient(elevation_map, resolution, resolution)[0])\
					,np.square(np.gradient(elevation_map, resolution, resolution)[1]))))) # Combining for now for less RAM usage
		self.obstacles = self.slopes <= maxSlope # obstacles is basically an "isPassable" function
		self.planet = planet
		self.NW_UTM = self.convertToUTM(NW_Coord) # a UTMCoord object, default set to Boston
		self.special_obstacles = set() # a list of coordinates of obstacles are not identified by the slope
		self.UUID = uuid

	def getGravity(self):
		if self.planet == 'Earth':
			return 9.81
		elif self.planet == 'Moon':
			return 1.622

	def setMaxSlope(self, maxSlope):
		self.obstacles = self.slopes <= maxSlope
		for obstacle in self.special_obstacles:
			self.setObstacle(obstacle) # "special obstacles" still should not be traversable

	def setObstacle(self, coordinates):
		row, col = self.convertToRowCol(coordinates) # of the form (row, column)
		self.obstacles[row][col] = False
		if coordinates not in self.special_obstacles:
			self.special_obstacles.add(coordinates)

	def eraseObstacle(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		self.obstacles[row][col] = True
		if coordinates in self.special_obstacles:
			self.special_obstacles.remove(coordinates)

	def getElevation(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		return self.elevations[row][col]

	def getSlope(self, coordinates):
		row, col = self.convertToRowCol(coordinates)
		return self.slopes[row][col]

	def setNoVal(self, noValResult):
		for i, row in enumerate(self.elevations):
			for j, item in enumerate(row):
				if item == noValResult:
					self.elevations[i][j] = None
					self.slopes[i][j] = None
					self.obstacles[i][j] = True

	def _inBounds(self, coordinates):
		# determines if a state is within the boundaries of the environmental model
		# a state is a tuple of the form (row, column)
		row = coordinates[0]
		col = coordinates[1]
		return (row in range(int(np.shape(self.elevations)[0]))) and (col in range(int(np.shape(self.elevations)[1])))

	def isPassable(self, coordinates):
		# determines if coordinates can be passed through
		row, col = self.convertToRowCol(coordinates) # coordinates is a tuple
		if not self._inBounds(coordinates):
			return False
		else:
			return self.obstacles[row][col]

	def _UTMtoRowCol(self, UTM):
		'''
		Converts from UTMCoord object to a tuple representing the row and the column
		in the matrix. There may be minor rounding issues

		As of right now both UTMtoRowCol and RowColToUTM only work if we do not cross a UTM line.
		Eventually for completeness sake this should probably be changed
		'''
		if UTM.zone == self.NW_UTM.zone:
			row = round((self.NW_UTM.northing - UTM.northing)/self.resolution)
			col = round((UTM.easting - self.NW_UTM.easting)/self.resolution)
			return row, col
		else:
			print "WARNING: map is in UTM zone " + repr(self.NW_UTM.zone)
			print "Zone given: " + repr(UTM.zone)
			pass # Later I could figure out how to account for zone changes just in case a site is
				 # just on the border of a UTM zone.

	def _rowColToUTM(self, coordinates):
		'''
		The reverse of the above function.
		'''
		row, col = coordinates

		xNW, yNW = self.NW_UTM.easting, self.NW_UTM.northing
		x = xNW + col * self.resolution
		y = yNW - row * self.resolution
		return UTMCoord(x, y, self.NW_UTM.zone, self.NW_UTM.zoneLetter)

	def convertToRowCol(self, position):
		"""
		This should be called before every function in EnvironmentalModel. Converts UTMCoord or LatLongCoord
		objects into a tuple representing the row and the column of the environmental map closest to that location.
		"""
		if type(position) is UTMCoord:
			return self._UTMtoRowCol(position)
		elif type(position) is LatLongCoord:
			easting, northing, zoneNumber, zoneLetter = transform.latLongToUTM(position)
			return self._UTMtoRowCol(UTMCoord(easting, northing, zoneNumber, zoneLetter))
		elif type(position) is tuple:
			return position
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

	def convertToLatLong(self, position):
		'''
		Converts a row/column or UTM coordinate into a LatLongCoord
		'''
		if type(position) is UTMCoord:
			lat, lon = transform.UTMToLatLong(position)
			return LatLongCoord(lat, lon)
		elif type(position) is LatLongCoord:
			return position
		elif type(position) is tuple:
			UTM = self._rowColToUTM(position)
			lat, lon = transform.UTMToLatLong(UTM)
			return LatLongCoord(lat, lon)
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

	def convertToUTM(self, position):
		'''
		Converts a row/column coordinate or LatLongCoord into a UTMCoord.
		'''
		if type(position) is UTMCoord:
			return position
		elif type(position) is LatLongCoord:
			easting, northing, zoneNumber, zoneLetter = transform.latLongToUTM(position)
			return UTMCoord(easting, northing, zoneNumber, zoneLetter)
		elif type(position) is tuple:
			return self._rowColToUTM(position)
		else:
			print "ERROR: only accepts UTMCoord, LatLongCoord, and tuple objects"
			print "Received " + repr(type(position)) + " object"
			return 0

def loadElevationMap(filePath, maxSlope = 15, planet = 'Earth', NWCorner = None, SECorner = None, desiredRes = None, no_val = -10000, zone=None, zoneLetter=None):
	'''
	Creates a EnvironmentalModel object from either a geoTiff file or a text file.

	Issue: computer sometimes freezes whenever loading a very large geoTIFF file (1GB+)

	Current modification: trying to enable loading a square sector of a geoTIFF file.
	2 new optional inputs: NWCorner and SE corner - UTM or LatLong coordinates of the subregion
	we would like to analyze.

	Using the parameter desiredRes doesn't work very well if there are a significant number of
	"unknown" points (usually denoted by a placeholder like -10000).
	'''
	# filePath is a string representing the location of the file
	extension = filePath.split('.')[-1] # this should be the file extension

	if extension == 'txt': # This likely needs some updating though it probably won't be used anytime soon
		f = open(filePath, r)
		inputs = [] #the values that end up in inputs will become numCols, numRows, xllcorner, yllcorner, cellsize, and NODATA_value
		for i in range(6):
			inputs.append(f.readline().split(' ')[-1]) # this should work given the current format
		numCols = inputs[0]
		numRows = inputs[1]
		mapArray = np.empty([numRows, numCols]) # initializes an empty array
		for i in range(numRows):
			x = f.readline().split(' ')
			if len(x) == numCols:
				mapArray[i] = x
			else:
				print "ERROR: expected " + str(numCols) + " columns. Got " + str(len(x)) + " columns"
				return 0
		return EnvironmentalModel(mapArray, inputs[4], maxSlope=maxSlope, planet=planet)
	elif extension == 'tif':
		# NOTE: Currently, SEXTANT only supports geoTIFF files that use the UTM projection and have "north up"
		gdal.UseExceptions()

		# copied from stackexchange
		dataset = gdal.Open(filePath)
		band = dataset.GetRasterBand(1)
		proj = dataset.GetProjection()

		srs = osr.SpatialReference(wkt=proj)
		projcs = srs.GetAttrValue('projcs') # This will be a string that looks something like
											# "NAD83 / UTM zone 5N"...hopefully

		if (projcs and (zone == None) and zoneLetter == None): # projcs is not None for the government Hawaii data
			zone = int(projcs.split(' ')[-1][0:-1])
			zoneLetter = projcs.split(' ')[-1][-1]

		datasetInfo = dataset.GetGeoTransform()
		# returns a list of length 6. Indices 0 and 3 are the easting and northing values of the upper left corner.
		# Indices 1 and 5 are the w-e and n-s pixel resolutions, index 5 is always negative. Indicies 2 and 4 are
		# set to zero for all maps pointing in a "North up" type projection; for now we will only be using maps where
		# North is set to up.
		NWeasting = datasetInfo[0]
		NWnorthing = datasetInfo[3]
		bufx = None
		bufy = None
		if datasetInfo[1] == -datasetInfo[5]: # SEXTANT does not support maps where the x-resolution differs from the y-resolution at the moment
			if desiredRes:
				resolution = desiredRes
				bufx = round(dataset.RasterXSize * datasetInfo[1] / resolution)
				bufy = round(dataset.RasterYSize * datasetInfo[1] / resolution)
			else:
				resolution = datasetInfo[1]
				bufx = None
				bufy = None

		NWCoord = UTMCoord(NWeasting, NWnorthing, zone, zoneLetter)

		if NWCorner == None and SECorner == None: #No NW and SE corner implies we want the entire map
			mapArray = band.ReadAsArray(buf_xsize = bufx, buf_ysize = bufy) #converts from a raster band to a numpy array
			return EnvironmentalModel(mapArray, resolution, maxSlope, NWCoord, planet)
		else:
			#TODO Kevin you have to call convertToUTM from within EnvionmentalModel, which you don't have here.
			# Alternately make it a standalone function
			top = convertToUTM(NWCorner.northing)
			bot = convertToUTM(SECorner.northing)
			left = convertToUTM(NWCorner.easting)
			right = convertToUTM(SECorner.easting)

			if bot > top or left > right:
				print "ERROR with NWCorner and SECorner"
				print "NWCorner: " + str(NWCorner) + " SWCorner: " + str(SECorner)
				return 0

			if left < NWeasting:
				left = NWeasting
			if right > NWeasting + datasetInfo[1] * (dataset.RasterXSize - 1):
				right = NWeasting + datasetInfo[1] * (dataset.RasterXSize - 1)
			if top > NWnorthing:
				top = NWnorthing
			if bot < NWnorthing + datasetInfo[5] * (dataset.RasterYSize - 1):
				bot = NWnorthing + datasetInfo[5] * (dataset.RasterYSize - 1)

			x_offset = int((left - NWeasting)/resolution)
			x_size = int((right - NWeasting)/resolution) + 1 - x_offset
			y_offset = int((NWnorthing - top)/resolution)
			y_size = ((NWnorthing - top)/resolution) + 1 - y_offset

			mapArray = band.ReadAsArray(x_offset, y_offset, x_size, y_size, bufx, bufy).astype(np.float)
			return EnvironmentalModel(mapArray, resolution, maxSlope, NWCoord, planet)
	else:
		print "ERROR: expected txt or tif file. Received " + extension + " type file"
		return 0<|endoftext|>