<commit_before>import random
import os
import datetime
import re
import tempfile
from nc_mlpa import settings
from grass import Grass
import utilities as mmutil
from models import *
from nc_mlpa.mlpa.models import *
#from Layers import *
from Layers import Layers

'''
Used to calculate summary statistics of a given mpa's impact on localized fishing
interests.  Uses grass to perform the calculations on Layers.  Much of the overall
and study region level statistics are precalculated and stored in the database
using the FishingImpactStats model.  Mpa level statistics are calculated at run-time.
'''
class Analysis:
    def __init__(self):
        self.SQ_MILES_IN_SQ_METER = 0.000000386102159
        self.SIG_DIGITS = 3
        #location under GRASS_ROOT to store grass mapsets
        self.MM_GRASS_LOCATION = 'mlpa_nc'
        #mapset under location for grass map layers
        self.MM_SRC_GRASS_MAPSET = 'PERMANENT'
        # Source for fishing impact layers used by Grass for analysis.
        self.FISHING_IMPACT_ANALYSIS_ROOT = settings.GIS_DATA_ROOT + 'analysis/'

        #Temporary names for intermediate grass analysis.  Can be reused each run.
        self.srMapName = 'studyRegion'
        self.mpaVectorMapName = 'mpaVectorMap'          # mpa vector
        self.mpaRasterMapName = 'mpaRasterMap'          # mpa raster
        self.srValueMaskMapName = 'srValueMaskMap'      # fishing value within study region
        self.mpaValueMaskMapName = 'mpaValueMaskMap'    # fishing value within mpa

        self.grass = None

    def setupGrass(self, temp_id=None):
        self.MM_DST_GRASS_MAPSET = temp_id
        grass = Grass(settings.GRASS_ROOT, self.MM_GRASS_LOCATION, self.MM_SRC_GRASS_MAPSET, self.MM_DST_GRASS_MAPSET)
        self.MM_GRASS_RAST_PATH = grass.getRastPath()
        return grass

    '''
    Fishing impact analysis driver.
    '''
    def run(self, mpa, layers):
        analResults = []
        for layer in layers:
            analResult = self.__runAnal(mpa, layer)
            analResults.append(analResult)
        return analResults

    '''
    Driver for preloading and precalculating grass map layers and
    overall/study region summary statistics.  This is invoked via
    the manage.py command defined in the management directory of
    this module
    '''
    def preload(self):
        #Setup grass to use the base mapset, we're going to load it
        self.grass = self.setupGrass()
        self.__preloadStudyRegion()
        layers_list = GetAllLayers()
        for layer in layers_list:
            map = self.__preloadDb(layer)
            self.__preloadGrassMap(map)
            self.__preloadMapStats(map)

    '''
    Preload information about fishing impact map into DB
    '''
    def __preloadDb(self, layer):
        #Check for existing record
        maps = FishingImpactAnalysisMap.objects.filter(
            group_name=layer.group_name,
            port_name=layer.port_name,
            species_name=layer.species_name
        )

        #If record already exists, update it
        if len(maps) > 0:
            map = maps[0]
            map.group_name = layer.group_name
            map.group_abbr = layer.group_abbr
            map.port_name = layer.port_name
            map.port_abbr = layer.port_abbr
            map.species_name = layer.species_name
            map.species_abbr = layer.species_abbr
            map.cell_size = layer.cell_size
            map.save()
        #Otherwise create a new record
        else:
            map = FishingImpactAnalysisMap(
                group_name=layer.group_name,
                group_abbr=layer.group_abbr,
                port_name=layer.port_name,
                port_abbr=layer.port_abbr,
                species_name=layer.species_name,
                species_abbr=layer.species_abbr,
                cell_size=layer.cell_size
            )
            map.save()

        #Add Allowed Targets to AnalysisMap
        targets = layer.target_names
        for target in targets:
            allowed_target = AllowedTarget.objects.filter(name=target)
            if len(allowed_target) > 0:
                map.allowed_targets.add(allowed_target[0])
        map.save()

        #Map the new AnalysisMap to one or more allowed uses
        #Filter by fishing type
        allowed_uses = AllowedUse.objects.filter(
            purpose__name=layer.fishing_type
        )

        #Get methods for current species
        methods = layer.take_methods
        species_name = layer.species_name

        #Add Allowed Uses to AnalysisMap
        for method_name in methods:
            uses = allowed_uses.filter(method__name=method_name, target__name=species_name)
            for use in uses:
                map.allowed_uses.add(use)
            map.save()

        return map

    '''
    Load study region from DB into grass.
    '''
    def __preloadStudyRegion(self):
        #Output study region to shapefile
        srPath = self.__srToShapefile()
        #Convert shapefile to grass vector map
        self.grass.v_in_ogr(srPath, self.srMapName)
        #Convert study vector map to raster map
        self.grass.v_to_r(self.srMapName, self.srMapName, 1)

    '''
    Preload a fishing impact map into grass for later use in analysis
    '''
    def __preloadGrassMap(self, map):
        #Build path to fishing impact grids
        mapPath = self.FISHING_IMPACT_ANALYSIS_ROOT+map.group_abbr+'/'
        #Get name of current grid
        mapName = map.getFullName()
        #Getpath to grass raster map
        grassRasterPath = os.path.join(self.MM_GRASS_RAST_PATH,mapName)
        #Load fishing impact grid into Grass for later use
        if not os.path.exists(grassRasterPath):
            self.grass.r_in_gdal(mapPath+map.getGridName(), mapName)

    '''
    Precalculates overall and study region level statistics and stores them in the database.
    '''
    def __preloadMapStats(self, map):
        self.fishingMapName = map.getFullName()       # fishing value map

        #Total fishing value.  Sums cell values in fishing map
        totalValue = self.grass.r_sum(self.fishingMapName)
        if totalValue is None:
            return None

        #Fishing value in study region.  Intersects fishing map with study regions and
        #sums remaining cells.
        self.grass.r_intersect(self.srValueMaskMapName, self.srMapName, self.fishingMapName)
        srValue = self.grass.r_sum(self.srValueMaskMapName)
        if srValue is None:
            return None

        # Total fishing area.  Makes all nonzero cells 1 and sums the values getting
        # the number of cells.  Use cell size to get area.
        (totalCells, totalArea) = self.grass.r_area(self.fishingMapName, map.cell_size)
        totalArea = totalArea * self.SQ_MILES_IN_SQ_METER

        # Fishing area in study area.  Uses previous fishing map intersected to study
        # region, makes all nonzero cells 1 and sums the values getting number of cells.
        # Use cell size to get area.
        (srCells, srArea) = self.grass.r_area(self.srValueMaskMapName, map.cell_size)
        srArea = srArea * self.SQ_MILES_IN_SQ_METER

        # Create a new stats record.  If one already exists, update it.
        stats = FishingImpactStats.objects.filter(map__group_name=map.group_name,
                                                  map__port_name=map.port_name,
                                                  map__species_name=map.species_name)

        #If record already exists, update it
        if len(stats) > 0:
            fi = stats[0]
            fi.totalCells=totalCells
            fi.srCells=srCells
            fi.totalArea=totalArea
            fi.srArea=srArea
            fi.totalValue=totalValue
            fi.srValue=srValue
            fi.save()
        #Otherwise create a new record
        else:
            fi = FishingImpactStats(map=map,
                                    totalCells=totalCells,
                                    srCells=srCells,
                                    totalArea=totalArea,
                                    srArea=srArea,
                                    totalValue=totalValue,
                                    srValue=srValue)
            fi.save()

    '''
    Performs a single grass analysis run.  Assumes that the source mapset has been
    preloaded with all of the necessary map layers for analysis.  Don't run this
    directly.  Use the analysis driver named 'run' and

    Returns -1 if failed to retrieve precomputed overall and study region statistics
    Returns -2 if mpa analysis failed.
    '''
    def __runAnal(self, mpa, map):
        self.fishingMapName = map.getFullName()       # fishing value map
        #self.fishingMapName = map.getGridName()
        timestamp = datetime.datetime.now().strftime('%m_%d_%y_%H%M')
        temp_id = 'mpa'+str(mpa.id)+'_user'+str(mpa.user_id)+'_'+timestamp+'_'+str(mmutil.getRandInt())
        self.tmpMapsetName = temp_id

        #Initialize grass, creating temporary mapset
        self.grass = self.setupGrass(self.tmpMapsetName)
        #Copy preloaded fishing map to temp mapset
        self.grass.copyMap('rast', self.fishingMapName)
        #Copy preloaded study region to temp mapset
        #self.grass.copyMap('vect', self.srMapName)

        #Output mpa to shapefile
        shapepath = self.__mpaToTempShapefile(mpa.id, temp_id)
        #Convert shapefile to grass vector map
        mpaVectorName = self.mpaVectorMapName+str(mpa.id)
        self.grass.v_in_ogr(shapepath, mpaVectorName)
        #Convert mpa vector map to raster map
        #######################################################################################
        #import time
        #time.sleep(5)
        #######################################################################################
        #pausing here for 5 seconds with time.sleep(5), seems to produce correct results on a consistent basis (on my windows machine)
        #could it be that v_in_ogr is not finishing otherwise?
        #spend some time trying to get grass.py runCmd to remove and redirect output from grass commands,
        #while producing accurate statistical results, and eventually gave up, hoping this works as is on aws servers...
        mpaRasterName = self.mpaRasterMapName+str(mpa.id)
        self.grass.v_to_r(mpaVectorName, mpaRasterName, 1)

        #Get precomputed map statistics
        stats = FishingImpactStats.objects.filter(map=map)
        if len(stats) < 1:
            return -1
        else:
            stats = stats[0]

        #Perhaps here we can check to see if we want to intersect or not
        #If there is an allowed use for this fishery, then no intersect is needed and resulting percentages should be 0.0
        setPercsToZero = False
        #Get list of targets from map
        #allowed_uses = AllowedUse.objects.filter(purpose__name=layer.fishing_type)
        map_targets = map.allowed_targets.all()
        #QUICK FIX...REMEMBER TO CHANGE THIS LATER!!!
        map_group = map.group_name
        if map_group in ['Commercial', 'Edible Seaweed']:
            map_purpose='commercial'
        else:
            map_purpose='recreational'
        for map_target in map_targets:
            if len(mpa.allowed_uses.filter(target__name=map_target, purpose__name=map_purpose)) != 0:
                setPercsToZero = True
        #The above is still somewhat unstable as it shouldn't just check for a single existing allowed target,
        #shouldn't it check to see if all targets reflected in this map are allowed within the mpa???

        #Intersect mpa raster with fishing map
        self.grass.r_intersect(self.mpaValueMaskMapName, mpaRasterName, self.fishingMapName)

        #Calculate percent area
        (mpaCells, mpaArea) = self.grass.r_area(self.mpaValueMaskMapName, map.cell_size)
        mpaArea = mpaArea * self.SQ_MILES_IN_SQ_METER

        mpaPercOverallArea = mmutil.percentage(mpaArea,stats.totalArea)

        #Calculate percent value
        mpaValue = self.grass.r_sum(self.mpaValueMaskMapName)
        if mpaValue is None:
            return -2

        mpaPercOverallValue = mmutil.percentage(mpaValue,stats.totalValue)

        if setPercsToZero:
            mpaPercOverallArea = 0.0
            mpaPercOverallValue = 0.0

        #Generate analysis result
        analResult = AnalysisResult(
            mpa.id,
            'mpa',
            map.group_name,
            map.port_name,
            map.species_name,
            mmutil.trueRound(mpaPercOverallArea,1),
            mmutil.trueRound(mpaPercOverallValue,1)
        )

        #Cleanup analysis
        self.__removeTempShapefile(temp_id)
        self.grass.cleanup()
        return analResult

    '''
    Given an mpa id and a full shapfile pathname (eg. /tmp/new_shapefile.shp)
    outputs the given mpa record to the specified file
    '''
    def __mpaToTempShapefile(self, mpa_id, temp_id):
        shp_filename = temp_id+'.shp'
        shapepath = os.path.join(tempfile.gettempdir(),shp_filename)
        shp_query = 'select id, user_id, geometry_final from mlpa_mlpampa where id='+str(mpa_id)
        command = settings.PGSQL2SHP+" -u "+settings.DATABASE_USER+" -P '"+settings.DATABASE_PASSWORD+"' -f "+shapepath+" "+settings.DATABASE_NAME+' "'+shp_query+'"'
        os.system(command)
        return shapepath

    '''
    Outputs current study region to /tmp/study_region.shp)
    '''
    def __srToShapefile(self):
        shp_filename = 'study_region.shp'
        shapepath = os.path.join(tempfile.gettempdir(),shp_filename)
        #the following query is specific on the server to id=4 as there are currently two Active study regions in the server-side db
        shp_query = 'select * from mm_study_region where active=TRUE'
        command = settings.PGSQL2SHP+" -u "+settings.DATABASE_USER+" -P '"+settings.DATABASE_PASSWORD+"' -f "+shapepath+" "+settings.DATABASE_NAME+' "'+shp_query+'"'
        os.system(command)
        return shapepath

    '''
    Removes a shape with the given name from the default temp directory
    '''
    def __removeTempShapefile(self, name):
        extensions = ['shp','shx','prj','dbf']
        for extension in extensions:
            filename = name+'.'+extension
            filePath = os.path.join(tempfile.gettempdir(),filename)
            if os.path.exists(filePath):
                os.remove(filePath)
        return True

class EmptyAnalysisResult:
    def __init__(self, group_name, port_name, species_name):
        self.user_grp = group_name
        self.port = port_name
        self.species = species_name
        self.mpaPercOverallArea = '---'
        self.mpaPercOverallValue = '---'

'''
An AnalysisResult represents the result of running the impact analysis
on a single Layer
'''
class AnalysisResult:
    def __init__(self,
                 id = None,
                 id_type = None,
                 user_grp = None,
                 port = None,
                 species = None,
                 mpaPercOverallArea = None,
                 mpaPercOverallValue = None
                 ):
        self.mpa_id = None
        self.array_id = None
        if id_type == 'mpa':
            self.mpa_id = id
        else:
            self.array_id = id

        self.user_grp = user_grp
        self.port = port
        self.species = species
        self.mpaPercOverallArea = mpaPercOverallArea
        self.mpaPercOverallValue = mpaPercOverallValue    #% of total fishing value captured by mpa

<commit_msg>continuing to clean up the code a bit...hoping this particular change might cause a slight increase in performance...<commit_after>import random
import os
import datetime
import re
import tempfile
from nc_mlpa import settings
from grass import Grass
import utilities as mmutil
from models import *
from nc_mlpa.mlpa.models import *
#from Layers import *
from Layers import Layers

'''
Used to calculate summary statistics of a given mpa's impact on localized fishing
interests.  Uses grass to perform the calculations on Layers.  Much of the overall
and study region level statistics are precalculated and stored in the database
using the FishingImpactStats model.  Mpa level statistics are calculated at run-time.
'''
class Analysis:
    def __init__(self):
        self.SQ_MILES_IN_SQ_METER = 0.000000386102159
        self.SIG_DIGITS = 3
        #location under GRASS_ROOT to store grass mapsets
        self.MM_GRASS_LOCATION = 'mlpa_nc'
        #mapset under location for grass map layers
        self.MM_SRC_GRASS_MAPSET = 'PERMANENT'
        # Source for fishing impact layers used by Grass for analysis.
        self.FISHING_IMPACT_ANALYSIS_ROOT = settings.GIS_DATA_ROOT + 'analysis/'

        #Temporary names for intermediate grass analysis.  Can be reused each run.
        self.srMapName = 'studyRegion'
        self.mpaVectorMapName = 'mpaVectorMap'          # mpa vector
        self.mpaRasterMapName = 'mpaRasterMap'          # mpa raster
        self.srValueMaskMapName = 'srValueMaskMap'      # fishing value within study region
        self.mpaValueMaskMapName = 'mpaValueMaskMap'    # fishing value within mpa

        self.grass = None

    def setupGrass(self, temp_id=None):
        self.MM_DST_GRASS_MAPSET = temp_id
        grass = Grass(settings.GRASS_ROOT, self.MM_GRASS_LOCATION, self.MM_SRC_GRASS_MAPSET, self.MM_DST_GRASS_MAPSET)
        self.MM_GRASS_RAST_PATH = grass.getRastPath()
        return grass

    '''
    Fishing impact analysis driver.
    '''
    def run(self, mpa, layers):
        analResults = []
        for layer in layers:
            analResult = self.__runAnal(mpa, layer)
            analResults.append(analResult)
        return analResults

    '''
    Driver for preloading and precalculating grass map layers and
    overall/study region summary statistics.  This is invoked via
    the manage.py command defined in the management directory of
    this module
    '''
    def preload(self):
        #Setup grass to use the base mapset, we're going to load it
        self.grass = self.setupGrass()
        self.__preloadStudyRegion()
        layers_list = GetAllLayers()
        for layer in layers_list:
            map = self.__preloadDb(layer)
            self.__preloadGrassMap(map)
            self.__preloadMapStats(map)

    '''
    Preload information about fishing impact map into DB
    '''
    def __preloadDb(self, layer):
        #Check for existing record
        maps = FishingImpactAnalysisMap.objects.filter(
            group_name=layer.group_name,
            port_name=layer.port_name,
            species_name=layer.species_name
        )

        #If record already exists, update it
        if len(maps) > 0:
            map = maps[0]
            map.group_name = layer.group_name
            map.group_abbr = layer.group_abbr
            map.port_name = layer.port_name
            map.port_abbr = layer.port_abbr
            map.species_name = layer.species_name
            map.species_abbr = layer.species_abbr
            map.cell_size = layer.cell_size
            map.save()
        #Otherwise create a new record
        else:
            map = FishingImpactAnalysisMap(
                group_name=layer.group_name,
                group_abbr=layer.group_abbr,
                port_name=layer.port_name,
                port_abbr=layer.port_abbr,
                species_name=layer.species_name,
                species_abbr=layer.species_abbr,
                cell_size=layer.cell_size
            )
            map.save()

        #Add Allowed Targets to AnalysisMap
        targets = layer.target_names
        for target in targets:
            allowed_target = AllowedTarget.objects.filter(name=target)
            if len(allowed_target) > 0:
                map.allowed_targets.add(allowed_target[0])
        map.save()

        #Map the new AnalysisMap to one or more allowed uses
        #Filter by fishing type
        allowed_uses = AllowedUse.objects.filter(
            purpose__name=layer.fishing_type
        )

        #Get methods for current species
        methods = layer.take_methods
        species_name = layer.species_name

        #Add Allowed Uses to AnalysisMap
        for method_name in methods:
            uses = allowed_uses.filter(method__name=method_name, target__name=species_name)
            for use in uses:
                map.allowed_uses.add(use)
            map.save()

        return map

    '''
    Load study region from DB into grass.
    '''
    def __preloadStudyRegion(self):
        #Output study region to shapefile
        srPath = self.__srToShapefile()
        #Convert shapefile to grass vector map
        self.grass.v_in_ogr(srPath, self.srMapName)
        #Convert study vector map to raster map
        self.grass.v_to_r(self.srMapName, self.srMapName, 1)

    '''
    Preload a fishing impact map into grass for later use in analysis
    '''
    def __preloadGrassMap(self, map):
        #Build path to fishing impact grids
        mapPath = self.FISHING_IMPACT_ANALYSIS_ROOT+map.group_abbr+'/'
        #Get name of current grid
        mapName = map.getFullName()
        #Getpath to grass raster map
        grassRasterPath = os.path.join(self.MM_GRASS_RAST_PATH,mapName)
        #Load fishing impact grid into Grass for later use
        if not os.path.exists(grassRasterPath):
            self.grass.r_in_gdal(mapPath+map.getGridName(), mapName)

    '''
    Precalculates overall and study region level statistics and stores them in the database.
    '''
    def __preloadMapStats(self, map):
        self.fishingMapName = map.getFullName()       # fishing value map

        #Total fishing value.  Sums cell values in fishing map
        totalValue = self.grass.r_sum(self.fishingMapName)
        if totalValue is None:
            return None

        #Fishing value in study region.  Intersects fishing map with study regions and
        #sums remaining cells.
        self.grass.r_intersect(self.srValueMaskMapName, self.srMapName, self.fishingMapName)
        srValue = self.grass.r_sum(self.srValueMaskMapName)
        if srValue is None:
            return None

        # Total fishing area.  Makes all nonzero cells 1 and sums the values getting
        # the number of cells.  Use cell size to get area.
        (totalCells, totalArea) = self.grass.r_area(self.fishingMapName, map.cell_size)
        totalArea = totalArea * self.SQ_MILES_IN_SQ_METER

        # Fishing area in study area.  Uses previous fishing map intersected to study
        # region, makes all nonzero cells 1 and sums the values getting number of cells.
        # Use cell size to get area.
        (srCells, srArea) = self.grass.r_area(self.srValueMaskMapName, map.cell_size)
        srArea = srArea * self.SQ_MILES_IN_SQ_METER

        # Create a new stats record.  If one already exists, update it.
        stats = FishingImpactStats.objects.filter(map__group_name=map.group_name,
                                                  map__port_name=map.port_name,
                                                  map__species_name=map.species_name)

        #If record already exists, update it
        if len(stats) > 0:
            fi = stats[0]
            fi.totalCells=totalCells
            fi.srCells=srCells
            fi.totalArea=totalArea
            fi.srArea=srArea
            fi.totalValue=totalValue
            fi.srValue=srValue
            fi.save()
        #Otherwise create a new record
        else:
            fi = FishingImpactStats(map=map,
                                    totalCells=totalCells,
                                    srCells=srCells,
                                    totalArea=totalArea,
                                    srArea=srArea,
                                    totalValue=totalValue,
                                    srValue=srValue)
            fi.save()

    '''
    Performs a single grass analysis run.  Assumes that the source mapset has been
    preloaded with all of the necessary map layers for analysis.  Don't run this
    directly.  Use the analysis driver named 'run' and

    Returns -1 if failed to retrieve precomputed overall and study region statistics
    Returns -2 if mpa analysis failed.
    '''
    def __runAnal(self, mpa, map):
        self.fishingMapName = map.getFullName()       # fishing value map
        #self.fishingMapName = map.getGridName()
        timestamp = datetime.datetime.now().strftime('%m_%d_%y_%H%M')
        temp_id = 'mpa'+str(mpa.id)+'_user'+str(mpa.user_id)+'_'+timestamp+'_'+str(mmutil.getRandInt())
        self.tmpMapsetName = temp_id

        #Initialize grass, creating temporary mapset
        self.grass = self.setupGrass(self.tmpMapsetName)
        #Copy preloaded fishing map to temp mapset
        self.grass.copyMap('rast', self.fishingMapName)

        #Output mpa to shapefile
        shapepath = self.__mpaToTempShapefile(mpa.id, temp_id)
        #Convert shapefile to grass vector map
        mpaVectorName = self.mpaVectorMapName+str(mpa.id)
        self.grass.v_in_ogr(shapepath, mpaVectorName)
        #######################################################################################
        #import time
        #time.sleep(5)
        #required for whatever reason in order to produce correct results on my windows machine
        #NOT NEEDED on aws servers
        #######################################################################################
        #Convert mpa vector map to raster map
        mpaRasterName = self.mpaRasterMapName+str(mpa.id)
        self.grass.v_to_r(mpaVectorName, mpaRasterName, 1)

        #Get precomputed map statistics
        stats = FishingImpactStats.objects.filter(map=map)
        if len(stats) < 1:
            return -1
        else:
            stats = stats[0]

        #If there is an allowed use for this fishery, then no intersect is needed and resulting percentages should be 0.0
        setPercsToZero = False
        #Get list of targets from map
        #allowed_uses = AllowedUse.objects.filter(purpose__name=layer.fishing_type)
        map_targets = map.allowed_targets.all()
        map_purpose = Layers.FISHING_TYPES[map.group_abbr]
        for map_target in map_targets:
            if len(mpa.allowed_uses.filter(target__name=map_target, purpose__name=map_purpose)) != 0:
                setPercsToZero = True

        if setPercsToZero:
            mpaPercOverallArea = 0.0
            mpaPercOverallValue = 0.0
        else:
            #Intersect mpa raster with fishing map
            self.grass.r_intersect(self.mpaValueMaskMapName, mpaRasterName, self.fishingMapName)

            #Calculate percent area
            (mpaCells, mpaArea) = self.grass.r_area(self.mpaValueMaskMapName, map.cell_size)
            mpaArea = mpaArea * self.SQ_MILES_IN_SQ_METER

            mpaPercOverallArea = mmutil.percentage(mpaArea,stats.totalArea)

            #Calculate percent value
            mpaValue = self.grass.r_sum(self.mpaValueMaskMapName)
            if mpaValue is None:
                return -2

            mpaPercOverallValue = mmutil.percentage(mpaValue,stats.totalValue)

        #Generate analysis result
        analResult = AnalysisResult(
            mpa.id,
            'mpa',
            map.group_name,
            map.port_name,
            map.species_name,
            mmutil.trueRound(mpaPercOverallArea,1),
            mmutil.trueRound(mpaPercOverallValue,1)
        )

        #Cleanup analysis
        self.__removeTempShapefile(temp_id)
        self.grass.cleanup()
        return analResult

    '''
    Given an mpa id and a full shapfile pathname (eg. /tmp/new_shapefile.shp)
    outputs the given mpa record to the specified file
    '''
    def __mpaToTempShapefile(self, mpa_id, temp_id):
        shp_filename = temp_id+'.shp'
        shapepath = os.path.join(tempfile.gettempdir(),shp_filename)
        shp_query = 'select id, user_id, geometry_final from mlpa_mlpampa where id='+str(mpa_id)
        command = settings.PGSQL2SHP+" -u "+settings.DATABASE_USER+" -P '"+settings.DATABASE_PASSWORD+"' -f "+shapepath+" "+settings.DATABASE_NAME+' "'+shp_query+'"'
        os.system(command)
        return shapepath

    '''
    Outputs current study region to /tmp/study_region.shp)
    '''
    def __srToShapefile(self):
        shp_filename = 'study_region.shp'
        shapepath = os.path.join(tempfile.gettempdir(),shp_filename)
        #the following query is specific on the server to id=4 as there are currently two Active study regions in the server-side db
        shp_query = 'select * from mm_study_region where active=TRUE'
        command = settings.PGSQL2SHP+" -u "+settings.DATABASE_USER+" -P '"+settings.DATABASE_PASSWORD+"' -f "+shapepath+" "+settings.DATABASE_NAME+' "'+shp_query+'"'
        os.system(command)
        return shapepath

    '''
    Removes a shape with the given name from the default temp directory
    '''
    def __removeTempShapefile(self, name):
        extensions = ['shp','shx','prj','dbf']
        for extension in extensions:
            filename = name+'.'+extension
            filePath = os.path.join(tempfile.gettempdir(),filename)
            if os.path.exists(filePath):
                os.remove(filePath)
        return True

class EmptyAnalysisResult:
    def __init__(self, group_name, port_name, species_name):
        self.user_grp = group_name
        self.port = port_name
        self.species = species_name
        self.mpaPercOverallArea = '---'
        self.mpaPercOverallValue = '---'

'''
An AnalysisResult represents the result of running the impact analysis
on a single Layer
'''
class AnalysisResult:
    def __init__(self,
                 id = None,
                 id_type = None,
                 user_grp = None,
                 port = None,
                 species = None,
                 mpaPercOverallArea = None,
                 mpaPercOverallValue = None
                 ):
        self.mpa_id = None
        self.array_id = None
        if id_type == 'mpa':
            self.mpa_id = id
        else:
            self.array_id = id

        self.user_grp = user_grp
        self.port = port
        self.species = species
        self.mpaPercOverallArea = mpaPercOverallArea
        self.mpaPercOverallValue = mpaPercOverallValue    #% of total fishing value captured by mpa

<|endoftext|>


<commit_before>import os
import warnings

from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.core.exceptions import ImproperlyConfigured
from django.utils.importlib import import_module

from staticfiles import utils
from staticfiles.settings import URL as STATIC_URL, ROOT as STATIC_ROOT, PREPEND_LABEL_APPS

class StaticFilesStorage(FileSystemStorage):
    """
    Standard file system storage for site media files.

    The defaults for ``location`` and ``base_url`` are
    ``STATIC_ROOT`` and ``STATIC_URL``.
    """
    def __init__(self, location=None, base_url=None, *args, **kwargs):
        if location is None:
            location = STATIC_ROOT
        if base_url is None:
            base_url = STATIC_URL
        if not location:
            raise ImproperlyConfigured("You're using the staticfiles app "
                "without having set the STATIC_ROOT setting. Set it to "
                "the absolute path of the directory that holds static media.")
        # check for None since we might use a root URL (``/``)
        if base_url is None:
            raise ImproperlyConfigured("You're using the staticfiles app "
                "without having set the STATIC_URL setting. Set it to "
                "URL that handles the files served from STATIC_ROOT.")
        if settings.DEBUG:
            utils.check_settings()
        super(StaticFilesStorage, self).__init__(location, base_url,
                                                 *args, **kwargs)

class StaticFileStorage(StaticFilesStorage):

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The storage backend 'staticfiles.storage.StaticFileStorage' "
            "was renamed to 'staticfiles.storage.StaticFilesStorage'.",
            PendingDeprecationWarning)
        super(StaticFileStorage, self).__init__(*args, **kwargs)

class AppStaticStorage(FileSystemStorage):
    """
    A file system storage backend that takes an app module and works
    for the ``static`` directory of it.
    """
    source_dir = 'static'

    def __init__(self, app, *args, **kwargs):
        """
        Returns a static file storage if available in the given app.
        """
        # app is actually the models module of the app. Remove the '.models'.
        bits = app.__name__.split('.')[:-1]
        self.app_name = bits[-1]
        self.app_module = '.'.join(bits)
        # The models module (app) may be a package in which case
        # dirname(app.__file__) would be wrong. Import the actual app
        # as opposed to the models module.
        app = import_module(self.app_module)
        location = self.get_location(os.path.dirname(app.__file__))
        super(AppStaticStorage, self).__init__(location, *args, **kwargs)

    def get_location(self, app_root):
        """
        Given the app root, return the location of the static files of an app,
        by default 'static'. We special case the admin app here since it has
        its static files in 'media'.
        """
        if self.app_module == 'django.contrib.admin':
            return os.path.join(app_root, 'media')
        return os.path.join(app_root, self.source_dir)

    def get_prefix(self):
        """
        Return the path name that should be prepended to files for this app.
        """
        if self.app_module in PREPEND_LABEL_APPS:
            return app_name
        return None

    def get_files(self, ignore_patterns=[]):
        """
        Return a list containing the relative source paths for all files that
        should be copied for an app.
        """
        files = []
        prefix = self.get_prefix()
        for path in utils.get_files(self, ignore_patterns):
            if prefix:
                path = '/'.join([prefix, path])
            files.append(path)
        return files
<commit_msg>Fixed NameError.<commit_after>import os
import warnings

from django.conf import settings
from django.core.files.storage import FileSystemStorage
from django.core.exceptions import ImproperlyConfigured
from django.utils.importlib import import_module

from staticfiles import utils
from staticfiles.settings import URL as STATIC_URL, ROOT as STATIC_ROOT, PREPEND_LABEL_APPS

class StaticFilesStorage(FileSystemStorage):
    """
    Standard file system storage for site media files.

    The defaults for ``location`` and ``base_url`` are
    ``STATIC_ROOT`` and ``STATIC_URL``.
    """
    def __init__(self, location=None, base_url=None, *args, **kwargs):
        if location is None:
            location = STATIC_ROOT
        if base_url is None:
            base_url = STATIC_URL
        if not location:
            raise ImproperlyConfigured("You're using the staticfiles app "
                "without having set the STATIC_ROOT setting. Set it to "
                "the absolute path of the directory that holds static media.")
        # check for None since we might use a root URL (``/``)
        if base_url is None:
            raise ImproperlyConfigured("You're using the staticfiles app "
                "without having set the STATIC_URL setting. Set it to "
                "URL that handles the files served from STATIC_ROOT.")
        if settings.DEBUG:
            utils.check_settings()
        super(StaticFilesStorage, self).__init__(location, base_url,
                                                 *args, **kwargs)

class StaticFileStorage(StaticFilesStorage):

    def __init__(self, *args, **kwargs):
        warnings.warn(
            "The storage backend 'staticfiles.storage.StaticFileStorage' "
            "was renamed to 'staticfiles.storage.StaticFilesStorage'.",
            PendingDeprecationWarning)
        super(StaticFileStorage, self).__init__(*args, **kwargs)

class AppStaticStorage(FileSystemStorage):
    """
    A file system storage backend that takes an app module and works
    for the ``static`` directory of it.
    """
    source_dir = 'static'

    def __init__(self, app, *args, **kwargs):
        """
        Returns a static file storage if available in the given app.
        """
        # app is actually the models module of the app. Remove the '.models'.
        bits = app.__name__.split('.')[:-1]
        self.app_name = bits[-1]
        self.app_module = '.'.join(bits)
        # The models module (app) may be a package in which case
        # dirname(app.__file__) would be wrong. Import the actual app
        # as opposed to the models module.
        app = import_module(self.app_module)
        location = self.get_location(os.path.dirname(app.__file__))
        super(AppStaticStorage, self).__init__(location, *args, **kwargs)

    def get_location(self, app_root):
        """
        Given the app root, return the location of the static files of an app,
        by default 'static'. We special case the admin app here since it has
        its static files in 'media'.
        """
        if self.app_module == 'django.contrib.admin':
            return os.path.join(app_root, 'media')
        return os.path.join(app_root, self.source_dir)

    def get_prefix(self):
        """
        Return the path name that should be prepended to files for this app.
        """
        if self.app_module in PREPEND_LABEL_APPS:
            return self.app_name
        return None

    def get_files(self, ignore_patterns=[]):
        """
        Return a list containing the relative source paths for all files that
        should be copied for an app.
        """
        files = []
        prefix = self.get_prefix()
        for path in utils.get_files(self, ignore_patterns):
            if prefix:
                path = '/'.join([prefix, path])
            files.append(path)
        return files
<|endoftext|>


<commit_before># Copyright (c) 2013-2014 Will Thames <will@thames.id.au>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import unittest

import ansiblelint
from ansiblelint import Runner, RulesCollection
import ansiblelint.formatters

class TestRule(unittest.TestCase):

    def setUp(self):
        rulesdir = os.path.join('lib', 'ansiblelint', 'rules')
        self.rules = RulesCollection([rulesdir])

    def test_runner_count(self):
        filename = 'test/nomatchestest.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_unicode_runner_count(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 1)

    def test_unicode_standard_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.Formatter()
        formatter.format(matches[0])

    def test_unicode_parseable_colored_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.ParseableFormatter()
        formatter.format(matches[0], colored=True)

    def test_unicode_quiet_colored_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.QuietFormatter()
        formatter.format(matches[0], colored=True)

    def test_unicode_standard_color_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.Formatter()
        formatter.format(matches[0], colored=True)

    def test_runner_exclude_var_expansion(self):
        filename = 'example/lots_of_warnings.yml'
        os.environ['EXCLUDE_PATH'] = filename
        excludes = ['$EXCLUDE_PATH']
        runner = Runner(self.rules, filename, [], [], excludes)
        assert filename in runner.exclude_paths

    def test_runner_exclude_user_expansion(self):
        filename = 'example/lots_of_warnings.yml'
        excludes = ['~']
        runner = Runner(self.rules, filename, [], [], excludes)
        assert os.path.expanduser('~') in runner.exclude_paths

    def test_runner_excludes_paths(self):
        filename = 'examples/lots_of_warnings.yml'
        excludes = ['examples/lots_of_warnings.yml']
        runner = Runner(self.rules, filename, [], [], excludes)
        assert (len(runner.run()) == 0)

    def test_runner_block_count(self):
        filename = 'test/block.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_become_count(self):
        filename = 'test/become.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_empty_tags_count(self):
        filename = 'test/emptytags.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_encrypted_secrets(self):
        filename = 'test/contains_secrets.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_dir_with_trailing_slash(self):
        filename = 'test/'
        runner = Runner(self.rules, filename, [], [], [])
        assert (list(runner.playbooks)[0][1] == 'role')

    def test_dir_with_fullpath(self):
        filename = os.path.abspath('test')
        runner = Runner(self.rules, filename, [], [], [])
        assert (list(runner.playbooks)[0][1] == 'role')

    def test_files_not_scanned_twice(self):
        checked_files = set()

        filename = os.path.abspath('test/common-include-1.yml')
        runner = Runner(self.rules, filename, [], [], [], 0, checked_files)
        run1 = runner.run()

        filename = os.path.abspath('test/common-include-2.yml')
        runner = Runner(self.rules, filename, [], [], [], 0, checked_files)
        run2 = runner.run()

        assert ((len(run1) + len(run2)) == 1)
<commit_msg>Move runner expansion tests out of class<commit_after># Copyright (c) 2013-2014 Will Thames <will@thames.id.au>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import unittest

import ansiblelint
from ansiblelint import default_rulesdir, Runner, RulesCollection
import ansiblelint.formatters

class TestRule(unittest.TestCase):

    def setUp(self):
        rulesdir = os.path.join('lib', 'ansiblelint', 'rules')
        self.rules = RulesCollection([rulesdir])

    def test_runner_count(self):
        filename = 'test/nomatchestest.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_unicode_runner_count(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 1)

    def test_unicode_standard_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.Formatter()
        formatter.format(matches[0])

    def test_unicode_parseable_colored_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.ParseableFormatter()
        formatter.format(matches[0], colored=True)

    def test_unicode_quiet_colored_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.QuietFormatter()
        formatter.format(matches[0], colored=True)

    def test_unicode_standard_color_formatting(self):
        filename = 'test/unicode.yml'
        runner = Runner(self.rules, filename, [], [], [])
        matches = runner.run()
        formatter = ansiblelint.formatters.Formatter()
        formatter.format(matches[0], colored=True)

    def test_runner_excludes_paths(self):
        filename = 'examples/lots_of_warnings.yml'
        excludes = ['examples/lots_of_warnings.yml']
        runner = Runner(self.rules, filename, [], [], excludes)
        assert (len(runner.run()) == 0)

    def test_runner_block_count(self):
        filename = 'test/block.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_become_count(self):
        filename = 'test/become.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_empty_tags_count(self):
        filename = 'test/emptytags.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_runner_encrypted_secrets(self):
        filename = 'test/contains_secrets.yml'
        runner = Runner(self.rules, filename, [], [], [])
        assert (len(runner.run()) == 0)

    def test_dir_with_trailing_slash(self):
        filename = 'test/'
        runner = Runner(self.rules, filename, [], [], [])
        assert (list(runner.playbooks)[0][1] == 'role')

    def test_dir_with_fullpath(self):
        filename = os.path.abspath('test')
        runner = Runner(self.rules, filename, [], [], [])
        assert (list(runner.playbooks)[0][1] == 'role')

    def test_files_not_scanned_twice(self):
        checked_files = set()

        filename = os.path.abspath('test/common-include-1.yml')
        runner = Runner(self.rules, filename, [], [], [], 0, checked_files)
        run1 = runner.run()

        filename = os.path.abspath('test/common-include-2.yml')
        runner = Runner(self.rules, filename, [], [], [], 0, checked_files)
        run2 = runner.run()

        assert ((len(run1) + len(run2)) == 1)

def test_runner_exclude_var_expansion(monkeypatch):
    rules = RulesCollection([default_rulesdir])
    filename = 'example/lots_of_warnings.yml'
    monkeypatch.setenv('EXCLUDE_PATH', filename)
    excludes = ['$EXCLUDE_PATH']
    runner = Runner(rules, filename, [], [], excludes)
    assert filename in runner.exclude_paths

def test_runner_exclude_user_expansion():
    rules = RulesCollection([default_rulesdir])
    filename = 'example/lots_of_warnings.yml'
    excludes = ['~']
    runner = Runner(rules, filename, [], [], excludes)
    assert os.path.expanduser('~') in runner.exclude_paths
<|endoftext|>


<commit_before>"""
age.py -- show age of code
"""

# list files in Git branch
# git ls-tree --name-status --full-tree -r v4.0.0

import datetime
import os
import re

from django.core.management.base import BaseCommand
from git import Repo
import matplotlib.pyplot as plt

def get_tags(repo):
    tags = [tag.name.lstrip('v') for tag in repo.tags
        if tag.name.startswith('v3.') or tag.name == 'v4.0.0']
    tags.sort(key=lambda st: map(int, st.lstrip('v').split('.')))
    tags = [repo.tags['v'+version] for version in tags]
    return tags

def show_version_diffs(tags):
    tags = list(tags)
    old = tags.pop(0)
    for new in tags:
        diff_index = old.commit.diff(new)
        print '{:7} - {:7}: {} commits'.format(
            old.name, new.name, len(diff_index))
        old = new

# IDEA: use iter_change_type('A') to get all file paths, even if they've been renamed/deleted
# Or: git.diff('v3.0.0..v4.0.0',name_status=True)
# ALSO
# - git.diff('v3.0.0..v3.1.0',dirstat=True)
# TODO: git blame --porcelain v3.0.0..v4.0.0 ip-fou.8

range_1 = 'v3.0.0..v3.1.0'
range_all = 'v3.0.0..v4.0.0'

# TODO: skip "total" at end
def parse_diff_changes(diff_text):
    re_path_num = re.compile(
        r'^\s(\S+) .+? \| \s+ (\d+)',
        re.MULTILINE | re.VERBOSE)
    return re_path_num.finditer(diff_text)

def format_path_changes(all_paths, path_changes):
    def format_diff_value(value):
        return '?.-+*ABCD'[len(value)]

    change_dict = dict(match.groups() for match in path_changes)
    def format_chars():
        for path in all_paths:
            if path not in change_dict:
                yield ' '
            else:
                yield format_diff_value(change_dict[path])
    return ''.join(format_chars())

class Matcher(object):
    def __init__(self, pattern):
        self.match = re.compile(pattern).search

class ManpageMatcher(Matcher):
    def __init__(self):
        super(ManpageMatcher, self).__init__('man/')

all_matcher = str

def get_paths(repo):
    diff_text = repo.git.diff(range_all, stat=True)
    all_paths = (match.group(1) for match in parse_diff_changes(diff_text))
    return sorted(all_paths)

def get_tag_paths(repo, tag):
    ls_tree_text = repo.git.ls_tree(
        tag, full_tree=True, name_status=True, r=True)
    return ls_tree_text.split('\n')

def render_image(repo, matchfunc):
    good_paths = filter(matchfunc, get_paths(repo))
    print good_paths
    tags = get_tags(repo)

    good_index = dict((path, index)
        for index,path in enumerate(good_paths))

# TODO: currently does git-diff, then re-does diff to get each
# "good" path's contribution.  Simplify to just do git-diff once.

    symbols = []
    old = tags.pop(0)
    for y,new in enumerate(tags):
        diff_index = old.commit.diff(new)
        diff_versions = '{}..{}'.format(old.name, new.name)
        old = new
        print '{:7}: {:3}'.format(new.name, len(diff_index))
        diff_paths = set(d.a_path for d in diff_index) & set(good_paths)
        if not diff_paths:
            # version has diffs, but not in the region of interest
            print '- skip:', new
            continue
        diff_text = repo.git.diff(diff_versions, *diff_paths, stat=True)
        diff_path_changes = parse_diff_changes(diff_text)
        for path,diff_count in (
            match.groups() for match in diff_path_changes):
            try:
                x = good_index[path]
            except IndexError:
                print '?', path
                continue
            area = 6 * len(diff_count)
            symbols.append((x, y, area))

    xs, ys, areas = zip(*symbols)
    plt.scatter(xs, ys, s=areas)
    plt.xlabel('file index')
    plt.ylabel('version index')
    plt.savefig('z.png')
    plt.savefig('z.svg')

def render_text(repo, matchfunc):
    def format_age(mycommit):
        """
        format commit age as single character
        * recent (1-9 days); + 10-99 days, - 100-999 days
        """
        delta = mycommit.authored_datetime - datetime.datetime.now(
            mycommit.authored_datetime.tzinfo)
        delta_num = len(str(delta.days)) - 1
        return '?*+-.'[delta_num]

    tag = 'v4.0.0'
    for path in filter(matchfunc, get_tag_paths(repo, tag)):
        blame = repo.blame(tag, path)
        def path_age():
            for commit, regions in blame:
                yield format_age(commit) * len(regions)
        print path,':', ''.join(path_age())

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument('--match', choices=('manpage', 'source'))
        parser.add_argument('--style', choices=('text', 'image'),
            default='image')
        parser.add_argument('project_dirs', nargs='+')

    def handle(self, *args, **options):
        render_func = globals()['render_{}'.format(options['style'])]
        matchfunc = {
        # manpage -- in "man" subdir ending in .8 or .in
        'manpage': Matcher(r'man/.*[0-9n]$').match,
        'source': Matcher(r'\.[ch]$').match,
        }.get(options['match']) or all_matcher

        for project_dir in options['project_dirs']:
            repo = Repo(os.path.expanduser(project_dir))
            render_func(repo, matchfunc)
<commit_msg>text: output char for age, one char per line in orig source<commit_after>"""
age.py -- show age of code
"""
# pylint: disable=W0640

# list files in Git branch
# git ls-tree --name-status --full-tree -r v4.0.0

import datetime
import os
import re

from django.core.management.base import BaseCommand
from git import Repo
import matplotlib.pyplot as plt

def get_tags(repo):
    tags = [tag.name.lstrip('v') for tag in repo.tags
        if tag.name.startswith('v3.') or tag.name == 'v4.0.0']
    tags.sort(key=lambda st: map(int, st.lstrip('v').split('.')))
    tags = [repo.tags['v'+version] for version in tags]
    return tags

def show_version_diffs(tags):
    tags = list(tags)
    old = tags.pop(0)
    for new in tags:
        diff_index = old.commit.diff(new)
        print '{:7} - {:7}: {} commits'.format(
            old.name, new.name, len(diff_index))
        old = new

# IDEA: use iter_change_type('A') to get all file paths, even if they've been renamed/deleted
# Or: git.diff('v3.0.0..v4.0.0',name_status=True)
# ALSO
# - git.diff('v3.0.0..v3.1.0',dirstat=True)
# TODO: git blame --porcelain v3.0.0..v4.0.0 ip-fou.8

range_1 = 'v3.0.0..v3.1.0'
range_all = 'v3.0.0..v4.0.0'

# TODO: skip "total" at end
def parse_diff_changes(diff_text):
    re_path_num = re.compile(
        r'^\s(\S+) .+? \| \s+ (\d+)',
        re.MULTILINE | re.VERBOSE)
    return re_path_num.finditer(diff_text)

def format_path_changes(all_paths, path_changes):
    def format_diff_value(value):
        return '?.-+*ABCD'[len(value)]

    change_dict = dict(match.groups() for match in path_changes)
    def format_chars():
        for path in all_paths:
            if path not in change_dict:
                yield ' '
            else:
                yield format_diff_value(change_dict[path])
    return ''.join(format_chars())

class Matcher(object):
    def __init__(self, pattern):
        self.match = re.compile(pattern).search

class ManpageMatcher(Matcher):
    def __init__(self):
        super(ManpageMatcher, self).__init__('man/')

all_matcher = str

def get_paths(repo):
    diff_text = repo.git.diff(range_all, stat=True)
    all_paths = (match.group(1) for match in parse_diff_changes(diff_text))
    return sorted(all_paths)

def get_tag_paths(repo, tag):
    ls_tree_text = repo.git.ls_tree(
        tag, full_tree=True, name_status=True, r=True)
    return ls_tree_text.split('\n')

# X probably buggy
def get_latest_datetime(repo, tag):
    latest_opts = {'1': True, 'date': 'short', 'format': '%cd'}
    log_text = repo.git.log(tag, **latest_opts)
    log_date = datetime.datetime.strptime(log_text, '%Y-%m-%d')
    return log_date

def render_image(repo, matchfunc):
    good_paths = filter(matchfunc, get_paths(repo))
    print good_paths
    tags = get_tags(repo)

    good_index = dict((path, index)
        for index,path in enumerate(good_paths))

# TODO: currently does git-diff, then re-does diff to get each
# "good" path's contribution.  Simplify to just do git-diff once.

    symbols = []
    old = tags.pop(0)
    for y,new in enumerate(tags):
        diff_index = old.commit.diff(new)
        diff_versions = '{}..{}'.format(old.name, new.name)
        old = new
        print '{:7}: {:3}'.format(new.name, len(diff_index))
        diff_paths = set(d.a_path for d in diff_index) & set(good_paths)
        if not diff_paths:
            # version has diffs, but not in the region of interest
            print '- skip:', new
            continue
        diff_text = repo.git.diff(diff_versions, *diff_paths, stat=True)
        diff_path_changes = parse_diff_changes(diff_text)
        for path,diff_count in (
            match.groups() for match in diff_path_changes):
            try:
                x = good_index[path]
            except IndexError:
                print '?', path
                continue
            area = 6 * len(diff_count)
            symbols.append((x, y, area))

    xs, ys, areas = zip(*symbols)
    plt.scatter(xs, ys, s=areas)
    plt.xlabel('file index')
    plt.ylabel('version index')
    plt.savefig('z.png')
    plt.savefig('z.svg')

def render_text(repo, matchfunc):
    def format_age(mycommit, mylatest):
        """
        format commit age as single character
        * recent (0-9 days); + 10-99 days, - 100-999 days
        """
        authored_dt = mycommit.authored_datetime.replace(tzinfo=None)
        delta = mylatest - authored_dt
        delta_num = len(str(delta.days))
        return '?*+-.'[delta_num]

    tag = 'v4.0.0'
    latest = get_latest_datetime(repo, tag)
    for path in filter(matchfunc, get_tag_paths(repo, tag)):
        blame = repo.blame(tag, path)
        def path_age():
            for commit, regions in blame:
                yield format_age(commit, latest) * len(regions)
        print path,':', ''.join(path_age())

class Command(BaseCommand):
    help = __doc__

    def add_arguments(self, parser):
        parser.add_argument('--match', choices=('manpage', 'source'))
        parser.add_argument('--style', choices=('text', 'image'),
            default='image')
        parser.add_argument('project_dirs', nargs='+')

    def handle(self, *args, **options):
        render_func = globals()['render_{}'.format(options['style'])]
        matchfunc = {
        # manpage -- in "man" subdir ending in .8 or .in
        'manpage': Matcher(r'man/.*[0-9n]$').match,
        'source': Matcher(r'\.[ch]$').match,
        }.get(options['match']) or all_matcher

        for project_dir in options['project_dirs']:
            repo = Repo(os.path.expanduser(project_dir))
            render_func(repo, matchfunc)
<|endoftext|>


<commit_before>"""This is the command-line program for cosmic ray.

Here we manage command-line parsing and launching of the internal
machinery that does mutation testing.
"""
import itertools
import json
import logging
import sys

import docopt
import transducer.functional
import transducer.lazy
import transducer.transducers

import cosmic_ray.counting
import cosmic_ray.modules
import cosmic_ray.json_util
import cosmic_ray.worker
import cosmic_ray.testing
import cosmic_ray.timing

LOG = logging.getLogger()

OPTIONS = """cosmic-ray

Usage: cosmic-ray [--verbose] [--help] <command> [<args> ...]

options:
  --help     Show this screen.
  --verbose  Produce more verbose output

Available commands:
  load
  operators
  run
  test-runners
  worker

See 'cosmic-ray help <command>' for help on specific commands.
"""

# This is really an experiment in using transducers in "the real
# world". You could accomplish the same parsing goals in fewer lines
# (and probably more quckly) using more traditional means. But this
# approach does have a certain charm and elegance to it.
REMOVE_COMMENTS = transducer.transducers.mapping(lambda x: x.split('#')[0])
REMOVE_WHITESPACE = transducer.transducers.mapping(str.strip)
NON_EMPTY = transducer.transducers.filtering(bool)
CONFIG_FILE_PARSER = transducer.functional.compose(REMOVE_COMMENTS,
                                                   REMOVE_WHITESPACE,
                                                   NON_EMPTY)

def _load_file(config_file):
    """Read configuration from a file.

    This reads `config_file`, yielding each non-empty line with
    whitespace and comments stripped off.
    """
    with open(config_file, 'rt', encoding='utf-8') as f:
        yield from transducer.lazy.transduce(CONFIG_FILE_PARSER, f)

def handle_help(config):
    """usage: cosmic-ray help [<command>]

Get help on a specific command.
"""
    command = config['<command>']
    if not command:
        options = OPTIONS
    elif command not in COMMAND_HANDLER_MAP:
        LOG.error('"{}" is not a valid cosmic-ray command'.format(command))
        options = OPTIONS
    else:
        options = COMMAND_HANDLER_MAP[command].__doc__

    return docopt.docopt(options,
                         ['--help'],
                         version='cosmic-ray v.2')

def handle_load(config):
    """usage: cosmic-ray load <config-file>

Load a command configuration from a file and run it.
    """
    filename = config['<config-file>']
    argv = _load_file(filename)
    return main(argv=list(argv))

def handle_baseline(configuration):
    """usage: cosmic-ray baseline [options] <top-module> <test-dir>

options:
  --verbose           Produce verbose output
  --no-local-import   Allow importing module from the current directory
  --test-runner=R     Test-runner plugin to use [default: unittest]
"""
    sys.path.insert(0, '')
    test_runner = cosmic_ray.plugins.get_test_runner(
            configuration['--test-runner'],
            configuration['<test-dir>'])

    test_runner()

def handle_run(configuration):
    """usage: cosmic-ray run [options] [--exclude-modules=P ...] (--timeout=T | --baseline=M) <top-module> <test-dir>

options:
  --verbose           Produce verbose output
  --no-local-import   Allow importing module from the current directory
  --test-runner=R     Test-runner plugin to use [default: unittest]
  --exclude-modules=P Pattern of module names to exclude from mutation
  --num-testers=N     Number of concurrent testers to run (0 = os.cpu_count()) [default: 0]
"""
    # This lets us import modules from the current directory. Should probably
    # be optional, and needs to also be applied to workers!
    sys.path.insert(0, '')

    if configuration['--timeout'] is not None:
        timeout = float(configuration['--timeout'])
    else:
        baseline_mult = float(configuration['--baseline'])
        assert baseline_mult is not None
        timeout = baseline_mult * cosmic_ray.timing.run_baseline(
            configuration['--test-runner'],
            configuration['<top-module>'],
            configuration['<test-dir>'])

    LOG.info('timeout = {} seconds'.format(timeout))

    modules = cosmic_ray.modules.filtered_modules(
        cosmic_ray.modules.find_modules(configuration['<top-module>']),
        configuration['--exclude-modules'])

    operators = cosmic_ray.plugins.operator_names()

    counts = cosmic_ray.counting.count_mutants(modules, operators)

    LOG.info('Mutation counts: %s', counts)
    LOG.info('Total test runs: %s',
             sum(itertools.chain(
                 *(d.values() for d in counts.values()))))

    results = [
        cosmic_ray.worker.worker_task.delay(module.__name__,
                                            opname,
                                            occurrence,
                                            configuration['--test-runner'],
                                            configuration['<test-dir>'],
                                            timeout)
        for module, ops in counts.items()
        for opname, count in ops.items()
        for occurrence in range(count)]

    for r in results:
        print(r.get())

def handle_test_runners(config):
    """usage: cosmic-ray test-runners

List the available test-runner plugins.
"""
    print('\n'.join(cosmic_ray.plugins.test_runner_names()))
    return 0

def handle_operators(config):
    """usage: cosmic-ray operators

List the available operator plugins.
"""
    print('\n'.join(cosmic_ray.plugins.operator_names()))
    return 0

def handle_worker(config):
    """usage: cosmic-ray worker [options] <module> <operator> <occurrence> <test-runner> <test-dir> <timeout>

options:
  --verbose           Produce verbose output
  --no-local-import   Disallow importing module from the current directory
"""
    if not config['--no-local-import']:
        sys.path.insert(0, '')

    operator = cosmic_ray.plugins.get_operator(config['<operator>'])
    cosmic_ray.testing.test_runner = cosmic_ray.plugins.get_test_runner(
        config['<test-runner>'],
        config['<test-dir>'])

    result = cosmic_ray.worker.worker(
        config['<module>'],
        operator,
        int(config['<occurrence>']),
        cosmic_ray.testing.test_runner,
        float(config['<timeout>']))
    sys.stdout.write(
        json.dumps(result,
                   cls=cosmic_ray.json_util.JSONEncoder))

COMMAND_HANDLER_MAP = {
    'baseline':     handle_baseline,
    'help':         handle_help,
    'load':         handle_load,
    'run':          handle_run,
    'test-runners': handle_test_runners,
    'operators':    handle_operators,
    'worker':       handle_worker,
}

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    configuration = docopt.docopt(
        OPTIONS,
        argv=argv,
        options_first=True,
        version='cosmic-ray v.2')
    if configuration['--verbose']:
        logging.basicConfig(level=logging.INFO)

    command = configuration['<command>']
    if command is None:
        command == 'help'

    try:
        handler = COMMAND_HANDLER_MAP[command]
    except KeyError:
        LOG.error('"{}" is not a valid cosmic-ray command'.format(command))
        handler = handle_help
        argv = ['help']

    sub_config = docopt.docopt(
        handler.__doc__,
        argv,
        version='cosmic-ray v.2')

    sys.exit(handler(sub_config))

if __name__ == '__main__':
    main()
<commit_msg>Created "counts" sub-command.<commit_after>"""This is the command-line program for cosmic ray.

Here we manage command-line parsing and launching of the internal
machinery that does mutation testing.
"""
import itertools
import json
import logging
import pprint
import sys

import docopt
import transducer.functional
import transducer.lazy
import transducer.transducers

import cosmic_ray.counting
import cosmic_ray.modules
import cosmic_ray.json_util
import cosmic_ray.worker
import cosmic_ray.testing
import cosmic_ray.timing

LOG = logging.getLogger()

OPTIONS = """cosmic-ray

Usage: cosmic-ray [--verbose] [--help] <command> [<args> ...]

options:
  --help     Show this screen.
  --verbose  Produce more verbose output

Available commands:
  load
  operators
  run
  test-runners
  worker

See 'cosmic-ray help <command>' for help on specific commands.
"""

# This is really an experiment in using transducers in "the real
# world". You could accomplish the same parsing goals in fewer lines
# (and probably more quckly) using more traditional means. But this
# approach does have a certain charm and elegance to it.
REMOVE_COMMENTS = transducer.transducers.mapping(lambda x: x.split('#')[0])
REMOVE_WHITESPACE = transducer.transducers.mapping(str.strip)
NON_EMPTY = transducer.transducers.filtering(bool)
CONFIG_FILE_PARSER = transducer.functional.compose(REMOVE_COMMENTS,
                                                   REMOVE_WHITESPACE,
                                                   NON_EMPTY)

def _load_file(config_file):
    """Read configuration from a file.

    This reads `config_file`, yielding each non-empty line with
    whitespace and comments stripped off.
    """
    with open(config_file, 'rt', encoding='utf-8') as f:
        yield from transducer.lazy.transduce(CONFIG_FILE_PARSER, f)

def handle_help(config):
    """usage: cosmic-ray help [<command>]

Get help on a specific command.
"""
    command = config['<command>']
    if not command:
        options = OPTIONS
    elif command not in COMMAND_HANDLER_MAP:
        LOG.error('"{}" is not a valid cosmic-ray command'.format(command))
        options = OPTIONS
    else:
        options = COMMAND_HANDLER_MAP[command].__doc__

    return docopt.docopt(options,
                         ['--help'],
                         version='cosmic-ray v.2')

def handle_load(config):
    """usage: cosmic-ray load <config-file>

Load a command configuration from a file and run it.
    """
    filename = config['<config-file>']
    argv = _load_file(filename)
    return main(argv=list(argv))

def handle_baseline(configuration):
    """usage: cosmic-ray baseline [options] <top-module> <test-dir>

options:
  --verbose           Produce verbose output
  --no-local-import   Allow importing module from the current directory
  --test-runner=R     Test-runner plugin to use [default: unittest]
"""
    sys.path.insert(0, '')
    test_runner = cosmic_ray.plugins.get_test_runner(
            configuration['--test-runner'],
            configuration['<test-dir>'])

    test_runner()

def handle_run(configuration):
    """usage: cosmic-ray run [options] [--exclude-modules=P ...] (--timeout=T | --baseline=M) <top-module> <test-dir>

options:
  --verbose           Produce verbose output
  --no-local-import   Allow importing module from the current directory
  --test-runner=R     Test-runner plugin to use [default: unittest]
  --exclude-modules=P Pattern of module names to exclude from mutation
  --num-testers=N     Number of concurrent testers to run (0 = os.cpu_count()) [default: 0]
"""
    # This lets us import modules from the current directory. Should probably
    # be optional, and needs to also be applied to workers!
    sys.path.insert(0, '')

    if configuration['--timeout'] is not None:
        timeout = float(configuration['--timeout'])
    else:
        baseline_mult = float(configuration['--baseline'])
        assert baseline_mult is not None
        timeout = baseline_mult * cosmic_ray.timing.run_baseline(
            configuration['--test-runner'],
            configuration['<top-module>'],
            configuration['<test-dir>'])

    LOG.info('timeout = {} seconds'.format(timeout))

    modules = cosmic_ray.modules.filtered_modules(
        cosmic_ray.modules.find_modules(configuration['<top-module>']),
        configuration['--exclude-modules'])

    operators = cosmic_ray.plugins.operator_names()

    counts = cosmic_ray.counting.count_mutants(modules, operators)

    results = [
        cosmic_ray.worker.worker_task.delay(module.__name__,
                                            opname,
                                            occurrence,
                                            configuration['--test-runner'],
                                            configuration['<test-dir>'],
                                            timeout)
        for module, ops in counts.items()
        for opname, count in ops.items()
        for occurrence in range(count)]

    for r in results:
        print(r.get())

def handle_counts(configuration):
    """usage: cosmic-ray counts [options] [--exclude-modules=P ...] <top-module>

options:
  --no-local-import   Allow importing module from the current directory
  --test-runner=R     Test-runner plugin to use [default: unittest]
  --exclude-modules=P Pattern of module names to exclude from mutation
  --num-testers=N     Number of concurrent testers to run (0 = os.cpu_count()) [default: 0]
"""
    sys.path.insert(0, '')

    modules = cosmic_ray.modules.filtered_modules(
        cosmic_ray.modules.find_modules(configuration['<top-module>']),
        configuration['--exclude-modules'])

    operators = cosmic_ray.plugins.operator_names()

    counts = cosmic_ray.counting.count_mutants(modules, operators)

    print('[Counts]')
    pprint.pprint(counts)
    print('\n[Total test runs]\n',
          sum(itertools.chain(
              *(d.values() for d in counts.values()))))

def handle_test_runners(config):
    """usage: cosmic-ray test-runners

List the available test-runner plugins.
"""
    print('\n'.join(cosmic_ray.plugins.test_runner_names()))
    return 0

def handle_operators(config):
    """usage: cosmic-ray operators

List the available operator plugins.
"""
    print('\n'.join(cosmic_ray.plugins.operator_names()))
    return 0

def handle_worker(config):
    """usage: cosmic-ray worker [options] <module> <operator> <occurrence> <test-runner> <test-dir> <timeout>

options:
  --verbose           Produce verbose output
  --no-local-import   Disallow importing module from the current directory
"""
    if not config['--no-local-import']:
        sys.path.insert(0, '')

    operator = cosmic_ray.plugins.get_operator(config['<operator>'])
    cosmic_ray.testing.test_runner = cosmic_ray.plugins.get_test_runner(
        config['<test-runner>'],
        config['<test-dir>'])

    result = cosmic_ray.worker.worker(
        config['<module>'],
        operator,
        int(config['<occurrence>']),
        cosmic_ray.testing.test_runner,
        float(config['<timeout>']))
    sys.stdout.write(
        json.dumps(result,
                   cls=cosmic_ray.json_util.JSONEncoder))

COMMAND_HANDLER_MAP = {
    'baseline':     handle_baseline,
    'counts':       handle_counts,
    'help':         handle_help,
    'load':         handle_load,
    'run':          handle_run,
    'test-runners': handle_test_runners,
    'operators':    handle_operators,
    'worker':       handle_worker,
}

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    configuration = docopt.docopt(
        OPTIONS,
        argv=argv,
        options_first=True,
        version='cosmic-ray v.2')
    if configuration['--verbose']:
        logging.basicConfig(level=logging.INFO)

    command = configuration['<command>']
    if command is None:
        command == 'help'

    try:
        handler = COMMAND_HANDLER_MAP[command]
    except KeyError:
        LOG.error('"{}" is not a valid cosmic-ray command'.format(command))
        handler = handle_help
        argv = ['help']

    sub_config = docopt.docopt(
        handler.__doc__,
        argv,
        version='cosmic-ray v.2')

    sys.exit(handler(sub_config))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import os
import re
import subprocess
from datetime import datetime

from elasticsearch import Elasticsearch, helpers
from elasticsearch.client.xpack import SqlClient
from elasticsearch.helpers import BulkIndexError
from pandas import DataFrame
from pandas.io.json import json_normalize
from ray.tune.logger import Logger

def create_elastic_client(**kwargs):
    """
    Create and configure :class:`elasticsearch.Elasticsearch` client

    The following environment variables are used to configure the client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored. ("cloud_id")
        - **ELASTIC_HOSTS**: List of nodes we should connect to. ("hosts")
        - **ELASTIC_AUTH**: http auth information ("http_auth")

    :param kwargs: Used to override the environment variables or pass extra
                   parameters to the :class:`elasticsearch.Elasticsearch`.
    :type kwargs: dict
    :return: Configured :class:`elasticsearch.Elasticsearch` client
    :rtype: :class:`Elasticsearch`
    """
    hosts = os.environ.get("ELASTIC_HOST")
    hosts = None if hosts is None else hosts.split(",")
    elasticsearch_args = {
        "cloud_id": os.environ.get("ELASTIC_CLOUD_ID"),
        "hosts": hosts,
        "http_auth": os.environ.get("ELASTIC_AUTH")
    }

    # Update elasticsearch client arguments from configuration if present
    elasticsearch_args.update(kwargs)
    return Elasticsearch(**elasticsearch_args)

class ElasticsearchLogger(Logger):
    """
    Elasticsearch Logging interface for `ray.tune`.

    This logger will upload all the results to an elasticsearch index.
    In addition to the regular ray tune log entry, this logger will add the
    the last git commit information and the current `logdir` to the results.

    The following environment variables are used to configure the
    :class:`elasticsearch.Elasticsearch` client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored
        - **ELASTIC_HOST**: hostname of the elasticsearch node
        - **ELASTIC_AUTH**: http auth information ('user:password')

    You may override the environment variables or pass extra parameters to the
    :class:`elasticsearch.Elasticsearch` client for the specific experiment
    using the "elasticsearch_client" configuration key.

    The elasticsearch index name is based on the current results root path. You
    may override this behavior and use a specific index name for your experiment
    using the configuration key `elasticsearch_index`.
    """

    def _init(self):
        elasticsearch_args = self.config.get("elasticsearch_client", {})
        self.client = create_elastic_client(**elasticsearch_args)

        # Save git information
        self.git_remote = subprocess.check_output(
            ["git", "ls-remote", "--get-url"]).decode("ascii").strip()
        self.git_branch = subprocess.check_output(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("ascii").strip()
        self.git_sha = subprocess.check_output(
            ["git", "rev-parse", "HEAD"]).decode("ascii").strip()
        self.git_user = subprocess.check_output(
            ["git", "log", "-n", "1", "--pretty=format:%an"]).decode("ascii").strip()

        # Check for elasticsearch index name in configuration
        index_name = self.config.get("elasticsearch_index")
        if index_name is None:
            # Create default index name based on log path and git repo name
            git_root = subprocess.check_output(
                ["git", "rev-parse", "--show-toplevel"]).decode("ascii").strip()
            repo_name = os.path.basename(self.git_remote).rstrip(".git")
            path_name = os.path.relpath(self.config["path"], git_root)
            index_name = os.path.join(repo_name, path_name)

            # slugify index name
            index_name = re.sub(r"[\W_]+", "-", index_name)

        self.index_name = index_name

        self.logdir = os.path.basename(self.logdir)
        self.experiment_name = self.config["name"]
        self.buffer = []

    def on_result(self, result):
        """Given a result, appends it to the existing log."""
        log_entry = {
            "git": {
                "remote": self.git_remote,
                "branch": self.git_branch,
                "sha": self.git_sha,
                "user": self.git_user
            },
            "logdir": self.logdir
        }
        # Convert timestamp to ISO-8601
        timestamp = result["timestamp"]
        result["timestamp"] = datetime.utcfromtimestamp(timestamp).isoformat()

        log_entry.update(result)
        self.buffer.append(log_entry)

    def close(self):
        self.flush()

    def flush(self):
        if len(self.buffer) > 0:
            results = helpers.parallel_bulk(client=self.client,
                                            actions=self.buffer,
                                            index=self.index_name,
                                            doc_type=self.experiment_name)
            errors = [status for success, status in results if not success]
            if errors:
                raise BulkIndexError("{} document(s) failed to index.".
                                     format(len(errors)), errors)

            self.buffer.clear()

def elastic_dsl(client, dsl, index, **kwargs):
    """
    Sends DSL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param dsl: Elasticsearch DSL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html  # noqa: E501
    :type dsl: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html # noqa: E501
    :type index: str

    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call
    :type kwargs: dict

    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    response = helpers.scan(client=client, query=dsl, index=index, **kwargs)
    data = []
    for row in response:
        # Normalize nested dicts in '_source' such as 'config' or 'git'
        source = json_normalize(row["_source"]) if "_source" in row else {}

        # Squeeze scalar fields returned as arrays in the response by the search API
        fields = row.get("fields", {})
        fields = {k: v[0] if len(v) == 1 else v for k, v in fields.items()}

        data.append({
            "_index": row["_index"],
            "_type": row["_type"],
            **fields,
            **source,
        })
    return DataFrame(data)

def elastic_sql(client, sql, index, **kwargs):
    """
    Sends SQL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param sql: Elasticsearch SQL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-sql.html  # noqa: E501
    :type sql: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html
    :type index: str
    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call

    :type kwargs: dict
    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    sql_client = SqlClient(client)
    # FIXME: SQL API does not support arrays. See https://github.com/elastic/elasticsearch/issues/33204  # noqa: E501
    # For now we translate the SQL into elasticsearch DSL query and use the
    # 'search' API to fetch the results
    dsl = sql_client.translate({"query": sql})

    # Ignore score
    if "query" in dsl:
        query = dsl["query"]
        dsl["query"] = {"constant_score": {"filter": query}}

    return elastic_dsl(client, dsl, index, **kwargs)
<commit_msg>Fix Elastic Logger env variable<commit_after>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import os
import re
import subprocess
from datetime import datetime

from elasticsearch import Elasticsearch, helpers
from elasticsearch.client.xpack import SqlClient
from elasticsearch.helpers import BulkIndexError
from pandas import DataFrame
from pandas.io.json import json_normalize
from ray.tune.logger import Logger

def create_elastic_client(**kwargs):
    """
    Create and configure :class:`elasticsearch.Elasticsearch` client

    The following environment variables are used to configure the client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored. ("cloud_id")
        - **ELASTIC_HOSTS**: List of nodes we should connect to. ("hosts")
        - **ELASTIC_AUTH**: http auth information ("http_auth")

    :param kwargs: Used to override the environment variables or pass extra
                   parameters to the :class:`elasticsearch.Elasticsearch`.
    :type kwargs: dict
    :return: Configured :class:`elasticsearch.Elasticsearch` client
    :rtype: :class:`Elasticsearch`
    """
    hosts = os.environ.get("ELASTIC_HOSTS")
    hosts = None if hosts is None else hosts.split(",")
    elasticsearch_args = {
        "cloud_id": os.environ.get("ELASTIC_CLOUD_ID"),
        "hosts": hosts,
        "http_auth": os.environ.get("ELASTIC_AUTH")
    }

    # Update elasticsearch client arguments from configuration if present
    elasticsearch_args.update(kwargs)
    return Elasticsearch(**elasticsearch_args)

class ElasticsearchLogger(Logger):
    """
    Elasticsearch Logging interface for `ray.tune`.

    This logger will upload all the results to an elasticsearch index.
    In addition to the regular ray tune log entry, this logger will add the
    the last git commit information and the current `logdir` to the results.

    The following environment variables are used to configure the
    :class:`elasticsearch.Elasticsearch` client:

        - **ELASTIC_CLOUD_ID**: The Cloud ID from ElasticCloud. Other host
                                connection params will be ignored
        - **ELASTIC_HOST**: hostname of the elasticsearch node
        - **ELASTIC_AUTH**: http auth information ('user:password')

    You may override the environment variables or pass extra parameters to the
    :class:`elasticsearch.Elasticsearch` client for the specific experiment
    using the "elasticsearch_client" configuration key.

    The elasticsearch index name is based on the current results root path. You
    may override this behavior and use a specific index name for your experiment
    using the configuration key `elasticsearch_index`.
    """

    def _init(self):
        elasticsearch_args = self.config.get("elasticsearch_client", {})
        self.client = create_elastic_client(**elasticsearch_args)

        # Save git information
        self.git_remote = subprocess.check_output(
            ["git", "ls-remote", "--get-url"]).decode("ascii").strip()
        self.git_branch = subprocess.check_output(
            ["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode("ascii").strip()
        self.git_sha = subprocess.check_output(
            ["git", "rev-parse", "HEAD"]).decode("ascii").strip()
        self.git_user = subprocess.check_output(
            ["git", "log", "-n", "1", "--pretty=format:%an"]).decode("ascii").strip()

        # Check for elasticsearch index name in configuration
        index_name = self.config.get("elasticsearch_index")
        if index_name is None:
            # Create default index name based on log path and git repo name
            git_root = subprocess.check_output(
                ["git", "rev-parse", "--show-toplevel"]).decode("ascii").strip()
            repo_name = os.path.basename(self.git_remote).rstrip(".git")
            path_name = os.path.relpath(self.config["path"], git_root)
            index_name = os.path.join(repo_name, path_name)

            # slugify index name
            index_name = re.sub(r"[\W_]+", "-", index_name)

        self.index_name = index_name

        self.logdir = os.path.basename(self.logdir)
        self.experiment_name = self.config["name"]
        self.buffer = []

    def on_result(self, result):
        """Given a result, appends it to the existing log."""
        log_entry = {
            "git": {
                "remote": self.git_remote,
                "branch": self.git_branch,
                "sha": self.git_sha,
                "user": self.git_user
            },
            "logdir": self.logdir
        }
        # Convert timestamp to ISO-8601
        timestamp = result["timestamp"]
        result["timestamp"] = datetime.utcfromtimestamp(timestamp).isoformat()

        log_entry.update(result)
        self.buffer.append(log_entry)

    def close(self):
        self.flush()

    def flush(self):
        if len(self.buffer) > 0:
            results = helpers.parallel_bulk(client=self.client,
                                            actions=self.buffer,
                                            index=self.index_name,
                                            doc_type=self.experiment_name)
            errors = [status for success, status in results if not success]
            if errors:
                raise BulkIndexError("{} document(s) failed to index.".
                                     format(len(errors)), errors)

            self.buffer.clear()

def elastic_dsl(client, dsl, index, **kwargs):
    """
    Sends DSL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param dsl: Elasticsearch DSL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html  # noqa: E501
    :type dsl: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html # noqa: E501
    :type index: str

    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call
    :type kwargs: dict

    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    response = helpers.scan(client=client, query=dsl, index=index, **kwargs)
    data = []
    for row in response:
        # Normalize nested dicts in '_source' such as 'config' or 'git'
        source = json_normalize(row["_source"]) if "_source" in row else {}

        # Squeeze scalar fields returned as arrays in the response by the search API
        fields = row.get("fields", {})
        fields = {k: v[0] if len(v) == 1 else v for k, v in fields.items()}

        data.append({
            "_index": row["_index"],
            "_type": row["_type"],
            **fields,
            **source,
        })
    return DataFrame(data)

def elastic_sql(client, sql, index, **kwargs):
    """
    Sends SQL query to elasticsearch and returns the results as a
    :class:`pandas.DataFrame`.

    :param client: Configured elasticseach client. See :func:`create_elastic_client`
    :type client: :class:`elasticseach.Elasticsearch`
    :param sql: Elasticsearch SQL query statement
                See https://www.elastic.co/guide/en/elasticsearch/reference/current/xpack-sql.html  # noqa: E501
    :type sql: str
    :param index: Index pattern. Usually the same as 'from' part of the SQL
                  See https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html
    :type index: str
    :param kwargs: Any additional keyword arguments will be passed to the initial
                   :meth:`elasticsearch.Elasticsearch.search` call

    :type kwargs: dict
    :return: results as a :class:`pandas.DataFrame`.
    :rtype: :class:`pandas.DataFrame`
    """
    sql_client = SqlClient(client)
    # FIXME: SQL API does not support arrays. See https://github.com/elastic/elasticsearch/issues/33204  # noqa: E501
    # For now we translate the SQL into elasticsearch DSL query and use the
    # 'search' API to fetch the results
    dsl = sql_client.translate({"query": sql})

    # Ignore score
    if "query" in dsl:
        query = dsl["query"]
        dsl["query"] = {"constant_score": {"filter": query}}

    return elastic_dsl(client, dsl, index, **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Tests to try out table. Potentially ephemeral

'''

from salt.transport import table

def test_table():
    bob_pub =  table.Public()
    print bob_pub.backend
    print bob_pub.sec_backend
    print bob_pub.keydata
    print bob_pub.public
    print bob_pub.secret

if __name__ == "__main__":
    test_table()
<commit_msg>Fix white-space lint issues<commit_after># -*- coding: utf-8 -*-
'''
Tests to try out table. Potentially ephemeral

'''

# Import Salt Libs
from salt.transport import table

def test_table():
    bob_pub = table.Public()
    print bob_pub.backend
    print bob_pub.sec_backend
    print bob_pub.keydata
    print bob_pub.public
    print bob_pub.secret

if __name__ == "__main__":
    test_table()
<|endoftext|>


<commit_before># entities and collections of entities (clients, devices, etc) go here

__all__ = []
<commit_msg>Removed entities for now<commit_after><|endoftext|>