<commit_before># coding=utf-8
import os
import tempfile
import shutil

from django.test import TransactionTestCase
from django.core.files.uploadedfile import UploadedFile
from django.contrib.auth.models import Group

from hs_core.testing import MockIRODSTestCaseMixin
from hs_core import hydroshare
from hs_file_types.utils import raster_extract_metadata
from hs_file_types.models import GeoRasterLogicalFile

class RasterFileTypeMetaData(MockIRODSTestCaseMixin, TransactionTestCase):
    def setUp(self):
        super(RasterFileTypeMetaData, self).setUp()
        self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
        self.user = hydroshare.create_account(
            'user1@nowhere.com',
            username='user1',
            first_name='Creator_FirstName',
            last_name='Creator_LastName',
            superuser=False,
            groups=[self.group]
        )

        self.composite_resource = hydroshare.create_resource(
            resource_type='CompositeResource',
            owner=self.user,
            title='Test Raster File Metadata'
        )

        self.temp_dir = tempfile.mkdtemp()
        self.raster_file_name = 'small_logan.tif'
        self.raster_file = 'hs_file_types/tests/{}'.format(self.raster_file_name)
        target_temp_raster_file = os.path.join(self.temp_dir, self.raster_file_name)
        shutil.copy(self.raster_file, target_temp_raster_file)
        self.raster_file_obj = open(target_temp_raster_file, 'r')

    def tearDown(self):
        super(RasterFileTypeMetaData, self).tearDown()
        if os.path.exists(self.temp_dir):
            shutil.rmtree(self.temp_dir)

    def test_metadata_extraction(self):
        self.raster_file_obj = open(self.raster_file, 'r')
        self.composite_resource = hydroshare.create_resource(
            resource_type='CompositeResource',
            owner=self.user,
            title='Test Raster File Type Metadata',
            files=(self.raster_file_obj,)
        )
        self.assertEqual(self.composite_resource.files.all().count(), 1)
        res_file = self.composite_resource.files.first()

        # check that the resource file is associated with GenericLogicalFile by default
        self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
        # no metadata associated with generirclogicalfile
        self.assertEqual(res_file.logical_file.has_metadata, False)
        raster_extract_metadata(self.composite_resource, res_file.id, self.user)

        # test the resource now has 2 files (vrt file added as part of metadata extraction)
        self.assertEqual(self.composite_resource.files.all().count(), 2)

        # check that the 2 resource files are now associated with GeoRasterLogicalFile
        for res_file in self.composite_resource.files.all():
            self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")

        # check that the logicalfile is associated with 2 files
        self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
        logical_file = res_file.logical_file
        self.assertEqual(logical_file.has_metadata, True)
        self.assertEqual(logical_file.files.all().count(), 2)
        self.assertEqual(set(self.composite_resource.files.all()),
                         set(logical_file.files.all()))

        # there should be 2 format elements associated with resource
        self.assertEqual(self.composite_resource.metadata.formats.all().count(), 2)
        self.assertEqual(
            self.composite_resource.metadata.formats.all().filter(value='application/vrt').count(),
            1)
        self.assertEqual(self.composite_resource.metadata.formats.all().filter(
            value='image/tiff').count(), 1)

        # test extracted metadata for the file type

        # there should be 1 coverage element - box type
        self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
        self.assertEqual(logical_file.metadata.coverages.all().filter(type='box').count(), 1)

        box_coverage = logical_file.metadata.coverages.all().filter(type='box').first()
        self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326')
        self.assertEqual(box_coverage.value['units'], 'Decimal degrees')
        # TODO: adjust these values to make the test pass
        self.assertEqual(box_coverage.value['northlimit'], 42.11071605314457)
        self.assertEqual(box_coverage.value['eastlimit'], -111.45699925047542)
        self.assertEqual(box_coverage.value['southlimit'], 41.66417975061928)
        self.assertEqual(box_coverage.value['westlimit'], -111.81761887121905)

        # testing extended metadata element: original coverage
        ori_coverage = logical_file.metadata.originalCoverage
        self.assertNotEqual(ori_coverage, None)

        # TODO: adjust these values to make the test pass
        self.assertEqual(ori_coverage.value['northlimit'], 4662392.446916306)
        self.assertEqual(ori_coverage.value['eastlimit'], 461954.01909127034)
        self.assertEqual(ori_coverage.value['southlimit'], 4612592.446916306)
        self.assertEqual(ori_coverage.value['westlimit'], 432404.01909127034)
        self.assertEqual(ori_coverage.value['units'], 'meter')
        self.assertEqual(ori_coverage.value['projection'],
                         'NAD83 / UTM zone 12N Transverse_Mercator')

        # testing extended metadata element: cell information
        cell_info = logical_file.metadata.cellInformation
        # TODO: adjust these values to make the test pass
        self.assertEqual(cell_info.rows, 1660)
        self.assertEqual(cell_info.columns, 985)
        self.assertEqual(cell_info.cellSizeXValue, 30.0)
        self.assertEqual(cell_info.cellSizeYValue, 30.0)
        self.assertEqual(cell_info.cellDataType, 'Float32')

        # testing extended metadata element: band information
        self.assertEqual(logical_file.metadata.bandInformation.count(), 1)
        band_info = logical_file.metadata.bandInformation.first()
        # TODO: adjust these values to make the test pass
        self.assertEqual(band_info.noDataValue, '-3.40282346639e+38')
        self.assertEqual(band_info.maximumValue, '3031.44311523')
        self.assertEqual(band_info.minimumValue, '1358.33459473')

<commit_msg>[#1477] added more unit tests for raster file type<commit_after># coding=utf-8
import os
import tempfile
import shutil

from django.test import TransactionTestCase
from django.db import IntegrityError
from django.contrib.auth.models import Group
from django.core.exceptions import ValidationError

from hs_core.testing import MockIRODSTestCaseMixin
from hs_core import hydroshare
from hs_file_types.utils import raster_extract_metadata
from hs_file_types.models import GeoRasterLogicalFile

class RasterFileTypeMetaData(MockIRODSTestCaseMixin, TransactionTestCase):
    def setUp(self):
        super(RasterFileTypeMetaData, self).setUp()
        self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
        self.user = hydroshare.create_account(
            'user1@nowhere.com',
            username='user1',
            first_name='Creator_FirstName',
            last_name='Creator_LastName',
            superuser=False,
            groups=[self.group]
        )

        self.composite_resource = hydroshare.create_resource(
            resource_type='CompositeResource',
            owner=self.user,
            title='Test Raster File Metadata'
        )

        self.temp_dir = tempfile.mkdtemp()
        self.raster_file_name = 'small_logan.tif'
        self.raster_file = 'hs_file_types/tests/{}'.format(self.raster_file_name)
        target_temp_raster_file = os.path.join(self.temp_dir, self.raster_file_name)
        shutil.copy(self.raster_file, target_temp_raster_file)
        self.raster_file_obj = open(target_temp_raster_file, 'r')

    def tearDown(self):
        super(RasterFileTypeMetaData, self).tearDown()
        if os.path.exists(self.temp_dir):
            shutil.rmtree(self.temp_dir)

    def test_metadata_extraction(self):
        self.raster_file_obj = open(self.raster_file, 'r')
        self._create_composite_resource()

        self.assertEqual(self.composite_resource.files.all().count(), 1)
        res_file = self.composite_resource.files.first()

        # check that the resource file is associated with GenericLogicalFile by default
        self.assertEqual(res_file.logical_file_type_name, "GenericLogicalFile")
        # no metadata associated with generirclogicalfile
        self.assertEqual(res_file.logical_file.has_metadata, False)
        raster_extract_metadata(self.composite_resource, res_file.id, self.user)

        # test the resource now has 2 files (vrt file added as part of metadata extraction)
        self.assertEqual(self.composite_resource.files.all().count(), 2)

        # check that the 2 resource files are now associated with GeoRasterLogicalFile
        for res_file in self.composite_resource.files.all():
            self.assertEqual(res_file.logical_file_type_name, "GeoRasterLogicalFile")

        # check that the logicalfile is associated with 2 files
        self.assertEqual(GeoRasterLogicalFile.objects.count(), 1)
        logical_file = res_file.logical_file
        self.assertEqual(logical_file.dataset_name, 'small_logan')
        self.assertEqual(logical_file.has_metadata, True)
        self.assertEqual(logical_file.files.all().count(), 2)
        self.assertEqual(set(self.composite_resource.files.all()),
                         set(logical_file.files.all()))

        # there should be 2 format elements associated with resource
        self.assertEqual(self.composite_resource.metadata.formats.all().count(), 2)
        self.assertEqual(
            self.composite_resource.metadata.formats.all().filter(value='application/vrt').count(),
            1)
        self.assertEqual(self.composite_resource.metadata.formats.all().filter(
            value='image/tiff').count(), 1)

        # test extracted metadata for the file type

        # there should be 1 coverage element - box type
        self.assertEqual(logical_file.metadata.coverages.all().count(), 1)
        self.assertEqual(logical_file.metadata.coverages.all().filter(type='box').count(), 1)

        box_coverage = logical_file.metadata.coverages.all().filter(type='box').first()
        self.assertEqual(box_coverage.value['projection'], 'WGS 84 EPSG:4326')
        self.assertEqual(box_coverage.value['units'], 'Decimal degrees')
        self.assertEqual(box_coverage.value['northlimit'], 42.049364058252266)
        self.assertEqual(box_coverage.value['eastlimit'], -111.57773718106195)
        self.assertEqual(box_coverage.value['southlimit'], 41.987884327209976)
        self.assertEqual(box_coverage.value['westlimit'], -111.69756293084055)

        # testing extended metadata element: original coverage
        ori_coverage = logical_file.metadata.originalCoverage
        self.assertNotEqual(ori_coverage, None)
        self.assertEqual(ori_coverage.value['northlimit'], 4655492.446916306)
        self.assertEqual(ori_coverage.value['eastlimit'], 452144.01909127034)
        self.assertEqual(ori_coverage.value['southlimit'], 4648592.446916306)
        self.assertEqual(ori_coverage.value['westlimit'], 442274.01909127034)
        self.assertEqual(ori_coverage.value['units'], 'meter')
        self.assertEqual(ori_coverage.value['projection'],
                         'NAD83 / UTM zone 12N Transverse_Mercator')

        # testing extended metadata element: cell information
        cell_info = logical_file.metadata.cellInformation
        # TODO: adjust these values to make the test pass
        self.assertEqual(cell_info.rows, 230)
        self.assertEqual(cell_info.columns, 329)
        self.assertEqual(cell_info.cellSizeXValue, 30.0)
        self.assertEqual(cell_info.cellSizeYValue, 30.0)
        self.assertEqual(cell_info.cellDataType, 'Float32')

        # testing extended metadata element: band information
        self.assertEqual(logical_file.metadata.bandInformation.count(), 1)
        band_info = logical_file.metadata.bandInformation.first()
        self.assertEqual(band_info.noDataValue, '-3.40282346639e+38')
        self.assertEqual(band_info.maximumValue, '2880.00708008')
        self.assertEqual(band_info.minimumValue, '1870.63659668')

    def test_metadata_CRUD(self):
        self.raster_file_obj = open(self.raster_file, 'r')
        self._create_composite_resource()

        self.assertEqual(self.composite_resource.files.all().count(), 1)
        res_file = self.composite_resource.files.first()
        # extract metadata
        raster_extract_metadata(self.composite_resource, res_file.id, self.user)
        res_file = self.composite_resource.files.first()

        # test that we can update raster specific metadata at the file level

        # delete default original coverage metadata
        logical_file = res_file.logical_file
        self.assertNotEquals(logical_file.metadata.originalCoverage, None)
        logical_file.metadata.originalCoverage.delete()

        # create new original coverage metadata with meaningful value
        value = {"northlimit": 12, "projection": "transverse_mercator", "units": "meter",
                 "southlimit": 10,
                 "eastlimit": 23, "westlimit": 2}
        logical_file.metadata.create_element('originalcoverage', value=value)

        self.assertEquals(logical_file.metadata.originalCoverage.value, value)

        # multiple original coverage elements are not allowed - should raise exception
        with self.assertRaises(IntegrityError):
            logical_file.metadata.create_element('originalcoverage', value=value)

        # delete default cell information element
        self.assertNotEquals(logical_file.metadata.cellInformation, None)
        logical_file.metadata.cellInformation.delete()

        # create new cell information metadata with meaningful value
        logical_file.metadata.create_element('cellinformation', name='cellinfo',
                                             cellDataType='Float32',
                                             rows=1660, columns=985, cellSizeXValue=30.0,
                                             cellSizeYValue=30.0,
                                             )

        cell_info = logical_file.metadata.cellInformation
        self.assertEquals(cell_info.rows, 1660)
        self.assertEquals(cell_info.columns, 985)
        self.assertEquals(cell_info.cellSizeXValue, 30.0)
        self.assertEquals(cell_info.cellSizeYValue, 30.0)
        self.assertEquals(cell_info.cellDataType, 'Float32')
        # multiple cell Information elements are not allowed - should raise exception
        with self.assertRaises(IntegrityError):
            logical_file.metadata.create_element('cellinformation', name='cellinfo',
                                                 cellDataType='Float32',
                                                 rows=1660, columns=985,
                                                 cellSizeXValue=30.0, cellSizeYValue=30.0,
                                                 )
        # delete default band information element
        self.assertNotEquals(logical_file.metadata.bandInformation, None)
        logical_file.metadata.bandInformation.first().delete()

        # create band information element with meaningful value
        logical_file.metadata.create_element('bandinformation', name='bandinfo',
                                             variableName='diginal elevation',
                                             variableUnit='meter',
                                             method='this is method',
                                             comment='this is comment',
                                             maximumValue=1000, minimumValue=0,
                                             noDataValue=-9999)

        band_info = logical_file.metadata.bandInformation.first()
        self.assertEquals(band_info.name, 'bandinfo')
        self.assertEquals(band_info.variableName, 'diginal elevation')
        self.assertEquals(band_info.variableUnit, 'meter')
        self.assertEquals(band_info.method, 'this is method')
        self.assertEquals(band_info.comment, 'this is comment')
        self.assertEquals(band_info.maximumValue, '1000')
        self.assertEquals(band_info.minimumValue, '0')
        self.assertEquals(band_info.noDataValue, '-9999')

        # multiple band information elements are allowed
        logical_file.metadata.create_element('bandinformation', name='bandinfo',
                                             variableName='diginal elevation2',
                                             variableUnit='meter',
                                             method='this is method',
                                             comment='this is comment',
                                             maximumValue=1000, minimumValue=0,
                                             noDataValue=-9999)
        self.assertEquals(logical_file.metadata.bandInformation.all().count(), 2)

        # test metadata delete

        # original coverage deletion is not allowed
        with self.assertRaises(ValidationError):
            logical_file.metadata.delete_element('originalcoverage',
                                                 logical_file.metadata.originalCoverage.id)

        # cell information deletion is not allowed
        with self.assertRaises(ValidationError):
            logical_file.metadata.delete_element('cellinformation',
                                                 logical_file.metadata.cellInformation.id)

        # band information deletion is not allowed
        with self.assertRaises(ValidationError):
            logical_file.metadata.delete_element('bandinformation',
                                                logical_file.metadata.bandInformation.first().id)

        # test metadata update

        # update original coverage element
        value_2 = {"northlimit": 12.5, "projection": "transverse_mercator", "units": "meter",
                   "southlimit": 10.5,
                   "eastlimit": 23.5, "westlimit": 2.5}
        logical_file.metadata.update_element('originalcoverage',
                                             logical_file.metadata.originalCoverage.id,
                                             value=value_2)

        self.assertEquals(logical_file.metadata.originalCoverage.value, value_2)

        # update cell info element
        logical_file.metadata.update_element('cellinformation',
                                             logical_file.metadata.cellInformation.id,
                                             name='cellinfo', cellDataType='Double',
                                             rows=166, columns=98,
                                             cellSizeXValue=3.0, cellSizeYValue=3.0,
                                            )

        cell_info = logical_file.metadata.cellInformation
        self.assertEquals(cell_info.rows, 166)
        self.assertEquals(cell_info.columns, 98)
        self.assertEquals(cell_info.cellSizeXValue, 3.0)
        self.assertEquals(cell_info.cellSizeYValue, 3.0)
        self.assertEquals(cell_info.cellDataType, 'Double')

        # update band info element
        logical_file.metadata.update_element('bandinformation',
                                             logical_file.metadata.bandInformation.first().id,
                                             name='bandinfo',
                                             variableName='precipitation',
                                             variableUnit='mm/h',
                                             method='this is method2',
                                             comment='this is comment2',
                                             maximumValue=1001, minimumValue=1,
                                             noDataValue=-9998
                                             )

        band_info = logical_file.metadata.bandInformation.first()
        self.assertEquals(band_info.name, 'bandinfo')
        self.assertEquals(band_info.variableName, 'precipitation')
        self.assertEquals(band_info.variableUnit, 'mm/h')
        self.assertEquals(band_info.method, 'this is method2')
        self.assertEquals(band_info.comment, 'this is comment2')
        self.assertEquals(band_info.maximumValue, '1001')
        self.assertEquals(band_info.minimumValue, '1')
        self.assertEquals(band_info.noDataValue, '-9998')

    def test_file_metadata_on_file_delete(self):
        # test that when any file in GeorasterFileType is deleted
        # all metadata associated with GeoRasterFileType is deleted
        # test for both .tif and .vrt delete
        pass

    def test_file_metadata_on_logical_file_delete(self):
        # test that when the GeorasterFileType is deleted
        # all metadata associated with GeoRasterFileType is deleted
        pass

    def test_file_metadata_on_resource_delete(self):
        # test that when the composite resource is deleted
        # all metadata associated with GeoRasterFileType is deleted
        pass

    def test_logical_file_delete(self):
        # test that when an instance GeorasterFileType is deleted
        # all files associated with GeoRasterFileType is deleted
        pass

    def test_content_file_delete(self):
        # test that when any file that is part of an instance GeoRasterFileType is deleted
        # all files associated with GeoRasterFileType is deleted
        pass

    def _create_composite_resource(self):
        self.composite_resource = hydroshare.create_resource(
            resource_type='CompositeResource',
            owner=self.user,
            title='Test Raster File Type Metadata',
            files=(self.raster_file_obj,)
        )
<|endoftext|>


<commit_before>#
#  DIALS viewer_frame
#
#  Copyright (C) 2014 Diamond Light Source
#
#  Author: Luis Fuentes-Montero (Luiso)
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package."

import wx
from wx.lib.buttons import GenBitmapTextButton
from dials.viewer.viewer_utilities \
     import np_to_bmp

from dials.viewer.reflection_data_navigator import table_s_navigator

class ReflectionFrame(wx.Frame):
  def __init__(self, *args, **kwargs):
    wx.Frame.__init__(self, *args, **kwargs)

    #self.MaxImageSizeX = 280
    #self.MaxImageSizeY = 150
    self.MaxImageSizeX = 320
    self.MaxImageSizeY = 260

    nxt_ref = "Reflection -->"
    btn_nxt_refl = wx.Button(self, -1, nxt_ref)
    prv_ref = "<-- Reflection"
    btn_prv_refl = wx.Button(self, -1, prv_ref)
    btn_read = wx.Button(self, -1, "Reflection #")

    radio1 = wx.RadioButton(self, -1, "data, background, mask"
                           , style = wx.RB_GROUP)
    radio2 = wx.RadioButton(self, -1, "3 layers of data")
    radio3 = wx.RadioButton(self, -1, "3 layers of background")
    radio4 = wx.RadioButton(self, -1, "3 layers of mask")

    img_tmp = wx.ArtProvider.GetBitmap(wx.ART_GO_FORWARD, wx.ART_OTHER, (32, 32))
    btn_nxt_slice = wx.lib.buttons.GenBitmapTextButton(self
                   , label = 'prueba', bitmap = img_tmp)

    btn_prv_slice = wx.Button(self, -1, "<--  slice")

    self.data_txt_01 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))
    self.data_txt_02 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))
    self.data_txt_03 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))

    self.Image = [None, None, None]

    for indx in range(len(self.Image)):
      self.Image[indx] = wx.StaticBitmap(self, bitmap = wx.EmptyBitmap(
                                         self.MaxImageSizeX, self.MaxImageSizeY))

    self.text01 = wx.TextCtrl(self, -1, "enter number", size = (100, -1))

    u_box = wx.BoxSizer(wx.VERTICAL)
    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_prv_refl, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(btn_nxt_refl, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_read, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(self.text01, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    u_box.Add(radio1, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio2, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio3, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio4, 0, wx.LEFT | wx.ALL,3)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_prv_slice, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(btn_nxt_slice, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    h_box = wx.BoxSizer(wx.HORIZONTAL)

    h_box.Add(u_box)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(self.Image[0], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)

    div_h_box.Add(self.Image[1], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)

    div_h_box.Add(self.Image[2], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)
    div_u_box = wx.BoxSizer(wx.VERTICAL)
    div_u_box.Add(div_h_box)
    div_u_box.Add(self.data_txt_01, 0, wx.CENTER | wx.ALL,3)
    div_u_box.Add(self.data_txt_02, 0, wx.CENTER | wx.ALL,3)
    div_u_box.Add(self.data_txt_03, 0, wx.CENTER | wx.ALL,3)
    h_box.Add(div_u_box)

    self.frame_scale = 0.4
    self.opt = 0

    self.sizing_counter = 0
    self.SetSizerAndFit(h_box)

    btn_nxt_refl.Bind(wx.EVT_BUTTON, self._DisplayNext_refl)
    btn_prv_refl.Bind(wx.EVT_BUTTON, self._DisplayPrev_refl)
    btn_nxt_slice.Bind(wx.EVT_BUTTON, self._DisplayNext_slice)
    btn_prv_slice.Bind(wx.EVT_BUTTON, self._DisplayPrev_slice)

    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio1, radio1)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio2, radio2)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio3, radio3)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio4, radio4)

    self.Bind(wx.EVT_BUTTON, self._read_num_from_txt, btn_read)

    self.Bind(wx.EVT_SIZE, self._OnSize)

    self.bmp = np_to_bmp()
    self.arr_img = [None, None, None]
    self.arr_ttl = [None, None, None]
    self.My_Img = [None, None, None]
    self.wx_Img = [None, None, None]
    wx.EVT_CLOSE(self, self._On_Close_Window)

  def tabl_to_frame(self, loc_tabl):
    self.tabl = table_s_navigator(loc_tabl)
    self._DisplayPrev_refl()
  def _DisplayNext_refl(self, event = None):
    self.tabl.next_Reflection()
    self._My_Update()
  def _DisplayPrev_refl(self, event = None):
    self.tabl.Previous_Reflection()
    self._My_Update()
  def _DisplayNext_slice(self, event = None):
    self.tabl.next_slice()
    self._My_Update()
  def _DisplayPrev_slice(self, event = None):
    self.tabl.Previous_slice()
    self._My_Update()

  def _read_num_from_txt(self, event = None):
    # TODO check if the user entered a number
    a = int(self.text01.GetValue())
    self.tabl.Jump_to_Reflection(a)
    self._My_Update()

  def _OnRadio1(self, event = None):
    self.opt = 0
    self._My_Update(request_new_data = True)

  def _OnRadio2(self, event = None):
    self.opt = 1
    self._My_Update(request_new_data = True)

  def _OnRadio3(self, event = None):
    self.opt = 2
    self._My_Update(request_new_data = True)

  def _OnRadio4(self, event = None):
    self.opt = 3
    self._My_Update(request_new_data = True)

  def _OnSize(self, event = None):
    if( self.sizing_counter > 5 ):
      siz_data = self.GetSize()
      #print "New size of window =", siz_data
      optm_aspec_ratio = 4.21
      #print "siz_data = ", siz_data[0], siz_data[1]
      #print "aspect ratio = ", float(siz_data[0])/ float(siz_data[1])
      aspec_ratio = float(siz_data[0])/ float(siz_data[1])

      if(aspec_ratio > optm_aspec_ratio):
        #print "use float(siz_data[1] (Height) to calculate new size"
        self.frame_scale = float(siz_data[1]) * 0.5 / 295.0
        #(1227, 295)
      else:
        #print "use float(siz_data[0] (with) to calculate new size"
        self.frame_scale = float(siz_data[0]) * 0.5 / 1227.0
        #(1227, 295)
      self._My_Update(request_new_data = False)
      #print "resizing"
      #print "aspec_ratio =", aspec_ratio
    else:
      self.sizing_counter += 1

  def _My_Update(self, request_new_data = True):

    if( request_new_data == True ):
      self.arr_img, self.arr_ttl = self.tabl(opt = self.opt)
      ref_max = self.tabl.Get_Max(self.opt)
      #box_lmt = self.tabl.Get_bbox()
      self.xyz_px = self.tabl.Get_xyz()
      self.bbox_px = self.tabl.Get_bbox()
      self.hkl_data = self.tabl.Get_hkl()
      for indx in range(len(self.arr_img)):
        if( self.opt == 0 and indx == len(self.arr_img) - 1 ):
          Imax = 10
        else:
          Imax = ref_max
        self.wx_Img[indx] = self.bmp(np_img_2d = self.arr_img[indx]
                          , Intst_max = Imax, ofst = self.bbox_px
                          , xyz = self.xyz_px, title = self.arr_ttl[indx])
      self.text01.SetValue(str(self.tabl.Get_ref_num()))

    for indx in range(len(self.arr_img)):
      if( self.arr_img[indx] == None ):
        empty_flag = True
      else:
        empty_flag = False
      self.My_Img[indx] = self.bmp.from_wx_image_to_wx_bitmap(self.wx_Img[indx]
                        , self.frame_scale, empty = empty_flag)
      self.Image[indx].SetBitmap(self.My_Img[indx])
      if( self.xyz_px != None ):
        my_str = ( " Centroid Position = ( "
                 + str(self.xyz_px[0]) + ", "
                 + str(self.xyz_px[1]) + ", "
                 + str(self.xyz_px[2]) + ") ")
        self.data_txt_01.SetLabel(my_str)
      else:
        self.data_txt_01.SetLabel(" No (x, y, z) Data")

      if( self.bbox_px != None ):
        my_str_01 = ( " Bbox = ( "
                 + str(self.bbox_px[0]) + ", "
                 + str(self.bbox_px[1]) + ", "
                 + str(self.bbox_px[2]) + ", "
                 + str(self.bbox_px[3]) + ", "
                 + str(self.bbox_px[4]) + ", "
                 + str(self.bbox_px[5]) + ") ")
        self.data_txt_02.SetLabel(my_str_01)
      else:
        self.data_txt_02.SetLabel(" No Bbox Data")

      try:

        hkl_string = 'HKL:    ' + str(self.hkl_data[0]) + \
                            '    ' + str(self.hkl_data[1]) + \
                            '    ' + str(self.hkl_data[2]) + ' '
      except:
        hkl_string = ' No HKL data'
      self.data_txt_03.SetLabel(hkl_string)

    self.Layout()
    self.Refresh()

  def _On_Close_Window(self, event):
    self.Destroy()
<commit_msg>mix of icon and text label in the four command buttons where is needed<commit_after>#
#  DIALS viewer_frame
#
#  Copyright (C) 2014 Diamond Light Source
#
#  Author: Luis Fuentes-Montero (Luiso)
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package."

import wx
from wx.lib.buttons import GenBitmapTextButton
from dials.viewer.viewer_utilities \
     import np_to_bmp

from dials.viewer.reflection_data_navigator import table_s_navigator

class ReflectionFrame(wx.Frame):
  def __init__(self, *args, **kwargs):
    wx.Frame.__init__(self, *args, **kwargs)

    #self.MaxImageSizeX = 280
    #self.MaxImageSizeY = 150
    self.MaxImageSizeX = 320
    self.MaxImageSizeY = 260

    lbl_ref = "Reflection"

    img_tmp = wx.ArtProvider.GetBitmap(wx.ART_GO_FORWARD, wx.ART_OTHER, (16, 16))
    btn_nxt_refl = wx.lib.buttons.GenBitmapTextButton(self
                   , label = lbl_ref, bitmap = img_tmp)
    img_tmp = wx.ArtProvider.GetBitmap(wx.ART_GO_BACK, wx.ART_OTHER, (16, 16))
    btn_prv_refl = wx.lib.buttons.GenBitmapTextButton(self
                   , label = lbl_ref, bitmap = img_tmp)
    btn_read = wx.Button(self, -1, "Reflection #")

    radio1 = wx.RadioButton(self, -1, "data, background, mask"
                           , style = wx.RB_GROUP)
    radio2 = wx.RadioButton(self, -1, "3 layers of data")
    radio3 = wx.RadioButton(self, -1, "3 layers of background")
    radio4 = wx.RadioButton(self, -1, "3 layers of mask")

    img_tmp = wx.ArtProvider.GetBitmap(wx.ART_GO_FORWARD, wx.ART_OTHER, (16, 16))
    btn_nxt_slice = wx.lib.buttons.GenBitmapTextButton(self
                   , label = 'slice', bitmap = img_tmp)

    img_tmp = wx.ArtProvider.GetBitmap(wx.ART_GO_BACK, wx.ART_OTHER, (16, 16))
    btn_prv_slice = wx.lib.buttons.GenBitmapTextButton(self
                   , label = 'slice', bitmap = img_tmp)

    self.data_txt_01 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))
    self.data_txt_02 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))
    self.data_txt_03 = wx.StaticText(self, -1, "(data_txt)", size = (800, 16))

    self.Image = [None, None, None]

    for indx in range(len(self.Image)):
      self.Image[indx] = wx.StaticBitmap(self, bitmap = wx.EmptyBitmap(
                                         self.MaxImageSizeX, self.MaxImageSizeY))

    self.text01 = wx.TextCtrl(self, -1, "enter number", size = (100, -1))

    u_box = wx.BoxSizer(wx.VERTICAL)
    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_prv_refl, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(btn_nxt_refl, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_read, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(self.text01, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    u_box.Add(radio1, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio2, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio3, 0, wx.LEFT | wx.ALL,3)
    u_box.Add(radio4, 0, wx.LEFT | wx.ALL,3)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(btn_prv_slice, 0, wx.CENTER | wx.ALL,3)
    div_h_box.Add(btn_nxt_slice, 0, wx.CENTER | wx.ALL,3)
    u_box.Add(div_h_box)

    h_box = wx.BoxSizer(wx.HORIZONTAL)

    h_box.Add(u_box)

    div_h_box = wx.BoxSizer(wx.HORIZONTAL)
    div_h_box.Add(self.Image[0], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)

    div_h_box.Add(self.Image[1], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)

    div_h_box.Add(self.Image[2], 0
            , wx.ALIGN_CENTER_HORIZONTAL | wx.ALL | wx.ADJUST_MINSIZE, 5)
    div_u_box = wx.BoxSizer(wx.VERTICAL)
    div_u_box.Add(div_h_box)
    div_u_box.Add(self.data_txt_01, 0, wx.CENTER | wx.ALL,3)
    div_u_box.Add(self.data_txt_02, 0, wx.CENTER | wx.ALL,3)
    div_u_box.Add(self.data_txt_03, 0, wx.CENTER | wx.ALL,3)
    h_box.Add(div_u_box)

    self.frame_scale = 0.4
    self.opt = 0

    self.sizing_counter = 0
    self.SetSizerAndFit(h_box)

    btn_nxt_refl.Bind(wx.EVT_BUTTON, self._DisplayNext_refl)
    btn_prv_refl.Bind(wx.EVT_BUTTON, self._DisplayPrev_refl)
    btn_nxt_slice.Bind(wx.EVT_BUTTON, self._DisplayNext_slice)
    btn_prv_slice.Bind(wx.EVT_BUTTON, self._DisplayPrev_slice)

    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio1, radio1)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio2, radio2)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio3, radio3)
    self.Bind(wx.EVT_RADIOBUTTON, self._OnRadio4, radio4)

    self.Bind(wx.EVT_BUTTON, self._read_num_from_txt, btn_read)

    self.Bind(wx.EVT_SIZE, self._OnSize)

    self.bmp = np_to_bmp()
    self.arr_img = [None, None, None]
    self.arr_ttl = [None, None, None]
    self.My_Img = [None, None, None]
    self.wx_Img = [None, None, None]
    wx.EVT_CLOSE(self, self._On_Close_Window)

  def tabl_to_frame(self, loc_tabl):
    self.tabl = table_s_navigator(loc_tabl)
    self._DisplayPrev_refl()
  def _DisplayNext_refl(self, event = None):
    self.tabl.next_Reflection()
    self._My_Update()
  def _DisplayPrev_refl(self, event = None):
    self.tabl.Previous_Reflection()
    self._My_Update()
  def _DisplayNext_slice(self, event = None):
    self.tabl.next_slice()
    self._My_Update()
  def _DisplayPrev_slice(self, event = None):
    self.tabl.Previous_slice()
    self._My_Update()

  def _read_num_from_txt(self, event = None):
    # TODO check if the user entered a number
    a = int(self.text01.GetValue())
    self.tabl.Jump_to_Reflection(a)
    self._My_Update()

  def _OnRadio1(self, event = None):
    self.opt = 0
    self._My_Update(request_new_data = True)

  def _OnRadio2(self, event = None):
    self.opt = 1
    self._My_Update(request_new_data = True)

  def _OnRadio3(self, event = None):
    self.opt = 2
    self._My_Update(request_new_data = True)

  def _OnRadio4(self, event = None):
    self.opt = 3
    self._My_Update(request_new_data = True)

  def _OnSize(self, event = None):
    if( self.sizing_counter > 5 ):
      siz_data = self.GetSize()
      #print "New size of window =", siz_data
      optm_aspec_ratio = 4.21
      #print "siz_data = ", siz_data[0], siz_data[1]
      #print "aspect ratio = ", float(siz_data[0])/ float(siz_data[1])
      aspec_ratio = float(siz_data[0])/ float(siz_data[1])

      if(aspec_ratio > optm_aspec_ratio):
        #print "use float(siz_data[1] (Height) to calculate new size"
        self.frame_scale = float(siz_data[1]) * 0.5 / 295.0
        #(1227, 295)
      else:
        #print "use float(siz_data[0] (with) to calculate new size"
        self.frame_scale = float(siz_data[0]) * 0.5 / 1227.0
        #(1227, 295)
      self._My_Update(request_new_data = False)
      #print "resizing"
      #print "aspec_ratio =", aspec_ratio
    else:
      self.sizing_counter += 1

  def _My_Update(self, request_new_data = True):

    if( request_new_data == True ):
      self.arr_img, self.arr_ttl = self.tabl(opt = self.opt)
      ref_max = self.tabl.Get_Max(self.opt)
      #box_lmt = self.tabl.Get_bbox()
      self.xyz_px = self.tabl.Get_xyz()
      self.bbox_px = self.tabl.Get_bbox()
      self.hkl_data = self.tabl.Get_hkl()
      for indx in range(len(self.arr_img)):
        if( self.opt == 0 and indx == len(self.arr_img) - 1 ):
          Imax = 10
        else:
          Imax = ref_max
        self.wx_Img[indx] = self.bmp(np_img_2d = self.arr_img[indx]
                          , Intst_max = Imax, ofst = self.bbox_px
                          , xyz = self.xyz_px, title = self.arr_ttl[indx])
      self.text01.SetValue(str(self.tabl.Get_ref_num()))

    for indx in range(len(self.arr_img)):
      if( self.arr_img[indx] == None ):
        empty_flag = True
      else:
        empty_flag = False
      self.My_Img[indx] = self.bmp.from_wx_image_to_wx_bitmap(self.wx_Img[indx]
                        , self.frame_scale, empty = empty_flag)
      self.Image[indx].SetBitmap(self.My_Img[indx])
      if( self.xyz_px != None ):
        my_str = ( " Centroid Position = ( "
                 + str(self.xyz_px[0]) + ", "
                 + str(self.xyz_px[1]) + ", "
                 + str(self.xyz_px[2]) + ") ")
        self.data_txt_01.SetLabel(my_str)
      else:
        self.data_txt_01.SetLabel(" No (x, y, z) Data")

      if( self.bbox_px != None ):
        my_str_01 = ( " Bbox = ( "
                 + str(self.bbox_px[0]) + ", "
                 + str(self.bbox_px[1]) + ", "
                 + str(self.bbox_px[2]) + ", "
                 + str(self.bbox_px[3]) + ", "
                 + str(self.bbox_px[4]) + ", "
                 + str(self.bbox_px[5]) + ") ")
        self.data_txt_02.SetLabel(my_str_01)
      else:
        self.data_txt_02.SetLabel(" No Bbox Data")

      try:

        hkl_string = 'HKL:    ' + str(self.hkl_data[0]) + \
                            '    ' + str(self.hkl_data[1]) + \
                            '    ' + str(self.hkl_data[2]) + ' '
      except:
        hkl_string = ' No HKL data'
      self.data_txt_03.SetLabel(hkl_string)

    self.Layout()
    self.Refresh()

  def _On_Close_Window(self, event):
    self.Destroy()
<|endoftext|>


<commit_before>#!/usr/bin/python

"""This is the master parsing file. It will take a folder of raw texts, metadata, and marcXML
	and construct the necessary input.txt and jsoncatalog.txt for Presidio to compile."""

#build core files for Presidio database

import re
import string
import os, sys
import subprocess
import xml.etree.ElementTree as ET
#may not need some of these, preserve memory
sys.setrecursionlimit(60000)

#generator that returns next record in specified folder

class folderWalk:
	def __init__(self, folder, extension):
		self.folder = folder
		self.extension = extension

	def __iter__(self):
        	return self

	def openDir(folder, extension):
		for item in os.listdir(folder):
	    		if (os.path.isfile(os.path.join(folder, item))):
	        	    return item
	        	    #combine readItem here

def readItem(item):
	file = open(item, 'r')
	data = file.read()
	file.close()
	return data

class Components:

	def __init__(self):
		#self.marcDir = openDir('/data/MARC', '_marc.xml')
		#self.xmlDir = openDir('MHL_download/mhl_meta_xml_files', '_meta.xml')
		#self.marcRoot = ET.fromstring(readItem(self.marcDir))
		#self.xmlRoot = ET.fromstring(readItem(self.xmlDir))

		def getDate(self):
			pass

		def getLibrary(self):
			pass

		def getLanguage(self):
			pass

		def getTitle(self):
			pass

		def getSearchstring(self):
			pass

		def getSubject(self):
			pass

		def getLocation(self):
			pass

def main():
	x = folderWalk()
	print x.openDir('/data/MARC', '_marc.xml')
	#buildJSON = Components()
	#print buildJSON.marcDir

main()

<commit_msg>added initializer to Components class<commit_after>#!/usr/bin/python

"""This is the master parsing file. It will take a folder of raw texts, metadata, and marcXML
	and construct the necessary input.txt and jsoncatalog.txt for Presidio to compile."""

#build core files for Presidio database

import re
import string
import os, sys
import subprocess
import xml.etree.ElementTree as ET
#may not need some of these, preserve memory
sys.setrecursionlimit(60000)

# open files, folders, read them
class fileHandlers:

	def __init__(self, directory):
		self.directory = directory

	#generator that returns next record in specified folder
	def openDir(self):
		for file in os.listdir(self.directory):
	    		if (os.path.isfile(os.path.join(self.directory, file))):
	        	    yield file

	#reads data within a file
	def readItem(self, item):
		file = open((os.path.join(self.directory, item)), 'r')
		data = file.read()
		file.close()
		return data

#functions that construct the JSON array
class Components:

	def __init__(self):
		self.marcDir = fileHandlers('/data/MARC')
		self.xmlDir = fileHandlers('/home/will/MHL_download/mhl_meta_xml_files')
		self.marcNext = self.marcDir.openDir().next()
		self.xmlNext = self.xmlDir.openDir().next()

		self.marcRoot = ET.fromstring(self.marcDir.readItem(self.marcNext))
		self.xmlRoot = ET.fromstring(self.xmlDir.readItem(self.xmlNext))

	def getDate(self):
		pass

	def getLibrary(self):
		pass

	def getLanguage(self):
		pass

	def getTitle(self):
		pass

	def getSearchstring(self):
		pass

	def getSubject(self):
		pass

	def getLocation(self):
		pass

def main():
	buildJSON = Components()
	print buildJSON.marcDir.openDir().next()

main()

<|endoftext|>


<commit_before>
from email.mime.text import MIMEText
import json
import os
import smtplib
import traceback

import bottle
from bottle import request, response

from bauble.server import app, API_ROOT, parse_accept_header, JSON_MIMETYPE, \
    TEXT_MIMETYPE, enable_cors, parse_auth_header, accept

@app.route(API_ROOT + "/contact", method=['OPTIONS', 'POST'])
def post_contact():

    if(request.method == 'OPTIONS'):
        return {}

    # make sure the content is JSON
    if JSON_MIMETYPE not in request.headers.get("Content-Type"):
        raise bottle.HTTPError('415 Unsupported Media Type',
                               'Expected application/json')

    try:
        data = json.loads(request.body.read().decode('utf-8'))
        msg = MIMEText(data['message'])
        msg["Subject"] = data["subject"] if "subject" in data else "Contact Form"
        msg["From"] = "{name} <{email}>".format(name=data["name"], email=data["email"])
        msg["To"] = "contact@bauble.io"
        server = smtplib.SMTP_SSL(os.environ.get('SMTP_SERVER', ''),
                                  os.environ.get('SMTP_PORT', 465))
        server.login(os.environ.get('SMTP_USERNAME', ''),
                     os.environ.get('SMTP_PASSWORD', ''))
        server.send_message(msg)
        server.quit()
    except Exception as exc:
        print(traceback.format_exc())
        bottle.abort(400, "Email could not be sent")
<commit_msg>change property name on json expected at /contact<commit_after>
from email.mime.text import MIMEText
import json
import os
import smtplib
import traceback

import bottle
from bottle import request, response

from bauble.server import app, API_ROOT, parse_accept_header, JSON_MIMETYPE, \
    TEXT_MIMETYPE, enable_cors, parse_auth_header, accept

@app.route(API_ROOT + "/contact", method=['OPTIONS', 'POST'])
def post_contact():

    if(request.method == 'OPTIONS'):
        return {}

    # make sure the content is JSON
    if JSON_MIMETYPE not in request.headers.get("Content-Type"):
        raise bottle.HTTPError('415 Unsupported Media Type',
                               'Expected application/json')

    try:
        data = json.loads(request.body.read().decode('utf-8'))
        msg = MIMEText(data['message'])
        msg["Subject"] = data["subject"] if "subject" in data else "Contact Form"
        msg["From"] = "{name} <{email}>".format(name=data["name"], email=data["from"])
        msg["To"] = "contact@bauble.io"
        server = smtplib.SMTP_SSL(os.environ.get('SMTP_SERVER', ''),
                                  os.environ.get('SMTP_PORT', 465))
        server.login(os.environ.get('SMTP_USERNAME', ''),
                     os.environ.get('SMTP_PASSWORD', ''))
        server.send_message(msg)
        server.quit()
    except Exception as exc:
        print(traceback.format_exc())
        bottle.abort(400, "Email could not be sent")
<|endoftext|>


<commit_before>""" ymir.commands
"""

import voluptuous, demjson

import os
import shutil

from fabric.colors import red
from fabric.contrib.console import confirm

from ymir.schema import schema
from ymir.service import AbstractService

YMIR_SRC = os.path.dirname(__file__)

def ymir_init(args):
    """ responsible for executing the 'ymir init' command """
    init_dir = os.path.abspath(args.init_dir)
    if os.path.exists(init_dir) and not args.force:
        err = ('this command is used to initialize a '
               'ymir project, the directory should'
               ' not already exist.')
        raise SystemExit(err)
    using_dot = init_dir == os.getcwd()
    if using_dot:
        os.chdir(os.path.dirname(init_dir))
    if os.path.exists(init_dir) and args.force:
        #if confirm(('you passed --force.  are you sure'
        #            ' you want to delete "{0}"?').format(
        #               init_dir)):
            shutil.rmtree(init_dir)
    skeleton_dir = os.path.join(YMIR_SRC, 'skeleton')
    if not os.path.exists(skeleton_dir):
        err = ('cannot find ymir skeleton project.  '
               'your ymir installation might be broken :(')
        raise SystemExit(err)
    print red('creating directory: ') + init_dir
    print red('copying ymir skeleton: '), skeleton_dir
    shutil.copytree(skeleton_dir, init_dir)
    if using_dot:
        os.path.chdir(init_dir)
def _load_json(fname):
    with open(fname) as fhandle:
        return demjson.decode(fhandle.read())

def _validate_file(fname):
    """ returns only error """
    tmp = _load_json(fname)
    try:
        schema(tmp)
    except voluptuous.Invalid, e:
        msg = "error validating {0}\n\t{1}"
        msg = msg.format(os.path.abspath(fname), e)
        return msg
    SERVICE_ROOT = os.path.dirname(fname)
    SERVICE_ROOT = os.path.abspath(SERVICE_ROOT)
    tmp.update(SERVICE_ROOT=SERVICE_ROOT)
    files = [tmp['puppet_setup']] + tmp['puppet']
    for _file in files:
        if not os.path.exists(os.path.join(SERVICE_ROOT, _file)):
            err= ('Files mentioned in service json '
                  'must exist relative to {0}, but {1}'
                  ' was not found').format(
                SERVICE_ROOT, _file)
            return err

def ymir_load(args, interactive=True):
    """ """
    ymir_validate(args, interactive=interactive)
    SERVICE_ROOT = os.path.dirname(args.service_json)
    SERVICE_ROOT = os.path.abspath(SERVICE_ROOT)
    service_json = _load_json(args.service_json)
    if interactive:
        lint = demjson.jsonlint()
        print red("Service root: "), '\n  ', SERVICE_ROOT
        print red("Service JSON: ")
        rc = lint.main(['-S','--format', args.service_json])
    dct = dict([
        [x.upper(), y] for x, y in service_json.items()])
    dct.update(SERVICE_ROOT=SERVICE_ROOT)
    ServiceFromJSON = type(
        'ServiceFromJSON',
        (AbstractService,), dct)
    if interactive:
        print red("Service definition loaded from JSON")
    return ServiceFromJSON()

def ymir_validate(args, interactive=True):
    """ """
    err = _validate_file(args.service_json)
    if err:
        raise SystemExit(err)
    elif interactive:
        print 'ok'
<commit_msg>minor changes<commit_after>""" ymir.commands
"""

import voluptuous, demjson

import os
import shutil

from fabric.colors import red
from fabric.contrib.console import confirm

from ymir.schema import schema
from ymir.service import AbstractService

YMIR_SRC = os.path.dirname(__file__)

#http://stackoverflow.com/questions/1868714/how-do-i-copy-an-entire-directory-of-files-into-an-existing-directory-using-pyth
def copytree(src, dst, symlinks=False, ignore=None):
    if not os.path.exists(dst):
        os.makedirs(dst)
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            copytree(s, d, symlinks, ignore)
        else:
            if not os.path.exists(d) or \
                   os.stat(src).st_mtime - os.stat(dst).st_mtime > 1:
                shutil.copy2(s, d)

def ymir_init(args):
    """ responsible for executing the 'ymir init' command """
    init_dir = os.path.abspath(args.init_dir)
    if os.path.exists(init_dir) and not args.force:
        err = ('this command is used to initialize a '
               'ymir project, the directory should'
               ' not already exist.')
        raise SystemExit(err)
    using_dot = init_dir == os.getcwd()
    if os.path.exists(init_dir) and args.force:
        #if confirm(('you passed --force.  are you sure'
        #            ' you want to delete "{0}"?').format(
        #               init_dir)):

        folder = init_dir
        for the_file in os.listdir(folder):
            file_path = os.path.join(folder, the_file)
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
                #elif os.path.isdir(file_path): shutil.rmtree(file_path)
            except Exception, e:
                print e

        #if using_dot:
        #    for x in os.listdir(init_dir):
        #        os.remove(x)#[shutil.rmtree(x)
        #else:
        #    shutil.rmtree(init_dir)
    skeleton_dir = os.path.join(YMIR_SRC, 'skeleton')
    if not os.path.exists(skeleton_dir):
        err = ('cannot find ymir skeleton project.  '
               'your ymir installation might be broken :(')
        raise SystemExit(err)
    print red('creating directory: ') + init_dir
    print red('copying ymir skeleton: '), skeleton_dir
    copytree(skeleton_dir, init_dir)

def _load_json(fname):
    with open(fname) as fhandle:
        return demjson.decode(fhandle.read())

def _validate_file(fname):
    """ returns only error """
    tmp = _load_json(fname)
    try:
        schema(tmp)
    except voluptuous.Invalid, e:
        msg = "error validating {0}\n\t{1}"
        msg = msg.format(os.path.abspath(fname), e)
        return msg
    SERVICE_ROOT = os.path.dirname(fname)
    SERVICE_ROOT = os.path.abspath(SERVICE_ROOT)
    tmp.update(SERVICE_ROOT=SERVICE_ROOT)
    files = [tmp['puppet_setup']] + tmp['puppet']
    for _file in files:
        if not os.path.exists(os.path.join(SERVICE_ROOT, _file)):
            err= ('Files mentioned in service json '
                  'must exist relative to {0}, but {1}'
                  ' was not found').format(
                SERVICE_ROOT, _file)
            return err

def ymir_load(args, interactive=True):
    """ """
    ymir_validate(args, interactive=interactive)
    SERVICE_ROOT = os.path.dirname(args.service_json)
    SERVICE_ROOT = os.path.abspath(SERVICE_ROOT)
    service_json = _load_json(args.service_json)
    if interactive:
        lint = demjson.jsonlint()
        print red("Service root: "), '\n  ', SERVICE_ROOT
        print red("Service JSON: ")
        rc = lint.main(['-S','--format', args.service_json])
    dct = dict([
        [x.upper(), y] for x, y in service_json.items()])
    dct.update(SERVICE_ROOT=SERVICE_ROOT)
    ServiceFromJSON = type(
        'ServiceFromJSON',
        (AbstractService,), dct)
    if interactive:
        print red("Service definition loaded from JSON")
    return ServiceFromJSON()

def ymir_validate(args, interactive=True):
    """ """
    err = _validate_file(args.service_json)
    if err:
        raise SystemExit(err)
    elif interactive:
        print 'ok'
<|endoftext|>


<commit_before># -------------------------------------------------------------------#
# Written by Mrinal Haloi
# Contact: mrinal.haloi11@gmail.com
# Copyright 2016, Mrinal Haloi
# -------------------------------------------------------------------#

# `pip install easydict` if you don't have it
from easydict import EasyDict as edict

__C = edict()

cfg = __C

__C.TRAIN = edict()
__C.VAL = edict()
__C.num_gpus = 1
__C.scale = 10000
__C.display = False
__C.max_step = 5000 * __C.scale
__C.memory_size = 100 * __C.scale

__C.random_start = 30
__C.cnn_format = 'NCHW'
__C.discount = 0.99
__C.target_q_update_step = 1 * __C.scale
__C.TRAIN.learning_rate = 0.00025
__C.TRAIN.learning_rate_minimum = 0.00025
__C.TRAIN.learning_rate_decay = 0.96
__C.TRAIN.learning_rate_decay_step = 5 * __C.scale
__C.TRAIN.buffer_size = 50000

__C.TRAIN.optname='rmsprop'
__C.ep_test = 0.01
__C.ep_end = 0.1
__C.ep_start = 1.
__C.ep_end_t = __C.memory_size
__C.tau = 0.001
__C.history_length = 4
__C.train_frequency = 4
__C.batch_size = 32
__C.learn_start = 5. * __C.scale

__C.min_delta = -1
__C.max_delta = 1

__C.double_q = False
__C.dueling = False

__C.test_step = 5 * __C.scale
__C.save_step = __C.test_step * 10

__C.env_name = 'Breakout-v0'

__C.screen_width = 84
__C.screen_height = 84
__C.max_reward = 1.
__C.min_reward = -1.

__C.model = 'MODEL1'

__C.backend = 'tf'
__C.env_type = 'detail'
__C.action_repeat = 1

__C.memory_allocation = 1
__C.TRAIN.batch_size = 16
__C.TRAIN.validation_split = 10
__C.TRAIN.learning_rate = 0.001
__C.TRAIN.num_epochs = 200
__C.TRAIN.training_size = 40000
__C.TRAIN.num_class = 5
__C.TRAIN.checkpoint_dir = '/media/Data/output/ckpt'
__C.TRAIN.im_depth = 3
__C.num_dataprocess_threads = 8
__C.num_readers = 4
__C.queue_memory_factor = 8
__C.input_queue_memory_factor = 8
__C.TRAIN.max_angle = 11
__C.TRAIN.sigma_max = 0.01
__C.num_threads = 8
__C.num_shards = 8
__C.output_dir = '/media/Data/output'
__C.TRAIN.fine_tune = False
__C.TRAIN.moving_average_decay = 0.9999
__C.TRAIN.rmsprop_decay = 0.9
__C.TRAIN.rmsprop_momentum = 0.9
__C.TRAIN.rmsprop_epsilon = 1.0
__C.TRAIN.sgd_momentum = 0.9
__C.TRAIN.adam_beta1 = 0.9
__C.TRAIN.adam_beat2 = 0.999
__C.TRAIN.adam_epsilon = 1e-08
__C.TRAIN.num_epochs_per_decay = 30
__C.TRAIN.learning_rate_decay_factor = 0.16
__C.TRAIN.max_steps = 9100000
__C.VAL.batch_size = 64
__C.VAL.subset = 'val'
<commit_msg>update config<commit_after># -------------------------------------------------------------------#
# Written by Mrinal Haloi
# Contact: mrinal.haloi11@gmail.com
# Copyright 2016, Mrinal Haloi
# -------------------------------------------------------------------#

# `pip install easydict` if you don't have it
from easydict import EasyDict as edict

__C = edict()

cfg = __C

__C.TRAIN = edict()
__C.VAL = edict()
__C.num_gpus = 1
__C.scale = 10000
__C.display = False
__C.max_step = 5000 * __C.scale
__C.memory_size = 100 * __C.scale

__C.random_start = 30
__C.cnn_format = 'NHWC'
__C.discount = 0.99
__C.target_q_update_step = 1 * __C.scale
__C.TRAIN.learning_rate = 0.00025
__C.TRAIN.learning_rate_minimum = 0.00025
__C.TRAIN.learning_rate_decay = 0.96
__C.TRAIN.learning_rate_decay_step = 5 * __C.scale
__C.TRAIN.buffer_size = 50000

__C.TRAIN.optname = 'rmsprop'
__C.ep_test = 0.01
__C.ep_end = 0.1
__C.ep_start = 1.
__C.ep_end_t = __C.memory_size
__C.tau = 0.001
__C.history_length = 4
__C.train_frequency = 4
__C.batch_size = 32
__C.learn_start = 5. * __C.scale

__C.min_delta = -1
__C.max_delta = 1

__C.double_q = False
__C.dueling = False

__C.test_step = 5 * __C.scale
__C.save_step = __C.test_step * 10

__C.env_name = 'Breakout-v0'

__C.screen_width = 84
__C.screen_height = 84
__C.max_reward = 1.
__C.min_reward = -1.

__C.model = 'MODEL1'

__C.backend = 'tf'
__C.env_type = 'detail'
__C.action_repeat = 1

__C.memory_allocation = 1
__C.TRAIN.batch_size = 16
__C.TRAIN.validation_split = 10
__C.TRAIN.learning_rate = 0.001
__C.TRAIN.num_epochs = 200
__C.TRAIN.training_size = 40000
__C.TRAIN.num_class = 5
__C.TRAIN.checkpoint_dir = '/media/Data/output/ckpt'
__C.TRAIN.im_depth = 3
__C.num_dataprocess_threads = 8
__C.num_readers = 4
__C.queue_memory_factor = 8
__C.input_queue_memory_factor = 8
__C.TRAIN.max_angle = 11
__C.TRAIN.sigma_max = 0.01
__C.num_threads = 8
__C.num_shards = 8
__C.output_dir = '/media/Data/output'
__C.TRAIN.fine_tune = False
__C.TRAIN.moving_average_decay = 0.9999
__C.TRAIN.rmsprop_decay = 0.9
__C.TRAIN.rmsprop_momentum = 0.9
__C.TRAIN.rmsprop_epsilon = 1.0
__C.TRAIN.sgd_momentum = 0.9
__C.TRAIN.adam_beta1 = 0.9
__C.TRAIN.adam_beta2 = 0.999
__C.TRAIN.rmsprop_beta1 = 0.9
__C.TRAIN.rmsprop_beta2 = 0.999
__C.TRAIN.adam_epsilon = 1e-08
__C.TRAIN.num_epochs_per_decay = 30
__C.TRAIN.learning_rate_decay_factor = 0.16
__C.TRAIN.max_steps = 9100000
__C.VAL.batch_size = 64
__C.VAL.subset = 'val'
<|endoftext|>


<commit_before>import json

from django.db.models import Count
from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest

from hmda.models import HMDARecord
from .models import Census2010RaceStats
from geo.views import get_censustract_geoids
from geo.models import Geo
from djqscsv import render_to_csv_response
from hmda.views import loan_originations_as_json
from respondents.models import Institution

def sum_lar_tuples(tups):
    return sum([tup[1] for tup in tups])

def assemble_stats(lar_data, tracts):
    """
    assembles a lender's applications by those made
    in low, medium and high minority areas;
    we might be assembling stats for a lender or peer;
    """
    lma = []
    mma = []
    hma = []
    for tract in tracts:
        stats = tract.census2010racestats
        if tract.geoid in lar_data.keys():
            tract_tuple = (tract, lar_data[tract.geoid]['volume'])
        else:
            tract_tuple = (tract, 0)
        if stats.total_pop:
            minority = stats.total_pop - stats.non_hisp_white_only
            minority_pct = 1.0 * minority / stats.total_pop
            if minority_pct < .5:
                lma.append(tract_tuple)
            elif minority_pct < .8:
                mma.append(tract_tuple)
            else:
                hma.append(tract_tuple)
    lma_ct = sum_lar_tuples(lma)
    mma_ct = sum_lar_tuples(mma)
    hma_ct = sum_lar_tuples(hma)
    lar_total = lma_ct + mma_ct + hma_ct
    if lar_total:
        lar_stats = {
                'lma': lma_ct,
                'lma_pct': 1.0 * lma_ct / lar_total,
                'mma': mma_ct,
                'mma_pct': 1.0 * mma_ct / lar_total,
                'hma': hma_ct,
                'hma_pct': 1.0 * hma_ct / lar_total,
                'lar_total': lar_total
                }
        return lar_stats
    else:
        return {
            'lar_total': 0,
            'lma': 0,
            'lma_pct': 0,
            'mma': 0,
            'mma_pct': 0,
            'hma': 0,
            'hma_pct': 0
            }

def tally_msa_minority_stats(tracts):
    """
    metro areas don't have attached census data,
    so we need to count up minority figures
    """
    pop, minority = 0, 0
    for tract in tracts:
        stats = tract.census2010racestats
        pop += stats.total_pop
        minority += (stats.total_pop - stats.non_hisp_white_only)
    if pop:
        return pop, minority, 1.0 * minority / pop
    else:
        return 0, 0, 0

def combine_peer_stats(collector):
    """
    calculates stats for a group of peers
    """
    peer_total = sum([entry['lar_total'] for entry in collector])
    if peer_total:
        lma = sum([entry['lma'] for entry in collector])
        lma_pct = 1.0 * lma / peer_total
        mma = sum([entry['mma'] for entry in collector])
        mma_pct = 1.0 * mma / peer_total
        hma = sum([entry['hma'] for entry in collector])
        hma_pct = 1.0 * hma / peer_total
        return {
                'lma': lma,
                'lma_pct': 1.0 * lma / peer_total,
                'mma': mma,
                'mma_pct': 1.0 * mma / peer_total,
                'hma': hma,
                'hma_pct': 1.0 * hma / peer_total,
                'lar_total': peer_total
                }
    else:
        return {
            'lar_total': 0,
            'lma': 0,
            'lma_pct': 0,
            'mma': 0,
            'mma_pct': 0,
            'hma': 0,
            'hma_pct': 0
            }

def minority_aggregation_as_json(request):
    """
    aggregates minority population ranges and LAR counts
    for a lender in an MSA
    by tract, msa and county
    """
    lar_data = loan_originations_as_json(request)
    lender = Institution.objects.get(institution_id=request.GET.get('lender'))
    tracts = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, cbsa=request.GET.get('metro'))
    metro = Geo.objects.get(geo_type=Geo.METRO_TYPE, geoid=request.GET.get('metro'))
    lender_stats = assemble_stats(lar_data, tracts)
    # MSA
    msa_pop, msa_minority_ct, msa_minority_pct = tally_msa_minority_stats(tracts)
    msa_stats = {
        'minority_ct': msa_minority_ct,
        'minority_pct': msa_minority_pct
    }
    # PEERS
    peers = lender.get_peer_list(metro, None, None)
    if peers:
        peer_data_collector = []
        for peer in peers:
            peer_request = HttpRequest()
            peer_request.GET['lender'] = peer.institution.institution_id
            peer_request.GET['metro']= metro.geoid
            peer_lar_data = loan_originations_as_json(peer_request)
            peer_data_collector.append(assemble_stats(peer_lar_data, tracts))
        if len(peer_data_collector) == 1:
            peer_stats = peer_data_collector[0]
        else:
            peer_stats = combine_peer_stats(peer_data_collector)
        # ODDS
        target_mm = lender_stats['hma'] + lender_stats['mma']
        target_non = lender_stats['lma']
        peer_mm = peer_stats['hma'] + peer_stats['mma']
        peer_non = peer_stats['lma']
        odds = odds_ratio(target_mm, target_non, peer_mm, peer_non)
    else:
        peer_stats = None
        odds = None
    # BY COUNTY -- TODO
    county_ids = sorted(set([tract.geoid[:5] for tract in tracts]))
    county_stats = {county_id: {} for county_id in county_ids}
    for county_id in county_ids:
        county_tracts = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, state=county_id[:2], county=county_id[2:])
        county_stats[county_id] = assemble_stats(lar_data, county_tracts)

    return {
        'msa': msa_stats,
        'lender': lender_stats,
        'peers': peer_stats,
        'odds': odds,
        'counties': county_stats,
        }

def odds_ratio(target_mm, target_non, peer_mm, peer_non):
    """
    intended to calculate a lender/peers odds ratio for minority lending,
    based on counts of loans in mostly minority areas vs mostly majority areas
    this algorithm has not been vetted, so its current use
    is only for mocking data flow to tables
    """
    portfolio_target = float(target_mm) / float(target_mm + target_non)
    portfolio_peer = float(peer_mm) / float(peer_mm + peer_non)

    odds_numer = portfolio_target / (1.0 - portfolio_target)
    odds_denom = portfolio_peer / (1.0 - portfolio_peer)

    return int(odds_numer / odds_denom)

def race_summary(request):
    """Race summary statistics"""
    geoids = get_censustract_geoids(request)
    if geoids:
        query = Census2010RaceStats.objects.filter(geoid_id__in=geoids)
        return query
    else:
        return HttpResponseBadRequest("Missing geoid or county")

def race_summary_as_json(request_dict):
    records = race_summary(request_dict)

    data = {}
    for stats in records:
        if stats.total_pop:
            MINPERC = 1.0 * (stats.total_pop - stats.non_hisp_white_only) / stats.total_pop
        else:
            MINPERC = 0
        data[stats.geoid_id] = {
            'total_pop': stats.total_pop,
            'total_pop': stats.total_pop,
            'hispanic': stats.hispanic,
            'non_hisp_white_only': stats.non_hisp_white_only,
            'non_hisp_black_only': stats.non_hisp_black_only,
            'non_hisp_asian_only': stats.non_hisp_asian_only,
            'minority_perc': MINPERC,
            'hispanic_perc': stats.hispanic_perc,
            'non_hisp_white_only_perc': stats.non_hisp_white_only_perc,
            'non_hisp_black_only_perc': stats.non_hisp_black_only_perc,
            'non_hisp_asian_only_perc': stats.non_hisp_asian_only_perc
        }
    return data

def race_summary_http(request):
    return HttpResponse(json.dumps(race_summary_as_json(request)))

def race_summary_csv(request):
    institution_id = request.GET.get('lender')
    metro = request.GET.get('metro')
    action_taken_param = request.GET.get('action_taken')
    action_taken_selected = action_taken_param.split(',')
    tracts_in_msa = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, cbsa=metro).values_list('geoid', flat=True)
    query = HMDARecord.objects.filter(institution_id=institution_id, geo_id__in=tracts_in_msa, property_type__in=[1,2], owner_occupancy=1, lien_status=1, action_taken__in=action_taken_selected).values('geo_id', 'geo__census2010households__total', 'geo__census2010racestats__total_pop', 'geo__census2010racestats__hispanic_perc', 'geo__census2010racestats__non_hisp_white_only_perc', 'geo__census2010racestats__non_hisp_black_only_perc', 'geo__census2010racestats__non_hisp_asian_only_perc').annotate(lar_count=Count('geo_id'))

    return render_to_csv_response(query)
<commit_msg>allow one decimal place in odds ratio result<commit_after>import json

from django.db.models import Count
from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest

from hmda.models import HMDARecord
from .models import Census2010RaceStats
from geo.views import get_censustract_geoids
from geo.models import Geo
from djqscsv import render_to_csv_response
from hmda.views import loan_originations_as_json
from respondents.models import Institution

def sum_lar_tuples(tups):
    return sum([tup[1] for tup in tups])

def assemble_stats(lar_data, tracts):
    """
    assembles a lender's applications by those made
    in low, medium and high minority areas;
    we might be assembling stats for a lender or peer;
    """
    lma = []
    mma = []
    hma = []
    for tract in tracts:
        stats = tract.census2010racestats
        if tract.geoid in lar_data.keys():
            tract_tuple = (tract, lar_data[tract.geoid]['volume'])
        else:
            tract_tuple = (tract, 0)
        if stats.total_pop:
            minority = stats.total_pop - stats.non_hisp_white_only
            minority_pct = 1.0 * minority / stats.total_pop
            if minority_pct < .5:
                lma.append(tract_tuple)
            elif minority_pct < .8:
                mma.append(tract_tuple)
            else:
                hma.append(tract_tuple)
    lma_ct = sum_lar_tuples(lma)
    mma_ct = sum_lar_tuples(mma)
    hma_ct = sum_lar_tuples(hma)
    lar_total = lma_ct + mma_ct + hma_ct
    if lar_total:
        lar_stats = {
                'lma': lma_ct,
                'lma_pct': 1.0 * lma_ct / lar_total,
                'mma': mma_ct,
                'mma_pct': 1.0 * mma_ct / lar_total,
                'hma': hma_ct,
                'hma_pct': 1.0 * hma_ct / lar_total,
                'lar_total': lar_total
                }
        return lar_stats
    else:
        return {
            'lar_total': 0,
            'lma': 0,
            'lma_pct': 0,
            'mma': 0,
            'mma_pct': 0,
            'hma': 0,
            'hma_pct': 0
            }

def tally_msa_minority_stats(tracts):
    """
    metro areas don't have attached census data,
    so we need to count up minority figures
    """
    pop, minority = 0, 0
    for tract in tracts:
        stats = tract.census2010racestats
        pop += stats.total_pop
        minority += (stats.total_pop - stats.non_hisp_white_only)
    if pop:
        return pop, minority, 1.0 * minority / pop
    else:
        return 0, 0, 0

def combine_peer_stats(collector):
    """
    calculates stats for a group of peers
    """
    peer_total = sum([entry['lar_total'] for entry in collector])
    if peer_total:
        lma = sum([entry['lma'] for entry in collector])
        lma_pct = 1.0 * lma / peer_total
        mma = sum([entry['mma'] for entry in collector])
        mma_pct = 1.0 * mma / peer_total
        hma = sum([entry['hma'] for entry in collector])
        hma_pct = 1.0 * hma / peer_total
        return {
                'lma': lma,
                'lma_pct': 1.0 * lma / peer_total,
                'mma': mma,
                'mma_pct': 1.0 * mma / peer_total,
                'hma': hma,
                'hma_pct': 1.0 * hma / peer_total,
                'lar_total': peer_total
                }
    else:
        return {
            'lar_total': 0,
            'lma': 0,
            'lma_pct': 0,
            'mma': 0,
            'mma_pct': 0,
            'hma': 0,
            'hma_pct': 0
            }

def minority_aggregation_as_json(request):
    """
    aggregates minority population ranges and LAR counts
    for a lender in an MSA
    by tract, msa and county
    """
    lar_data = loan_originations_as_json(request)
    lender = Institution.objects.get(institution_id=request.GET.get('lender'))
    tracts = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, cbsa=request.GET.get('metro'))
    metro = Geo.objects.get(geo_type=Geo.METRO_TYPE, geoid=request.GET.get('metro'))
    lender_stats = assemble_stats(lar_data, tracts)
    # MSA
    msa_pop, msa_minority_ct, msa_minority_pct = tally_msa_minority_stats(tracts)
    msa_stats = {
        'minority_ct': msa_minority_ct,
        'minority_pct': msa_minority_pct
    }
    # PEERS
    peers = lender.get_peer_list(metro, None, None)
    if peers:
        peer_data_collector = []
        for peer in peers:
            peer_request = HttpRequest()
            peer_request.GET['lender'] = peer.institution.institution_id
            peer_request.GET['metro']= metro.geoid
            peer_lar_data = loan_originations_as_json(peer_request)
            peer_data_collector.append(assemble_stats(peer_lar_data, tracts))
        if len(peer_data_collector) == 1:
            peer_stats = peer_data_collector[0]
        else:
            peer_stats = combine_peer_stats(peer_data_collector)
        # ODDS
        target_mm = lender_stats['hma'] + lender_stats['mma']
        target_non = lender_stats['lma']
        peer_mm = peer_stats['hma'] + peer_stats['mma']
        peer_non = peer_stats['lma']
        odds = odds_ratio(target_mm, target_non, peer_mm, peer_non)
    else:
        peer_stats = None
        odds = None
    # BY COUNTY -- TODO
    county_ids = sorted(set([tract.geoid[:5] for tract in tracts]))
    county_stats = {county_id: {} for county_id in county_ids}
    for county_id in county_ids:
        county_tracts = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, state=county_id[:2], county=county_id[2:])
        county_stats[county_id] = assemble_stats(lar_data, county_tracts)

    return {
        'msa': msa_stats,
        'lender': lender_stats,
        'peers': peer_stats,
        'odds': odds,
        'counties': county_stats,
        }

def odds_ratio(target_mm, target_non, peer_mm, peer_non):
    """
    intended to calculate a lender/peers odds ratio for minority lending,
    based on counts of loans in mostly minority areas vs mostly majority areas
    this algorithm has not been vetted, so its current use
    is only for mocking data flow to tables
    """
    portfolio_target = float(target_mm) / float(target_mm + target_non)
    portfolio_peer = float(peer_mm) / float(peer_mm + peer_non)

    odds_numer = portfolio_target / (1.0 - portfolio_target)
    odds_denom = portfolio_peer / (1.0 - portfolio_peer)

    # return int(odds_numer / odds_denom)
    return round((odds_numer / odds_denom), 1)
    # return odds_numer / odds_denom

def race_summary(request):
    """Race summary statistics"""
    geoids = get_censustract_geoids(request)
    if geoids:
        query = Census2010RaceStats.objects.filter(geoid_id__in=geoids)
        return query
    else:
        return HttpResponseBadRequest("Missing geoid or county")

def race_summary_as_json(request_dict):
    records = race_summary(request_dict)

    data = {}
    for stats in records:
        if stats.total_pop:
            MINPERC = 1.0 * (stats.total_pop - stats.non_hisp_white_only) / stats.total_pop
        else:
            MINPERC = 0
        data[stats.geoid_id] = {
            'total_pop': stats.total_pop,
            'total_pop': stats.total_pop,
            'hispanic': stats.hispanic,
            'non_hisp_white_only': stats.non_hisp_white_only,
            'non_hisp_black_only': stats.non_hisp_black_only,
            'non_hisp_asian_only': stats.non_hisp_asian_only,
            'minority_perc': MINPERC,
            'hispanic_perc': stats.hispanic_perc,
            'non_hisp_white_only_perc': stats.non_hisp_white_only_perc,
            'non_hisp_black_only_perc': stats.non_hisp_black_only_perc,
            'non_hisp_asian_only_perc': stats.non_hisp_asian_only_perc
        }
    return data

def race_summary_http(request):
    return HttpResponse(json.dumps(race_summary_as_json(request)))

def race_summary_csv(request):
    institution_id = request.GET.get('lender')
    metro = request.GET.get('metro')
    action_taken_param = request.GET.get('action_taken')
    action_taken_selected = action_taken_param.split(',')
    tracts_in_msa = Geo.objects.filter(geo_type=Geo.TRACT_TYPE, cbsa=metro).values_list('geoid', flat=True)
    query = HMDARecord.objects.filter(institution_id=institution_id, geo_id__in=tracts_in_msa, property_type__in=[1,2], owner_occupancy=1, lien_status=1, action_taken__in=action_taken_selected).values('geo_id', 'geo__census2010households__total', 'geo__census2010racestats__total_pop', 'geo__census2010racestats__hispanic_perc', 'geo__census2010racestats__non_hisp_white_only_perc', 'geo__census2010racestats__non_hisp_black_only_perc', 'geo__census2010racestats__non_hisp_asian_only_perc').annotate(lar_count=Count('geo_id'))

    return render_to_csv_response(query)
<|endoftext|>