<commit_before># -*- coding: utf-8 -*-
from suds.client import Client
from geoportailv3_geoportal.models import LuxMeasurementLoginCommune
from geoportailv3_geoportal.models import LuxMeasurementDirectory
from c2cgeoportal_commons.models import DBSession
from sqlalchemy import func
import logging
import os
import sys

log = logging.getLogger(__name__)

# Classe pour accéder aux attributs de la publicité foncière##
# Les classes importées en haut ont été générées par wsdl2py
# dépendances: pyxml,fpconst,ZSI

class PF():
    def __init__(self):
        self.client = \
            Client('https://titan.etat.lu/xxpfoWS/ParcelDetailVer1Service/ParcelDetailVer1Service.wsdl')
        self.client_mesurage = \
            Client('https://titan.etat.lu/xxpfoWS/Measure' +
                   'mentVer1Service/META-INF/wsdl/MeasurementVer1Service.wsdl')
        self.log = logging.getLogger(__name__)

    def get_client(self):
        return self.client

    def get_client_mesurage(self):
        return self.client_mesurage

    def get_detail(self, code_commune, code_section,
                   numero_principal, numero_supplementaire):

        req = self.client.factory.create('parcelDetailRequestVer1')
        req.townCode = code_commune
        req.sectionCode = code_section
        req.mainNumber = numero_principal
        req.additionalNumber = numero_supplementaire
        resp = self.client.service.getParcelDetail(req)
        d = resp.parcelDetail
        return d

    def get_measurement_list(self, number_prim, number_sec,
                             section_code, town_code, user, referer):

        req = self.client_mesurage.factory.create('measurementRequestVer1')
        req.numberFrom = number_prim
        req.numberTo = number_prim
        req.numberSuppFrom = number_sec
        req.numberSuppTo = number_sec
        req.sectionCode = section_code
        req.townCode = town_code
        try:
            resp = self.client_mesurage.service.searchMeasurement(req)
        except Exception as e:
            log.exception(e)
            return []

        if not hasattr(resp, 'measurementList'):
                return []

        measurements = resp.measurementList

        if len(town_code) == 0:
            self.log.debug("Erreur le town code ne peut pas �tre vide")

        parc_id = self._add_char_before(3, town_code)

        parc_id = parc_id + section_code
        parc_id = parc_id + self._add_char_before(5, number_prim)
        parc_id = parc_id + self._add_char_before(6, number_sec)

        measurement_list = []

        for measurement in measurements:
            if measurement.parcelId == parc_id:
                if measurement.measurementNumber > 0:
                    measurement_files = \
                        self._get_pdf_file_path(town_code,
                                                measurement.measurementNumber,
                                                user, referer)

                    measurement.files = measurement_files['files'] if 'files' \
                        in measurement_files else None
                    measurement.town_info = measurement_files['town_info'] \
                        if 'town_info' in measurement_files else None
                    measurement_list.append(dict(measurement))

        return measurement_list

    def _get_pdf_file_path(self, town_code, measurement_number, user, referer):
        try:
            town_info = self._get_town_by_code(int(town_code))
            if town_info is None:
                return {}
            files = os.listdir(town_info.get('path'))

            mes = self._add_char_before(5, measurement_number)
            files_list = []
            for cur_file in files:
                if cur_file.startswith(mes):
                    files_list.append({'filename': cur_file,
                                       'is_downloadable':
                                       self._is_download_authorized(
                                           town_info.get('name'),
                                           user, referer)
                                       })

            if len(files_list) == 0:
                return {}

            return {"town_info": town_info,
                    "files": files_list}
        except:
            print(sys.exc_info())
            return {}

    def _add_char_before(self, max_length, the_string):
        tmp = ""
        for i in range(max_length - len(str(the_string))):
            tmp = tmp + "0"
        return str(tmp) + str(the_string)

    # retourne l'information sur une commune
    def _get_town_by_code(self, town_code):
        cur_record = DBSession.query(LuxMeasurementDirectory).\
            filter(LuxMeasurementDirectory.town_code == town_code).first()
        if cur_record is not None:
            return {
                'name': cur_record.name,
                'town_code': cur_record.town_code,
                'path': cur_record.path
                }
        return None

    def _get_town_by_name(self, town_name):
        cur_record = DBSession.query(LuxMeasurementDirectory).\
            filter(LuxMeasurementDirectory.name == town_name).first()
        if cur_record is not None:
            return {
                'name': cur_record.name,
                'town_code': cur_record.town_code,
                'path': cur_record.path
                }
        return None

    def _is_download_authorized(self, town_name, user, referer):
        if referer is not None:
            if "bodfeature" in referer and "search4naila" in referer:
                return True
            if "weboffice" in referer:
                return True
        town_info = self._get_town_by_name(town_name)

        if (town_info is None or user is None or user.username is None):
            return False

        if (DBSession.query(LuxMeasurementLoginCommune).
                filter(func.lower(LuxMeasurementLoginCommune.login) ==
                       func.lower(func.geov3.getMainAccount(user.username))).
                filter(LuxMeasurementLoginCommune.num_commune ==
                       str(town_info.get('town_code'))).count() > 0):
            return True

        return False
<commit_msg>Update PF.py<commit_after># -*- coding: utf-8 -*-
from suds.client import Client
from geoportailv3_geoportal.models import LuxMeasurementLoginCommune
from geoportailv3_geoportal.models import LuxMeasurementDirectory
from c2cgeoportal_commons.models import DBSession
from sqlalchemy import func
import logging
import os
import sys

log = logging.getLogger(__name__)

# Classe pour accéder aux attributs de la publicité foncière##
# Les classes importées en haut ont été générées par wsdl2py
# dépendances: pyxml,fpconst,ZSI

class PF():
    def __init__(self):
        self.client = \
            Client('https://titan.etat.lu/xxpfoWS/ParcelDetailVer1Service/ParcelDetailVer1Service.wsdl')
        self.client_mesurage = \
            Client('https://titan.etat.lu/xxpfoWS/Measure' +
                   'mentVer1Service/META-INF/wsdl/MeasurementVer1Service.wsdl')
        self.log = logging.getLogger(__name__)

    def get_client(self):
        return self.client

    def get_client_mesurage(self):
        return self.client_mesurage

    def get_detail(self, code_commune, code_section,
                   numero_principal, numero_supplementaire):

        req = self.client.factory.create('parcelDetailRequestVer1')
        req.townCode = code_commune
        req.sectionCode = code_section
        req.mainNumber = numero_principal
        req.additionalNumber = numero_supplementaire
        resp = self.client.service.getParcelDetail(req)
        d = resp.parcelDetail
        return d

    def get_measurement_list(self, number_prim, number_sec,
                             section_code, town_code, user, referer):

        req = self.client_mesurage.factory.create('measurementRequestVer1')
        req.numberFrom = number_prim
        req.numberTo = number_prim
        req.numberSuppFrom = number_sec
        req.numberSuppTo = number_sec
        req.sectionCode = section_code
        req.townCode = town_code
        try:
            resp = self.client_mesurage.service.searchMeasurement(req)
        except Exception as e:
            log.exception(e)
            return []

        if not hasattr(resp, 'measurementList'):
                return []

        measurements = resp.measurementList

        if len(town_code) == 0:
            self.log.debug("Erreur le town code ne peut pas �tre vide")

        parc_id = self._add_char_before(3, town_code)

        parc_id = parc_id + section_code
        parc_id = parc_id + self._add_char_before(5, number_prim)
        parc_id = parc_id + self._add_char_before(6, number_sec)

        measurement_list = []

        for measurement in measurements:
            if measurement.parcelId == parc_id:
                if measurement.measurementNumber > 0:
                    measurement_files = \
                        self._get_pdf_file_path(town_code,
                                                measurement.measurementNumber,
                                                user, referer)

                    measurement.files = measurement_files['files'] if 'files' \
                        in measurement_files else None
                    measurement.town_info = measurement_files['town_info'] \
                        if 'town_info' in measurement_files else None
                    measurement_list.append(dict(measurement))

        return measurement_list

    def _get_pdf_file_path(self, town_code, measurement_number, user, referer):
        try:
            town_info = self._get_town_by_code(int(town_code))
            if town_info is None:
                return {}
            files = os.listdir(town_info.get('path'))

            mes = self._add_char_before(5, measurement_number)
            files_list = []
            for cur_file in files:
                if cur_file.startswith(mes):
                    files_list.append({'filename': cur_file,
                                       'is_downloadable':
                                       self._is_download_authorized(
                                           town_info.get('name'),
                                           user, referer)
                                       })

            if len(files_list) == 0:
                return {}

            return {"town_info": town_info,
                    "files": files_list}
        except:
            print(sys.exc_info())
            return {}

    def _add_char_before(self, max_length, the_string):
        tmp = ""
        for i in range(max_length - len(str(the_string))):
            tmp = tmp + "0"
        return str(tmp) + str(the_string)

    # retourne l'information sur une commune
    def _get_town_by_code(self, town_code):
        cur_record = DBSession.query(LuxMeasurementDirectory).\
            filter(LuxMeasurementDirectory.town_code == town_code).first()
        if cur_record is not None:
            return {
                'name': cur_record.name,
                'town_code': cur_record.town_code,
                'path': cur_record.path
                }
        return None

    def _get_town_by_name(self, town_name):
        cur_record = DBSession.query(LuxMeasurementDirectory).\
            filter(LuxMeasurementDirectory.name == town_name).first()
        if cur_record is not None:
            return {
                'name': cur_record.name,
                'town_code': cur_record.town_code,
                'path': cur_record.path
                }
        return None

    def _is_download_authorized(self, town_name, user, referer):
        if referer is not None:
            if "bodfeature" in referer and "search4naila" in referer:
                return True
            if "weboffice" in referer:
                return True
            if "weboffice_um" in referer:
                return True
        town_info = self._get_town_by_name(town_name)

        if (town_info is None or user is None or user.username is None):
            return False

        if (DBSession.query(LuxMeasurementLoginCommune).
                filter(func.lower(LuxMeasurementLoginCommune.login) ==
                       func.lower(func.geov3.getMainAccount(user.username))).
                filter(LuxMeasurementLoginCommune.num_commune ==
                       str(town_info.get('town_code'))).count() > 0):
            return True

        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import pyglet
import autoprop

from vecrec import Vector, Rect
from debugtools import p, pp, pv
from glooey import drawing
from glooey.widget import Widget
from glooey.images import Background
from glooey.containers import Stack, Deck

@autoprop
class Label(Widget):
    custom_text = ""
    custom_font_name = None
    custom_font_size = None
    custom_bold = None
    custom_italic = None
    custom_underline = None
    custom_kerning = None
    custom_baseline = None
    custom_color = 'green'
    custom_background_color = None
    custom_text_alignment = None
    custom_line_spacing = None

    def __init__(self, text="", line_wrap=None, **style):
        super().__init__()
        self._layout = None
        self._text = text or self.custom_text
        self._line_wrap_width = 0
        self._style = {}
        self.set_style(
                font_name=self.custom_font_name,
                font_size=self.custom_font_size,
                bold=self.custom_bold,
                italic=self.custom_italic,
                underline=self.custom_underline,
                kerning=self.custom_kerning,
                baseline=self.custom_baseline,
                color=self.custom_color,
                background_color=self.custom_background_color,
                align=self.custom_text_alignment,
                line_spacing=self.custom_line_spacing,
        )
        self.set_style(**style)
        if line_wrap:
            self.enable_line_wrap(line_wrap)

    def do_claim(self):
        # Make sure the label's text and style are up-to-date before we request
        # space.  Be careful!  This means that do_draw() can be called before
        # the widget has self.rect or self.group, which usually cannot happen.
        self.do_draw(ignore_rect=True)

        # Return the amount of space needed to render the label.
        return self._layout.content_width, self._layout.content_height

    def do_draw(self, ignore_rect=False):
        # Any time we need to draw this widget, just delete the underlying
        # label object and make a new one.  This isn't any slower than keeping
        # the old object, because all the vertex lists would be redrawn either
        # way.  And this is more flexible, because it allows us to reset the
        # batch, the group, and the wrap_lines attribute.
        if self._layout is not None:
            self._layout.delete()

        kwargs = {
                'multiline': True,
                'wrap_lines': False,
                'batch': self.batch,
                'group': self.group
        }
        # Enable line wrapping, if the user requested it.  The width of the
        # label is set to the value given by the user when line-wrapping was
        # enabled.  This will be overwritten a few lines later if the widget
        # has been given a rectangle.
        if self._line_wrap_width:
            kwargs['width'] = self._line_wrap_width
            kwargs['wrap_lines'] = True

        # Usually self.rect is guaranteed to be set by the time this method is
        # called, but that is not the case for this widget.  The do_claim()
        # method needs to call do_draw() to see how much space the text will
        # need, and that happens before self.rect is set (since it's part of
        # the process of setting self.rect).
        if not ignore_rect:
            kwargs['width'] = self.rect.width
            kwargs['height'] = self.rect.height

        # It's best to make a fresh document each time.  Previously I was
        # storing the document as a member variable, but I ran into corner
        # cases where the document would have an old style that wouldn't be
        # compatible with the new TextLayout (specifically 'align' != 'left' if
        # line wrapping is no loner enabled).
        document = pyglet.text.decode_text(self._text)
        document.push_handlers(self.on_insert_text, self.on_delete_text)
        self._layout = self.do_make_new_layout(document, kwargs)

        # Use begin_update() and end_update() to prevent the layout from
        # generating new vertex lists until the styles and coordinates have
        # been set.
        self._layout.begin_update()

        # The layout will crash if it doesn't have an explicit width and the
        # style specifies an alignment.  I
        if self._layout.width is None:
            self._layout.width = self._layout.content_width

        document.set_style(0, len(self._text), self._style)

        if not ignore_rect:
            self._layout.x = self.rect.bottom_left.x
            self._layout.y = self.rect.bottom_left.y

        self._layout.end_update()

    def do_undraw(self):
        if self._layout is not None:
            self._layout.delete()

    def do_make_new_layout(self, document, kwargs):
        return pyglet.text.layout.TextLayout(document, **kwargs)

    def on_insert_text(self, start, text):
        self._text = self._layout.document.text
        self.dispatch_event('on_edit_text', self)

    def on_delete_text(self, start, end):
        self._text = self._layout.document.text
        self.dispatch_event('on_edit_text', self)

    def get_text(self):
        return self._layout.document.text

    def set_text(self, text, width=None, **style):
        self._text = text
        if width is not None:
            self._line_wrap_width = width
        # This will repack.
        self.set_style(**style)

    def del_text(self):
        self.set_text("")

    def get_font_name(self):
        return self.get_style('font_name')

    def set_font_name(self, name):
        self.set_style(font_name=name)

    def del_font_name(self):
        return self.del_style('font_name')

    def get_font_size(self):
        return self.get_style('font_size')

    def set_font_size(self, size):
        self.set_style(font_size=size)

    def del_font_size(self):
        return self.del_style('font_size')

    def get_bold(self):
        return self.get_style('bold')

    def set_bold(self, bold):
        self.set_style(bold=bold)

    def del_bold(self):
        return self.del_style('bold')

    def get_italic(self):
        return self.get_style('italic')

    def set_italic(self, italic):
        self.set_style(italic=italic)

    def del_italic(self):
        return self.del_style('italic')

    def get_underline(self):
        return self.get_style('underline') is not None

    def set_underline(self, underline):
        self.set_style(underline=underline)

    def del_underline(self):
        return self.del_style('underline') is not None

    def get_kerning(self):
        return self.get_style('kerning')

    def set_kerning(self, kerning):
        self.set_style(kerning=kerning)

    def del_kerning(self):
        return self.del_style('kerning')

    def get_baseline(self):
        return self.get_style('baseline')

    def set_baseline(self, baseline):
        self.set_style(baseline=baseline)

    def del_baseline(self):
        return self.del_style('baseline')

    def get_color(self):
        return self.get_style('color')

    def set_color(self, color):
        self.set_style(color=color)

    def del_color(self):
        return self.del_style('color')

    def get_background_color(self):
        return self.get_style('background_color')

    def set_background_color(self, color):
        self.set_style(background_color=color)

    def del_background_color(self):
        return self.del_style('background_color')

    def get_text_alignment(self):
        return self.get_style('align')

    def set_text_alignment(self, alignment):
        self.set_style(align=alignment)

    def del_text_alignment(self):
        return self.del_style('align')

    def get_line_spacing(self):
        return self.get_style('line_spacing')

    def set_line_spacing(self, spacing):
        self.set_style(line_spacing=spacing)

    def del_line_spacing(self):
        self.del_style('line_spacing')

    def enable_line_wrap(self, width):
        self._line_wrap_width = width
        self.repack()

    def disable_line_wrap(self):
        self.enable_line_wrap(0)

    def get_style(self, style):
        return self._style.get(style)

    def set_style(self, **style):
        self._style.update({k:v for k,v in style.items() if v is not None})
        self._update_style()

    def del_style(self, style):
        del self._style[style]
        self._update_style()

    def _update_style(self):
        # I want users to be able to specify colors using strings or Color
        # objects, but pyglet expects tuples, so make the conversion here.

        if 'color' in self._style:
            self._style['color'] = drawing.Color.from_anything(
                    self._style['color']).tuple

        if 'background_color' in self._style:
            self._style['background_color'] = drawing.Color.from_anything(
                    self._style['background_color']).tuple

        # I want the underline attribute to behave as a boolean, but in the
        # TextLayout API it's a color.  So when it's set to either True or
        # False, I need to translate that to either being a color or not being
        # in the style dictionary.

        if 'underline' in self._style:
            if not self._style['underline']:
                del self._style['underline']
            else:
                self._style['underline'] = self.color

        self.repack()

Label.register_event_type('on_edit_text')

@autoprop
class EditableLabel(Label):
    custom_selection_color = 'black'
    custom_selection_background_color = None
    custom_unfocus_on_enter = True

    def __init__(self, text="", line_wrap=None, **style):
        super().__init__(text, line_wrap, **style)
        self._caret = None
        self._focus = False
        self._is_mouse_over = False
        self._unfocus_on_enter = self.custom_unfocus_on_enter

        # I'm surprised pyglet doesn't treat the selection colors like all the
        # other styles.  Instead they're attributes of IncrementalTextLayout.
        self._selection_color = self.custom_selection_color
        self._selection_background_color = self.custom_selection_background_color

    def do_claim(self):
        font = pyglet.font.load(self.font_name)
        min_size = font.ascent - font.descent
        return min_size, min_size

    def focus(self):
        # Push handlers directly to the window, so even if the user has
        # attached their own handlers (e.g. for hotkeys) above the GUI, the
        # form will still take focus.

        if not self._focus:
            self._focus = True
            self._caret.on_activate()
            self.window.push_handlers(self._caret)
            self.window.push_handlers(
                    on_mouse_press=self.on_window_mouse_press,
                    on_key_press=self.on_window_key_press,
                    on_key_release=self.on_window_key_release,
            )
            self.dispatch_event('on_focus', self)

    def unfocus(self):
        # Pop the handlers that were pushed to the window by focus().  Be
        # careful: this just pops the top handlers (as far as I know, pyglet
        # doesn't provide a way to be more specific), so if some other handlers
        # were pushed in the meantime, they'll be popped instead of the ones
        # pushed by the form.

        if self._focus:
            self._focus = False
            self._caret.on_deactivate()
            self.window.pop_handlers()
            self.window.pop_handlers()
            self.dispatch_event('on_unfocus', self)

    def on_mouse_enter(self, x, y):
        super().on_mouse_enter(x, y)
        self._is_mouse_over = True

    def on_mouse_leave(self, x, y):
        super().on_mouse_leave(x, y)
        self._is_mouse_over = False

    def on_mouse_press(self, x, y, button, modifiers):
        if not self._focus:
            self.focus()
            self._caret.on_mouse_press(x, y, button, modifiers)

    def on_window_mouse_press(self, x, y, button, modifiers):
        # Determine if the mouse is over the form by tracking mouse enter and
        # leave events.  This is more robust than checking the mouse
        # coordinates in this method, because it still works when the form has
        # a parent that changes its coordinates, like a ScrollBox.

        if not self._is_mouse_over:
            self.unfocus()

            # This event will get swallowed by the caret, so dispatch a new
            # event after the caret handlers have been popped.

            self.window.dispatch_event('on_mouse_press', x, y, button, modifiers)

    def on_window_key_press(self, symbol, modifiers):
        if self._unfocus_on_enter and symbol == pyglet.window.key.ENTER:
            self.unfocus()
        return True

    def on_window_key_release(self, symbol, modifiers):
        return True

    def do_make_new_layout(self, document, kwargs):
        # Create a layout and a caret that can be used to edit it.
        layout = pyglet.text.layout.IncrementalTextLayout(document, **kwargs)

        layout.selection_color = drawing.Color.from_anything(
                self._selection_color).tuple
        layout.selection_background_color = drawing.Color.from_anything(
                self._selection_background_color or self.color).tuple

        self._caret = pyglet.text.caret.Caret(layout, color=self.color[:3])
        self._caret.on_deactivate()

        return layout

    def get_selection_color(self):
        return self._selection_color

    def set_selection_color(self, new_color):
        self._selection_color = new_color
        self.draw()

    def get_selection_background_color(self):
        return self._selection_background_color

    def set_selection_background_color(self, new_color):
        self._selection_background_color = new_color
        self.draw()

    def get_unfocus_on_enter(self):
        return self._unfocus_on_enter

    def set_unfocus_on_enter(self, new_behavior):
        self._unfocus_on_enter = new_behavior

EditableLabel.register_event_type('on_focus')
EditableLabel.register_event_type('on_unfocus')

class Form(Widget):
    Label = EditableLabel
    Base = Background
    Focused = None
    Deck = Deck

    def __init__(self, text=""):
        super().__init__()

        self._stack = Stack()
        self._label = self.Label(text)

        # If there are two backgrounds, create a deck to switch between them.
        # Otherwise skip the extra layer of hierarchy.
        if self.Focused is None:
            self._bg = self.Base()

        else:
            self._bg = self.Deck('base')
            self._bg.add_states(
                    base=self.Base(),
                    focused=self.Focused(),
            )
            self._label.push_handlers(
                    on_focus=lambda w: self._bg.set_state('focused'),
                    on_unfocus=lambda w: self._bg.set_state('base'),
            )

        self._stack.add_front(self._label)
        self._stack.add_back(self._bg)
        self._attach_child(self._stack)

    def get_label(self):
        return self._label

    def get_text(self):
        return self._label.text

    def set_text(self, new_text):
        self._label.text = new_text

<commit_msg>Don't rely on pop_handlers() in EditableLabel.unfocus().<commit_after>#!/usr/bin/env python3

import pyglet
import autoprop

from vecrec import Vector, Rect
from debugtools import p, pp, pv
from glooey import drawing
from glooey.widget import Widget
from glooey.images import Background
from glooey.containers import Stack, Deck

@autoprop
class Label(Widget):
    custom_text = ""
    custom_font_name = None
    custom_font_size = None
    custom_bold = None
    custom_italic = None
    custom_underline = None
    custom_kerning = None
    custom_baseline = None
    custom_color = 'green'
    custom_background_color = None
    custom_text_alignment = None
    custom_line_spacing = None

    def __init__(self, text="", line_wrap=None, **style):
        super().__init__()
        self._layout = None
        self._text = text or self.custom_text
        self._line_wrap_width = 0
        self._style = {}
        self.set_style(
                font_name=self.custom_font_name,
                font_size=self.custom_font_size,
                bold=self.custom_bold,
                italic=self.custom_italic,
                underline=self.custom_underline,
                kerning=self.custom_kerning,
                baseline=self.custom_baseline,
                color=self.custom_color,
                background_color=self.custom_background_color,
                align=self.custom_text_alignment,
                line_spacing=self.custom_line_spacing,
        )
        self.set_style(**style)
        if line_wrap:
            self.enable_line_wrap(line_wrap)

    def do_claim(self):
        # Make sure the label's text and style are up-to-date before we request
        # space.  Be careful!  This means that do_draw() can be called before
        # the widget has self.rect or self.group, which usually cannot happen.
        self.do_draw(ignore_rect=True)

        # Return the amount of space needed to render the label.
        return self._layout.content_width, self._layout.content_height

    def do_draw(self, ignore_rect=False):
        # Any time we need to draw this widget, just delete the underlying
        # label object and make a new one.  This isn't any slower than keeping
        # the old object, because all the vertex lists would be redrawn either
        # way.  And this is more flexible, because it allows us to reset the
        # batch, the group, and the wrap_lines attribute.
        if self._layout is not None:
            self._layout.delete()

        kwargs = {
                'multiline': True,
                'wrap_lines': False,
                'batch': self.batch,
                'group': self.group
        }
        # Enable line wrapping, if the user requested it.  The width of the
        # label is set to the value given by the user when line-wrapping was
        # enabled.  This will be overwritten a few lines later if the widget
        # has been given a rectangle.
        if self._line_wrap_width:
            kwargs['width'] = self._line_wrap_width
            kwargs['wrap_lines'] = True

        # Usually self.rect is guaranteed to be set by the time this method is
        # called, but that is not the case for this widget.  The do_claim()
        # method needs to call do_draw() to see how much space the text will
        # need, and that happens before self.rect is set (since it's part of
        # the process of setting self.rect).
        if not ignore_rect:
            kwargs['width'] = self.rect.width
            kwargs['height'] = self.rect.height

        # It's best to make a fresh document each time.  Previously I was
        # storing the document as a member variable, but I ran into corner
        # cases where the document would have an old style that wouldn't be
        # compatible with the new TextLayout (specifically 'align' != 'left' if
        # line wrapping is no loner enabled).
        document = pyglet.text.decode_text(self._text)
        document.push_handlers(self.on_insert_text, self.on_delete_text)
        self._layout = self.do_make_new_layout(document, kwargs)

        # Use begin_update() and end_update() to prevent the layout from
        # generating new vertex lists until the styles and coordinates have
        # been set.
        self._layout.begin_update()

        # The layout will crash if it doesn't have an explicit width and the
        # style specifies an alignment.  I
        if self._layout.width is None:
            self._layout.width = self._layout.content_width

        document.set_style(0, len(self._text), self._style)

        if not ignore_rect:
            self._layout.x = self.rect.bottom_left.x
            self._layout.y = self.rect.bottom_left.y

        self._layout.end_update()

    def do_undraw(self):
        if self._layout is not None:
            self._layout.delete()

    def do_make_new_layout(self, document, kwargs):
        return pyglet.text.layout.TextLayout(document, **kwargs)

    def on_insert_text(self, start, text):
        self._text = self._layout.document.text
        self.dispatch_event('on_edit_text', self)

    def on_delete_text(self, start, end):
        self._text = self._layout.document.text
        self.dispatch_event('on_edit_text', self)

    def get_text(self):
        return self._layout.document.text

    def set_text(self, text, width=None, **style):
        self._text = text
        if width is not None:
            self._line_wrap_width = width
        # This will repack.
        self.set_style(**style)

    def del_text(self):
        self.set_text("")

    def get_font_name(self):
        return self.get_style('font_name')

    def set_font_name(self, name):
        self.set_style(font_name=name)

    def del_font_name(self):
        return self.del_style('font_name')

    def get_font_size(self):
        return self.get_style('font_size')

    def set_font_size(self, size):
        self.set_style(font_size=size)

    def del_font_size(self):
        return self.del_style('font_size')

    def get_bold(self):
        return self.get_style('bold')

    def set_bold(self, bold):
        self.set_style(bold=bold)

    def del_bold(self):
        return self.del_style('bold')

    def get_italic(self):
        return self.get_style('italic')

    def set_italic(self, italic):
        self.set_style(italic=italic)

    def del_italic(self):
        return self.del_style('italic')

    def get_underline(self):
        return self.get_style('underline') is not None

    def set_underline(self, underline):
        self.set_style(underline=underline)

    def del_underline(self):
        return self.del_style('underline') is not None

    def get_kerning(self):
        return self.get_style('kerning')

    def set_kerning(self, kerning):
        self.set_style(kerning=kerning)

    def del_kerning(self):
        return self.del_style('kerning')

    def get_baseline(self):
        return self.get_style('baseline')

    def set_baseline(self, baseline):
        self.set_style(baseline=baseline)

    def del_baseline(self):
        return self.del_style('baseline')

    def get_color(self):
        return self.get_style('color')

    def set_color(self, color):
        self.set_style(color=color)

    def del_color(self):
        return self.del_style('color')

    def get_background_color(self):
        return self.get_style('background_color')

    def set_background_color(self, color):
        self.set_style(background_color=color)

    def del_background_color(self):
        return self.del_style('background_color')

    def get_text_alignment(self):
        return self.get_style('align')

    def set_text_alignment(self, alignment):
        self.set_style(align=alignment)

    def del_text_alignment(self):
        return self.del_style('align')

    def get_line_spacing(self):
        return self.get_style('line_spacing')

    def set_line_spacing(self, spacing):
        self.set_style(line_spacing=spacing)

    def del_line_spacing(self):
        self.del_style('line_spacing')

    def enable_line_wrap(self, width):
        self._line_wrap_width = width
        self.repack()

    def disable_line_wrap(self):
        self.enable_line_wrap(0)

    def get_style(self, style):
        return self._style.get(style)

    def set_style(self, **style):
        self._style.update({k:v for k,v in style.items() if v is not None})
        self._update_style()

    def del_style(self, style):
        del self._style[style]
        self._update_style()

    def _update_style(self):
        # I want users to be able to specify colors using strings or Color
        # objects, but pyglet expects tuples, so make the conversion here.

        if 'color' in self._style:
            self._style['color'] = drawing.Color.from_anything(
                    self._style['color']).tuple

        if 'background_color' in self._style:
            self._style['background_color'] = drawing.Color.from_anything(
                    self._style['background_color']).tuple

        # I want the underline attribute to behave as a boolean, but in the
        # TextLayout API it's a color.  So when it's set to either True or
        # False, I need to translate that to either being a color or not being
        # in the style dictionary.

        if 'underline' in self._style:
            if not self._style['underline']:
                del self._style['underline']
            else:
                self._style['underline'] = self.color

        self.repack()

Label.register_event_type('on_edit_text')

@autoprop
class EditableLabel(Label):
    custom_selection_color = 'black'
    custom_selection_background_color = None
    custom_unfocus_on_enter = True

    def __init__(self, text="", line_wrap=None, **style):
        super().__init__(text, line_wrap, **style)
        self._caret = None
        self._focus = False
        self._is_mouse_over = False
        self._unfocus_on_enter = self.custom_unfocus_on_enter

        # I'm surprised pyglet doesn't treat the selection colors like all the
        # other styles.  Instead they're attributes of IncrementalTextLayout.
        self._selection_color = self.custom_selection_color
        self._selection_background_color = self.custom_selection_background_color

    def do_claim(self):
        font = pyglet.font.load(self.font_name)
        min_size = font.ascent - font.descent
        return min_size, min_size

    def focus(self):
        # Push handlers directly to the window, so even if the user has
        # attached their own handlers (e.g. for hotkeys) above the GUI, the
        # form will still take focus.

        if not self._focus:
            self._focus = True
            self._caret.on_activate()
            self.window.push_handlers(self._caret)
            self.window.push_handlers(
                    on_mouse_press=self.on_window_mouse_press,
                    on_key_press=self.on_window_key_press,
                    on_key_release=self.on_window_key_release,
            )
            self.dispatch_event('on_focus', self)

    def unfocus(self):
        if self._focus:
            self._focus = False
            self._caret.on_deactivate()
            self._layout.set_selection(0,0)
            self.window.remove_handlers(self._caret)
            self.window.remove_handlers(
                    on_mouse_press=self.on_window_mouse_press,
                    on_key_press=self.on_window_key_press,
                    on_key_release=self.on_window_key_release,
            )
            self.dispatch_event('on_unfocus', self)

    def on_mouse_enter(self, x, y):
        super().on_mouse_enter(x, y)
        self._is_mouse_over = True

    def on_mouse_leave(self, x, y):
        super().on_mouse_leave(x, y)
        self._is_mouse_over = False

    def on_mouse_press(self, x, y, button, modifiers):
        if not self._focus:
            self.focus()
            self._caret.on_mouse_press(x, y, button, modifiers)

    def on_window_mouse_press(self, x, y, button, modifiers):
        # Determine if the mouse is over the form by tracking mouse enter and
        # leave events.  This is more robust than checking the mouse
        # coordinates in this method, because it still works when the form has
        # a parent that changes its coordinates, like a ScrollBox.

        if not self._is_mouse_over:
            self.unfocus()

            # This event will get swallowed by the caret, so dispatch a new
            # event after the caret handlers have been popped.

            self.window.dispatch_event('on_mouse_press', x, y, button, modifiers)

    def on_window_key_press(self, symbol, modifiers):
        if self._unfocus_on_enter and symbol == pyglet.window.key.ENTER:
            self.unfocus()
        return True

    def on_window_key_release(self, symbol, modifiers):
        return True

    def do_make_new_layout(self, document, kwargs):
        # Create a layout and a caret that can be used to edit it.
        layout = pyglet.text.layout.IncrementalTextLayout(document, **kwargs)

        layout.selection_color = drawing.Color.from_anything(
                self._selection_color).tuple
        layout.selection_background_color = drawing.Color.from_anything(
                self._selection_background_color or self.color).tuple

        self._caret = pyglet.text.caret.Caret(layout, color=self.color[:3])
        self._caret.on_deactivate()

        return layout

    def get_selection_color(self):
        return self._selection_color

    def set_selection_color(self, new_color):
        self._selection_color = new_color
        self.draw()

    def get_selection_background_color(self):
        return self._selection_background_color

    def set_selection_background_color(self, new_color):
        self._selection_background_color = new_color
        self.draw()

    def get_unfocus_on_enter(self):
        return self._unfocus_on_enter

    def set_unfocus_on_enter(self, new_behavior):
        self._unfocus_on_enter = new_behavior

EditableLabel.register_event_type('on_focus')
EditableLabel.register_event_type('on_unfocus')

class Form(Widget):
    Label = EditableLabel
    Base = Background
    Focused = None
    Deck = Deck

    def __init__(self, text=""):
        super().__init__()

        self._stack = Stack()
        self._label = self.Label(text)

        # If there are two backgrounds, create a deck to switch between them.
        # Otherwise skip the extra layer of hierarchy.
        if self.Focused is None:
            self._bg = self.Base()

        else:
            self._bg = self.Deck('base')
            self._bg.add_states(
                    base=self.Base(),
                    focused=self.Focused(),
            )
            self._label.push_handlers(
                    on_focus=lambda w: self._bg.set_state('focused'),
                    on_unfocus=lambda w: self._bg.set_state('base'),
            )

        self._stack.add_front(self._label)
        self._stack.add_back(self._bg)
        self._attach_child(self._stack)

    def get_label(self):
        return self._label

    def get_text(self):
        return self._label.text

    def set_text(self, new_text):
        self._label.text = new_text

<|endoftext|>


<commit_before>"""Evaluates predictions from CNN."""

import random
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import argparse
import numpy
from keras import backend as K
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.deep_learning import cnn
from gewittergefahr.deep_learning import input_examples
from gewittergefahr.deep_learning import testing_io
from gewittergefahr.deep_learning import training_validation_io as trainval_io
from gewittergefahr.scripts import model_evaluation_helper as model_eval_helper

random.seed(6695)
numpy.random.seed(6695)

K.set_session(K.tf.Session(config=K.tf.ConfigProto(
    intra_op_parallelism_threads=1, inter_op_parallelism_threads=1,
    allow_soft_placement=False
)))

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

MODEL_FILE_ARG_NAME = 'input_model_file_name'
EXAMPLE_DIR_ARG_NAME = 'input_example_dir_name'
FIRST_SPC_DATE_ARG_NAME = 'first_spc_date_string'
LAST_SPC_DATE_ARG_NAME = 'last_spc_date_string'
NUM_EXAMPLES_ARG_NAME = 'num_examples'
NUM_BOOTSTRAP_ARG_NAME = 'num_bootstrap_reps'
CONFIDENCE_LEVEL_ARG_NAME = 'confidence_level'
CLASS_FRACTION_KEYS_ARG_NAME = 'class_fraction_keys'
CLASS_FRACTION_VALUES_ARG_NAME = 'class_fraction_values'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

MODEL_FILE_HELP_STRING = (
    'File path for trained CNN.  Will be read by `cnn.read_model`.')

EXAMPLE_DIR_HELP_STRING = (
    'Name of top-level directory with input examples.  Files therein will be '
    'found by `input_examples.find_example_file` and read by '
    '`input_examples.read_example_file`.')

SPC_DATE_HELP_STRING = (
    'SPC date (format "yyyymmdd").  This script will evaluate predictions on '
    'examples from the period `{0:s}`...`{1:s}`.'
).format(FIRST_SPC_DATE_ARG_NAME, LAST_SPC_DATE_ARG_NAME)

NUM_EXAMPLES_HELP_STRING = 'Number of examples to use.'

NUM_BOOTSTRAP_HELP_STRING = (
    'Number of bootstrap replicates.  If you do not want bootstrapping, leave '
    'this alone.')

CONFIDENCE_LEVEL_HELP_STRING = (
    '[used only if `{0:s}` > 1] Confidence level for bootstrapping, in range '
    '0...1.'
).format(NUM_BOOTSTRAP_ARG_NAME)

CLASS_FRACTION_KEYS_HELP_STRING = (
    'List of keys used to create input `class_to_sampling_fraction_dict` for '
    '`deep_learning_utils.sample_by_class`.  If you do not want class-'
    'conditional sampling, leave this alone.'
)

CLASS_FRACTION_VALUES_HELP_STRING = (
    'List of values used to create input `class_to_sampling_fraction_dict` for '
    '`deep_learning_utils.sample_by_class`.  If you do not want class-'
    'conditional sampling, leave this alone.'
)

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  Results will be saved here.')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + MODEL_FILE_ARG_NAME, type=str, required=True,
    help=MODEL_FILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + EXAMPLE_DIR_ARG_NAME, type=str, required=True,
    help=EXAMPLE_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + FIRST_SPC_DATE_ARG_NAME, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + LAST_SPC_DATE_ARG_NAME, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_EXAMPLES_ARG_NAME, type=int, required=True,
    help=NUM_EXAMPLES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_BOOTSTRAP_ARG_NAME, type=int, required=False, default=1,
    help=NUM_BOOTSTRAP_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CONFIDENCE_LEVEL_ARG_NAME, type=float, required=False, default=0.95,
    help=CONFIDENCE_LEVEL_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CLASS_FRACTION_KEYS_ARG_NAME, type=int, nargs='+',
    required=False, default=[0], help=CLASS_FRACTION_KEYS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CLASS_FRACTION_VALUES_ARG_NAME, type=float, nargs='+',
    required=False, default=[0.], help=CLASS_FRACTION_VALUES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _run(model_file_name, top_example_dir_name, first_spc_date_string,
         last_spc_date_string, num_examples, num_bootstrap_reps,
         confidence_level, class_fraction_keys, class_fraction_values,
         output_dir_name):
    """Evaluates CNN (convolutional neural net) predictions.

    This is effectively the main method.

    :param model_file_name: See documentation at top of file.
    :param top_example_dir_name: Same.
    :param first_spc_date_string: Same.
    :param last_spc_date_string: Same.
    :param num_examples: Same.
    :param num_bootstrap_reps: Same.
    :param confidence_level: Same.
    :param class_fraction_keys: Same.
    :param class_fraction_values: Same.
    :param output_dir_name: Same.
    :raises: ValueError: if the model does multi-class classification.
    """

    print('Reading model from: "{0:s}"...'.format(model_file_name))
    model_object = cnn.read_model(model_file_name)

    num_output_neurons = (
        model_object.layers[-1].output.get_shape().as_list()[-1]
    )

    if num_output_neurons > 2:
        error_string = (
            'The model has {0:d} output neurons, which suggests {0:d}-class '
            'classification.  This script handles only binary classification.'
        ).format(num_output_neurons)

        raise ValueError(error_string)

    model_directory_name, _ = os.path.split(model_file_name)
    model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name)

    print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
    model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]

    if len(class_fraction_keys) > 1:
        class_to_sampling_fraction_dict = dict(list(zip(
            class_fraction_keys, class_fraction_values
        )))
    else:
        class_to_sampling_fraction_dict = None

    training_option_dict[
        trainval_io.SAMPLING_FRACTIONS_KEY
    ] = class_to_sampling_fraction_dict

    example_file_names = input_examples.find_many_example_files(
        top_directory_name=top_example_dir_name, shuffled=False,
        first_spc_date_string=first_spc_date_string,
        last_spc_date_string=last_spc_date_string,
        raise_error_if_any_missing=False)

    training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names
    training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = (
        time_conversion.get_start_of_spc_date(first_spc_date_string)
    )
    training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = (
        time_conversion.get_end_of_spc_date(last_spc_date_string)
    )

    if model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None:
        generator_object = testing_io.gridrad_generator_2d_reduced(
            option_dict=training_option_dict,
            list_of_operation_dicts=model_metadata_dict[
                cnn.LAYER_OPERATIONS_KEY],
            num_examples_total=num_examples
        )

    elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
        generator_object = testing_io.myrorss_generator_2d3d(
            option_dict=training_option_dict, num_examples_total=num_examples)
    else:
        generator_object = testing_io.generator_2d_or_3d(
            option_dict=training_option_dict, num_examples_total=num_examples)

    include_soundings = (
        training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None
    )

    forecast_probabilities = numpy.array([])
    observed_labels = numpy.array([], dtype=int)

    for _ in range(len(example_file_names)):
        try:
            this_storm_object_dict = next(generator_object)
            print(SEPARATOR_STRING)
        except StopIteration:
            break

        observed_labels = numpy.concatenate((
            observed_labels, this_storm_object_dict[testing_io.TARGET_ARRAY_KEY]
        ))

        these_predictor_matrices = this_storm_object_dict[
            testing_io.INPUT_MATRICES_KEY]

        if include_soundings:
            this_sounding_matrix = these_predictor_matrices[-1]
        else:
            this_sounding_matrix = None

        if model_metadata_dict[cnn.CONV_2D3D_KEY]:
            if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
                this_probability_matrix = cnn.apply_2d_or_3d_cnn(
                    model_object=model_object,
                    radar_image_matrix=these_predictor_matrices[0],
                    sounding_matrix=this_sounding_matrix, verbose=True)
            else:
                this_probability_matrix = cnn.apply_2d3d_cnn(
                    model_object=model_object,
                    reflectivity_matrix_dbz=these_predictor_matrices[0],
                    azimuthal_shear_matrix_s01=these_predictor_matrices[1],
                    sounding_matrix=this_sounding_matrix, verbose=True)
        else:
            this_probability_matrix = cnn.apply_2d_or_3d_cnn(
                model_object=model_object,
                radar_image_matrix=these_predictor_matrices[0],
                sounding_matrix=this_sounding_matrix, verbose=True)

        print(SEPARATOR_STRING)

        forecast_probabilities = numpy.concatenate((
            forecast_probabilities, this_probability_matrix[:, -1]
        ))

    model_eval_helper.run_evaluation(
        forecast_probabilities=forecast_probabilities,
        observed_labels=observed_labels, num_bootstrap_reps=num_bootstrap_reps,
        confidence_level=confidence_level, output_dir_name=output_dir_name)

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        model_file_name=getattr(INPUT_ARG_OBJECT, MODEL_FILE_ARG_NAME),
        top_example_dir_name=getattr(INPUT_ARG_OBJECT, EXAMPLE_DIR_ARG_NAME),
        first_spc_date_string=getattr(
            INPUT_ARG_OBJECT, FIRST_SPC_DATE_ARG_NAME),
        last_spc_date_string=getattr(INPUT_ARG_OBJECT, LAST_SPC_DATE_ARG_NAME),
        num_examples=getattr(INPUT_ARG_OBJECT, NUM_EXAMPLES_ARG_NAME),
        num_bootstrap_reps=getattr(INPUT_ARG_OBJECT, NUM_BOOTSTRAP_ARG_NAME),
        confidence_level=getattr(INPUT_ARG_OBJECT, CONFIDENCE_LEVEL_ARG_NAME),
        class_fraction_keys=numpy.array(
            getattr(INPUT_ARG_OBJECT, CLASS_FRACTION_KEYS_ARG_NAME), dtype=int),
        class_fraction_values=numpy.array(
            getattr(INPUT_ARG_OBJECT, CLASS_FRACTION_VALUES_ARG_NAME),
            dtype=float),
        output_dir_name=getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_ARG_NAME)
    )
<commit_msg>evaluate_cnn.py now can handle nets that take only soundings as input.<commit_after>"""Evaluates predictions from CNN."""

import random
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import argparse
import numpy
from keras import backend as K
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.deep_learning import cnn
from gewittergefahr.deep_learning import input_examples
from gewittergefahr.deep_learning import testing_io
from gewittergefahr.deep_learning import training_validation_io as trainval_io
from gewittergefahr.scripts import model_evaluation_helper as model_eval_helper

random.seed(6695)
numpy.random.seed(6695)

K.set_session(K.tf.Session(config=K.tf.ConfigProto(
    intra_op_parallelism_threads=1, inter_op_parallelism_threads=1,
    allow_soft_placement=False
)))

SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n'

MODEL_FILE_ARG_NAME = 'input_model_file_name'
EXAMPLE_DIR_ARG_NAME = 'input_example_dir_name'
FIRST_SPC_DATE_ARG_NAME = 'first_spc_date_string'
LAST_SPC_DATE_ARG_NAME = 'last_spc_date_string'
NUM_EXAMPLES_ARG_NAME = 'num_examples'
NUM_BOOTSTRAP_ARG_NAME = 'num_bootstrap_reps'
CONFIDENCE_LEVEL_ARG_NAME = 'confidence_level'
CLASS_FRACTION_KEYS_ARG_NAME = 'class_fraction_keys'
CLASS_FRACTION_VALUES_ARG_NAME = 'class_fraction_values'
OUTPUT_DIR_ARG_NAME = 'output_dir_name'

MODEL_FILE_HELP_STRING = (
    'File path for trained CNN.  Will be read by `cnn.read_model`.')

EXAMPLE_DIR_HELP_STRING = (
    'Name of top-level directory with input examples.  Files therein will be '
    'found by `input_examples.find_example_file` and read by '
    '`input_examples.read_example_file`.')

SPC_DATE_HELP_STRING = (
    'SPC date (format "yyyymmdd").  This script will evaluate predictions on '
    'examples from the period `{0:s}`...`{1:s}`.'
).format(FIRST_SPC_DATE_ARG_NAME, LAST_SPC_DATE_ARG_NAME)

NUM_EXAMPLES_HELP_STRING = 'Number of examples to use.'

NUM_BOOTSTRAP_HELP_STRING = (
    'Number of bootstrap replicates.  If you do not want bootstrapping, leave '
    'this alone.')

CONFIDENCE_LEVEL_HELP_STRING = (
    '[used only if `{0:s}` > 1] Confidence level for bootstrapping, in range '
    '0...1.'
).format(NUM_BOOTSTRAP_ARG_NAME)

CLASS_FRACTION_KEYS_HELP_STRING = (
    'List of keys used to create input `class_to_sampling_fraction_dict` for '
    '`deep_learning_utils.sample_by_class`.  If you do not want class-'
    'conditional sampling, leave this alone.'
)

CLASS_FRACTION_VALUES_HELP_STRING = (
    'List of values used to create input `class_to_sampling_fraction_dict` for '
    '`deep_learning_utils.sample_by_class`.  If you do not want class-'
    'conditional sampling, leave this alone.'
)

OUTPUT_DIR_HELP_STRING = (
    'Name of output directory.  Results will be saved here.')

INPUT_ARG_PARSER = argparse.ArgumentParser()
INPUT_ARG_PARSER.add_argument(
    '--' + MODEL_FILE_ARG_NAME, type=str, required=True,
    help=MODEL_FILE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + EXAMPLE_DIR_ARG_NAME, type=str, required=True,
    help=EXAMPLE_DIR_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + FIRST_SPC_DATE_ARG_NAME, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + LAST_SPC_DATE_ARG_NAME, type=str, required=True,
    help=SPC_DATE_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_EXAMPLES_ARG_NAME, type=int, required=True,
    help=NUM_EXAMPLES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + NUM_BOOTSTRAP_ARG_NAME, type=int, required=False, default=1,
    help=NUM_BOOTSTRAP_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CONFIDENCE_LEVEL_ARG_NAME, type=float, required=False, default=0.95,
    help=CONFIDENCE_LEVEL_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CLASS_FRACTION_KEYS_ARG_NAME, type=int, nargs='+',
    required=False, default=[0], help=CLASS_FRACTION_KEYS_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + CLASS_FRACTION_VALUES_ARG_NAME, type=float, nargs='+',
    required=False, default=[0.], help=CLASS_FRACTION_VALUES_HELP_STRING)

INPUT_ARG_PARSER.add_argument(
    '--' + OUTPUT_DIR_ARG_NAME, type=str, required=True,
    help=OUTPUT_DIR_HELP_STRING)

def _run(model_file_name, top_example_dir_name, first_spc_date_string,
         last_spc_date_string, num_examples, num_bootstrap_reps,
         confidence_level, class_fraction_keys, class_fraction_values,
         output_dir_name):
    """Evaluates CNN (convolutional neural net) predictions.

    This is effectively the main method.

    :param model_file_name: See documentation at top of file.
    :param top_example_dir_name: Same.
    :param first_spc_date_string: Same.
    :param last_spc_date_string: Same.
    :param num_examples: Same.
    :param num_bootstrap_reps: Same.
    :param confidence_level: Same.
    :param class_fraction_keys: Same.
    :param class_fraction_values: Same.
    :param output_dir_name: Same.
    :raises: ValueError: if the model does multi-class classification.
    """

    print('Reading model from: "{0:s}"...'.format(model_file_name))
    model_object = cnn.read_model(model_file_name)

    num_output_neurons = (
        model_object.layers[-1].output.get_shape().as_list()[-1]
    )

    if num_output_neurons > 2:
        error_string = (
            'The model has {0:d} output neurons, which suggests {0:d}-class '
            'classification.  This script handles only binary classification.'
        ).format(num_output_neurons)

        raise ValueError(error_string)

    soundings_only = False

    if isinstance(model_object.input, list):
        list_of_input_tensors = model_object.input
    else:
        list_of_input_tensors = [model_object.input]

    if len(list_of_input_tensors) == 1:
        these_spatial_dim = numpy.array(
            list_of_input_tensors[0].get_shape().as_list()[1:-1], dtype=int
        )
        soundings_only = len(these_spatial_dim) == 1

    model_directory_name, _ = os.path.split(model_file_name)
    model_metafile_name = '{0:s}/model_metadata.p'.format(model_directory_name)

    print('Reading metadata from: "{0:s}"...'.format(model_metafile_name))
    model_metadata_dict = cnn.read_model_metadata(model_metafile_name)
    training_option_dict = model_metadata_dict[cnn.TRAINING_OPTION_DICT_KEY]

    if len(class_fraction_keys) > 1:
        class_to_sampling_fraction_dict = dict(list(zip(
            class_fraction_keys, class_fraction_values
        )))
    else:
        class_to_sampling_fraction_dict = None

    training_option_dict[
        trainval_io.SAMPLING_FRACTIONS_KEY
    ] = class_to_sampling_fraction_dict

    example_file_names = input_examples.find_many_example_files(
        top_directory_name=top_example_dir_name, shuffled=False,
        first_spc_date_string=first_spc_date_string,
        last_spc_date_string=last_spc_date_string,
        raise_error_if_any_missing=False)

    training_option_dict[trainval_io.EXAMPLE_FILES_KEY] = example_file_names
    training_option_dict[trainval_io.FIRST_STORM_TIME_KEY] = (
        time_conversion.get_start_of_spc_date(first_spc_date_string)
    )
    training_option_dict[trainval_io.LAST_STORM_TIME_KEY] = (
        time_conversion.get_end_of_spc_date(last_spc_date_string)
    )

    if soundings_only:
        generator_object = testing_io.sounding_generator(
            option_dict=training_option_dict, num_examples_total=num_examples)

    elif model_metadata_dict[cnn.LAYER_OPERATIONS_KEY] is not None:
        generator_object = testing_io.gridrad_generator_2d_reduced(
            option_dict=training_option_dict,
            list_of_operation_dicts=model_metadata_dict[
                cnn.LAYER_OPERATIONS_KEY],
            num_examples_total=num_examples
        )

    elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
        generator_object = testing_io.myrorss_generator_2d3d(
            option_dict=training_option_dict, num_examples_total=num_examples)
    else:
        generator_object = testing_io.generator_2d_or_3d(
            option_dict=training_option_dict, num_examples_total=num_examples)

    include_soundings = (
        training_option_dict[trainval_io.SOUNDING_FIELDS_KEY] is not None
    )

    forecast_probabilities = numpy.array([])
    observed_labels = numpy.array([], dtype=int)

    for _ in range(len(example_file_names)):
        try:
            this_storm_object_dict = next(generator_object)
            print(SEPARATOR_STRING)
        except StopIteration:
            break

        observed_labels = numpy.concatenate((
            observed_labels, this_storm_object_dict[testing_io.TARGET_ARRAY_KEY]
        ))

        these_predictor_matrices = this_storm_object_dict[
            testing_io.INPUT_MATRICES_KEY]

        if include_soundings:
            this_sounding_matrix = these_predictor_matrices[-1]
        else:
            this_sounding_matrix = None

        if soundings_only:
            this_probability_matrix = cnn.apply_cnn_soundings_only(
                model_object=model_object, sounding_matrix=this_sounding_matrix,
                verbose=True)
        elif model_metadata_dict[cnn.CONV_2D3D_KEY]:
            if training_option_dict[trainval_io.UPSAMPLE_REFLECTIVITY_KEY]:
                this_probability_matrix = cnn.apply_2d_or_3d_cnn(
                    model_object=model_object,
                    radar_image_matrix=these_predictor_matrices[0],
                    sounding_matrix=this_sounding_matrix, verbose=True)
            else:
                this_probability_matrix = cnn.apply_2d3d_cnn(
                    model_object=model_object,
                    reflectivity_matrix_dbz=these_predictor_matrices[0],
                    azimuthal_shear_matrix_s01=these_predictor_matrices[1],
                    sounding_matrix=this_sounding_matrix, verbose=True)
        else:
            this_probability_matrix = cnn.apply_2d_or_3d_cnn(
                model_object=model_object,
                radar_image_matrix=these_predictor_matrices[0],
                sounding_matrix=this_sounding_matrix, verbose=True)

        print(SEPARATOR_STRING)

        forecast_probabilities = numpy.concatenate((
            forecast_probabilities, this_probability_matrix[:, -1]
        ))

    model_eval_helper.run_evaluation(
        forecast_probabilities=forecast_probabilities,
        observed_labels=observed_labels, num_bootstrap_reps=num_bootstrap_reps,
        confidence_level=confidence_level, output_dir_name=output_dir_name)

if __name__ == '__main__':
    INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args()

    _run(
        model_file_name=getattr(INPUT_ARG_OBJECT, MODEL_FILE_ARG_NAME),
        top_example_dir_name=getattr(INPUT_ARG_OBJECT, EXAMPLE_DIR_ARG_NAME),
        first_spc_date_string=getattr(
            INPUT_ARG_OBJECT, FIRST_SPC_DATE_ARG_NAME),
        last_spc_date_string=getattr(INPUT_ARG_OBJECT, LAST_SPC_DATE_ARG_NAME),
        num_examples=getattr(INPUT_ARG_OBJECT, NUM_EXAMPLES_ARG_NAME),
        num_bootstrap_reps=getattr(INPUT_ARG_OBJECT, NUM_BOOTSTRAP_ARG_NAME),
        confidence_level=getattr(INPUT_ARG_OBJECT, CONFIDENCE_LEVEL_ARG_NAME),
        class_fraction_keys=numpy.array(
            getattr(INPUT_ARG_OBJECT, CLASS_FRACTION_KEYS_ARG_NAME), dtype=int),
        class_fraction_values=numpy.array(
            getattr(INPUT_ARG_OBJECT, CLASS_FRACTION_VALUES_ARG_NAME),
            dtype=float),
        output_dir_name=getattr(INPUT_ARG_OBJECT, OUTPUT_DIR_ARG_NAME)
    )
<|endoftext|>


<commit_before>{
    'target_defaults': {
        'default_configuration': 'Release',
        'configurations': {
            'Debug': {
                'defines': [ 'DEBUG', '_DEBUG' ],
                'msvs_settings': {
                    'VCCLCompilerTool': {
                        'RuntimeLibrary': 1,
                    },
                    'VCLinkerTool': {
                        'LinkTimeCodeGeneration': 1,
                        'OptimizeReferences': 2,
                        'EnableCOMDATFolding': 2,
                        'LinkIncremental': 1,
                        'GenerateDebugInformation': 'true',
                        'AdditionalLibraryDirectories': [
                        ]
                    }
                },
                'xcode_settings': {
                    'OTHER_LDFLAGS': [
                    ]
                }
            },
            'Release': {
                'defines': [ 'NDEBUG' ],
                'msvs_settings': {
                    'VCCLCompilerTool': {
                        'RuntimeLibrary': 0,
                        'Optimization': 3,
                        'FavorSizeOrSpeed': 1,
                        'InlineFunctionExpansion': 2,
                        'WholeProgramOptimization': 'true',
                        'OmitFramePointers': 'true',
                        'EnableFunctionLevelLinking': 'true',
                        'EnableIntrinsicFunctions': 'true'
                    },
                    'VCLinkerTool': {
                        'LinkTimeCodeGeneration': 1,
                        'OptimizeReferences': 2,
                        'EnableCOMDATFolding': 2,
                        'LinkIncremental': 1,
                        'AdditionalLibraryDirectories': [
                        ]
                    }
                },
                'xcode_settings': {
                    'OTHER_LDFLAGS': [
                    ]
                }
            }
        }
    },

    'targets': [
        {
            'target_name': 'test_gyp',
                'type': 'executable',
                'include_dirs': [
                    'test',
                ],
                'sources': [
                    'test/test_gyp.cpp',
                ],
        }
    ],
}
<commit_msg>fix build error<commit_after>{
    'target_defaults': {
        'default_configuration': 'Release',
        'configurations': {
            'Debug': {
                'defines': [ 'DEBUG', '_DEBUG' ],
                'msvs_settings': {
                    'VCCLCompilerTool': {
                        'RuntimeLibrary': 1,
                    },
                    'VCLinkerTool': {
                        'LinkTimeCodeGeneration': 1,
                        'OptimizeReferences': 2,
                        'EnableCOMDATFolding': 2,
                        'LinkIncremental': 1,
                        'GenerateDebugInformation': 'true',
                        'AdditionalLibraryDirectories': [
                        ]
                    }
                },
                'xcode_settings': {
                    'GCC_CW_ASM_SYNTAX': 'NO',
                    'GCC_ENABLE_PASCAL_STRINGS': 'NO',
                    'OTHER_LDFLAGS': [
                    ]
                },
            },
            'Release': {
                'defines': [ 'NDEBUG' ],
                'msvs_settings': {
                    'VCCLCompilerTool': {
                        'RuntimeLibrary': 0,
                        'Optimization': 3,
                        'FavorSizeOrSpeed': 1,
                        'InlineFunctionExpansion': 2,
                        'WholeProgramOptimization': 'true',
                        'OmitFramePointers': 'true',
                        'EnableFunctionLevelLinking': 'true',
                        'EnableIntrinsicFunctions': 'true'
                    },
                    'VCLinkerTool': {
                        'LinkTimeCodeGeneration': 1,
                        'OptimizeReferences': 2,
                        'EnableCOMDATFolding': 2,
                        'LinkIncremental': 1,
                        'AdditionalLibraryDirectories': [
                        ]
                    }
                },
                'xcode_settings': {
                    'GCC_CW_ASM_SYNTAX': 'NO',
                    'GCC_ENABLE_PASCAL_STRINGS': 'NO',
                    'OTHER_LDFLAGS': [
                    ]
                },
            }
        },
    },

    'targets': [
        {
            'target_name': 'test_gyp',
                'type': 'executable',
                'include_dirs': [
                    'test',
                ],
                'sources': [
                    'test/test_gyp.cpp',
                ],
        }
    ],
}
<|endoftext|>


<commit_before>
import os
import json
from xml.etree import ElementTree
import requests
from requests_futures.sessions import FuturesSession

from usgs import USGS_API, USGSError
from usgs import soap, xsi

TMPFILE = os.path.join("/", "tmp", "usgs")
NAMESPACES = {
    "SOAP-ENV": "http://schemas.xmlsoap.org/soap/envelope/",
    "ns1": "https://earthexplorer.usgs.gov/inventory/soap",
    "eemetadata": "http://earthexplorer.usgs.gov/eemetadata.xsd"
}

def _get_api_key():

    api_key = None

    if os.path.exists(TMPFILE):
        with open(TMPFILE, "r") as f:
            api_key = f.read()

    return api_key

def _check_for_usgs_error(root):

    fault_code_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultcode", NAMESPACES)

    if fault_code_el is None:
        return

    fault_string_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", NAMESPACES)

    fault_code = fault_code_el.text
    fault_string = fault_string_el.text

    raise USGSError('%s: %s' % (fault_code, fault_string))

def _get_extended(scene, resp):
    """
    Parse metadata returned from the metadataUrl of a USGS scene.

    :param scene:
        Dictionary representation of a USGS scene
    :param resp:
        Response object from requests/grequests
    """
    root = ElementTree.fromstring(resp.text)
    items = root.findall("eemetadata:metadataFields/eemetadata:metadataField", NAMESPACES)
    scene['extended'] = { item.attrib.get('name').strip(): xsi.get(item[0]) for item in items }

    return scene

def _async_requests(urls):
    """
    Sends multiple non-blocking requests. Returns
    a list of responses.

    :param urls:
        List of urls
    """

    session = FuturesSession(max_workers=30)
    futures = [ session.get(url) for url in urls ]
    return [ future.result() for future in futures ]

def _get_metadata_url(scene):
    return scene.get('metadataUrl')

def clear_bulk_download_order():
    raise NotImplementedError

def clear_order():
    raise NotImplementedError

def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None):

    api_key = _get_api_key()

    xml = soap.datasets(dataset, node, ll=ll, ur=ur, start_date=start_date, end_date=end_date, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetsResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def dataset_fields(dataset, node):

    api_key = _get_api_key()

    xml = soap.dataset_fields(dataset, node, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetFieldsResponse/return/item", NAMESPACES)
    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def download(dataset, node, entityids, product):
    """
    Though USGS supports multiple products in a single request, there's
    ambiguity in the returned list. This wrapper only allows a single
    product per request.

    Additionally, the response has no indiction which URL is associated
    with which scene/entity id. The URL can be parsed, but the structure
    varies depending on the product.
    """

    api_key = _get_api_key()

    xml = soap.download(dataset, node, entityids, [product], api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadResponse/return/item", NAMESPACES)

    data = map(lambda el: xsi.get(el), items)

    return data

def download_options(dataset, node, entityids):

    api_key = _get_api_key()

    xml = soap.download_options(dataset, node, entityids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadOptionsResponse/return/item/downloadOptions/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def get_bulk_download_products():
    raise NotImplementedError

def get_order_products():
    raise NotImplementedError

def hits():
    raise NotImplementedError

def item_basket():
    raise NotImplementedError

def login(username, password):
    xml = soap.login(username, password)
    r = requests.post(USGS_API, xml)

    if r.status_code is not 200:
        raise USGSError(r.text)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    element = root.find("SOAP-ENV:Body/ns1:loginResponse/return", NAMESPACES)

    api_key = element.text

    with open(TMPFILE, "w") as f:
        f.write(api_key)

    return api_key

def logout():

    api_key = _get_api_key()

    xml = soap.logout(api_key=api_key)
    r = requests.post(USGS_API, xml)

    if os.path.exists(TMPFILE):
        os.remove(TMPFILE)

    return True

def metadata(dataset, node, sceneids, extended=False, api_key=None):
    """
    Request metadata for a given scene in a USGS dataset.

    :param dataset:
    :param node:
    :param sceneids:
    :param extended:
        Send a second request to the metadata url to get extended metadata on the scene.
    :param api_key:
    """
    api_key = _get_api_key()

    xml = soap.metadata(dataset, node, sceneids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:metadataResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    if extended:
        metadata_urls = map(_get_metadata_url, data)
        results = _async_requests(metadata_urls)
        data = map(lambda idx: _get_extended(data[idx], results[idx]), range(len(data)))

    return data

def remove_bulk_download_scene():
    raise NotImplementedError

def remove_order_scene():
    raise NotImplementedError

def search(dataset, node, lat=None, lng=None, distance=100, ll=None, ur=None, start_date=None, end_date=None, where=None, max_results=50000, starting_number=1, sort_order="DESC", extended=False, api_key=None):
    """

    :param dataset:
        USGS dataset (e.g. EO1_HYP_PUB, LANDSAT_8)
    :param node:
        USGS node representing a dataset catalog (e.g. CWIC, EE, HDDS, LPVS)
    :param lat:
        Latitude
    :param lng:
        Longitude
    :param distance:
        Distance in meters used to for a radial search
    :param ll:
        Dictionary of longitude/latitude coordinates for the lower left corner
        of a bounding box search. e.g. { "longitude": 0.0, "latitude": 0.0 }
    :param ur:
        Dictionary of longitude/latitude coordinates for the upper right corner
        of a bounding box search. e.g. { "longitude": 0.0, "latitude": 0.0 }
    :param start_date:
        Start date for when a scene has been acquired
    :param end_date:
        End date for when a scene has been acquired
    :where:
        Dictionary representing key/values for finer grained conditional
        queries. Only a subset of metadata fields are supported. Available
        fields depend on the value of `dataset`, and maybe be found by
        submitting a dataset_fields query.
    :max_results:
        Maximum results returned by the server
    :starting_number:
        Starting offset for results of a query.
    :sort_order:
        Order in which results are sorted. Ascending or descending w.r.t the acquisition date.
    :extended:
        Boolean flag. When true a subsequent query will be sent to the `metadataUrl` returned by
        the first query.
    :api_key:
        API key for EROS. Not required for searching.

    .. todo:: Export metadata from the search results e.g.

        <numberReturned xsi:type="xsd:int">41</numberReturned>
        <totalHits xsi:type="xsd:int">41</totalHits>
        <firstRecord xsi:type="xsd:int">1</firstRecord>
        <lastRecord xsi:type="xsd:int">41</lastRecord>
        <nextRecord xsi:type="xsd:int">41</nextRecord>
    """
    api_key = _get_api_key()

    xml = soap.search(dataset, node, lat=lat, lng=lng, distance=100, ll=ll, ur=ur, start_date=start_date, end_date=end_date, where=where, max_results=max_results, starting_number=starting_number, sort_order=sort_order, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:searchResponse/return/results/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    if extended:
        metadata_urls = map(_get_metadata_url, data)
        results = _async_requests(metadata_urls)
        data = map(lambda idx: _get_extended(data[idx], results[idx]), range(len(data)))

    return data

def submit_bulk_order():
    raise NotImplementedError

def submit_order():
    raise NotImplementedError

def update_bulk_download_scene():
    raise NotImplementedError

def update_order_scene():
    raise NotImplementedError
    <commit_msg>print error response from usgs<commit_after>
import os
import json
from xml.etree import ElementTree
import requests
from requests_futures.sessions import FuturesSession

from usgs import USGS_API, USGSError
from usgs import soap, xsi

TMPFILE = os.path.join("/", "tmp", "usgs")
NAMESPACES = {
    "SOAP-ENV": "http://schemas.xmlsoap.org/soap/envelope/",
    "ns1": "https://earthexplorer.usgs.gov/inventory/soap",
    "eemetadata": "http://earthexplorer.usgs.gov/eemetadata.xsd"
}

def _get_api_key():

    api_key = None

    if os.path.exists(TMPFILE):
        with open(TMPFILE, "r") as f:
            api_key = f.read()

    return api_key

def _check_for_usgs_error(root):

    fault_code_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultcode", NAMESPACES)

    if fault_code_el is None:
        return

    fault_string_el = root.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", NAMESPACES)

    fault_code = fault_code_el.text
    fault_string = fault_string_el.text

    raise USGSError('%s: %s' % (fault_code, fault_string))

def _get_extended(scene, resp):
    """
    Parse metadata returned from the metadataUrl of a USGS scene.

    :param scene:
        Dictionary representation of a USGS scene
    :param resp:
        Response object from requests/grequests
    """
    root = ElementTree.fromstring(resp.text)
    items = root.findall("eemetadata:metadataFields/eemetadata:metadataField", NAMESPACES)
    scene['extended'] = { item.attrib.get('name').strip(): xsi.get(item[0]) for item in items }

    return scene

def _async_requests(urls):
    """
    Sends multiple non-blocking requests. Returns
    a list of responses.

    :param urls:
        List of urls
    """

    session = FuturesSession(max_workers=30)
    futures = [ session.get(url) for url in urls ]
    return [ future.result() for future in futures ]

def _get_metadata_url(scene):
    return scene.get('metadataUrl')

def clear_bulk_download_order():
    raise NotImplementedError

def clear_order():
    raise NotImplementedError

def datasets(dataset, node, ll=None, ur=None, start_date=None, end_date=None):

    api_key = _get_api_key()

    xml = soap.datasets(dataset, node, ll=ll, ur=ur, start_date=start_date, end_date=end_date, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetsResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def dataset_fields(dataset, node):

    api_key = _get_api_key()

    xml = soap.dataset_fields(dataset, node, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:datasetFieldsResponse/return/item", NAMESPACES)
    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def download(dataset, node, entityids, product):
    """
    Though USGS supports multiple products in a single request, there's
    ambiguity in the returned list. This wrapper only allows a single
    product per request.

    Additionally, the response has no indiction which URL is associated
    with which scene/entity id. The URL can be parsed, but the structure
    varies depending on the product.
    """

    api_key = _get_api_key()

    xml = soap.download(dataset, node, entityids, [product], api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadResponse/return/item", NAMESPACES)

    data = map(lambda el: xsi.get(el), items)

    return data

def download_options(dataset, node, entityids):

    api_key = _get_api_key()

    xml = soap.download_options(dataset, node, entityids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:downloadOptionsResponse/return/item/downloadOptions/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    return data

def get_bulk_download_products():
    raise NotImplementedError

def get_order_products():
    raise NotImplementedError

def hits():
    raise NotImplementedError

def item_basket():
    raise NotImplementedError

def login(username, password):
    xml = soap.login(username, password)
    r = requests.post(USGS_API, xml)

    if r.status_code is not 200:
        raise USGSError(r.text)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    element = root.find("SOAP-ENV:Body/ns1:loginResponse/return", NAMESPACES)

    api_key = element.text

    with open(TMPFILE, "w") as f:
        f.write(api_key)

    return api_key

def logout():

    api_key = _get_api_key()

    xml = soap.logout(api_key=api_key)
    r = requests.post(USGS_API, xml)

    if os.path.exists(TMPFILE):
        os.remove(TMPFILE)

    return True

def metadata(dataset, node, sceneids, extended=False, api_key=None):
    """
    Request metadata for a given scene in a USGS dataset.

    :param dataset:
    :param node:
    :param sceneids:
    :param extended:
        Send a second request to the metadata url to get extended metadata on the scene.
    :param api_key:
    """
    api_key = _get_api_key()

    xml = soap.metadata(dataset, node, sceneids, api_key=api_key)
    r = requests.post(USGS_API, xml)

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:metadataResponse/return/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    if extended:
        metadata_urls = map(_get_metadata_url, data)
        results = _async_requests(metadata_urls)
        data = map(lambda idx: _get_extended(data[idx], results[idx]), range(len(data)))

    return data

def remove_bulk_download_scene():
    raise NotImplementedError

def remove_order_scene():
    raise NotImplementedError

def search(dataset, node, lat=None, lng=None, distance=100, ll=None, ur=None, start_date=None, end_date=None, where=None, max_results=50000, starting_number=1, sort_order="DESC", extended=False, api_key=None):
    """

    :param dataset:
        USGS dataset (e.g. EO1_HYP_PUB, LANDSAT_8)
    :param node:
        USGS node representing a dataset catalog (e.g. CWIC, EE, HDDS, LPVS)
    :param lat:
        Latitude
    :param lng:
        Longitude
    :param distance:
        Distance in meters used to for a radial search
    :param ll:
        Dictionary of longitude/latitude coordinates for the lower left corner
        of a bounding box search. e.g. { "longitude": 0.0, "latitude": 0.0 }
    :param ur:
        Dictionary of longitude/latitude coordinates for the upper right corner
        of a bounding box search. e.g. { "longitude": 0.0, "latitude": 0.0 }
    :param start_date:
        Start date for when a scene has been acquired
    :param end_date:
        End date for when a scene has been acquired
    :where:
        Dictionary representing key/values for finer grained conditional
        queries. Only a subset of metadata fields are supported. Available
        fields depend on the value of `dataset`, and maybe be found by
        submitting a dataset_fields query.
    :max_results:
        Maximum results returned by the server
    :starting_number:
        Starting offset for results of a query.
    :sort_order:
        Order in which results are sorted. Ascending or descending w.r.t the acquisition date.
    :extended:
        Boolean flag. When true a subsequent query will be sent to the `metadataUrl` returned by
        the first query.
    :api_key:
        API key for EROS. Not required for searching.

    .. todo:: Export metadata from the search results e.g.

        <numberReturned xsi:type="xsd:int">41</numberReturned>
        <totalHits xsi:type="xsd:int">41</totalHits>
        <firstRecord xsi:type="xsd:int">1</firstRecord>
        <lastRecord xsi:type="xsd:int">41</lastRecord>
        <nextRecord xsi:type="xsd:int">41</nextRecord>
    """
    api_key = _get_api_key()

    xml = soap.search(dataset, node, lat=lat, lng=lng, distance=100, ll=ll, ur=ur, start_date=start_date, end_date=end_date, where=where, max_results=max_results, starting_number=starting_number, sort_order=sort_order, api_key=api_key)
    r = requests.post(USGS_API, xml)

    # Find out what's going on with usgs servers!
    try:
        r.raise_for_status()
    except:
        print r.text

    root = ElementTree.fromstring(r.text)
    _check_for_usgs_error(root)

    items = root.findall("SOAP-ENV:Body/ns1:searchResponse/return/results/item", NAMESPACES)

    data = map(lambda item: { el.tag: xsi.get(el) for el in item }, items)

    if extended:
        metadata_urls = map(_get_metadata_url, data)
        results = _async_requests(metadata_urls)
        data = map(lambda idx: _get_extended(data[idx], results[idx]), range(len(data)))

    return data

def submit_bulk_order():
    raise NotImplementedError

def submit_order():
    raise NotImplementedError

def update_bulk_download_scene():
    raise NotImplementedError

def update_order_scene():
    raise NotImplementedError
    <|endoftext|>