<commit_before>"""
Copyright (c) 2013, Dave Mankoff
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Dave Mankoff nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL DAVE MANKOFF BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from __future__ import unicode_literals
try:
  from html import escape
except ImportError:
  from cgi import escape

import re
try:
  from html.parser import HTMLParser
except ImportError:
  from HTMLParser import HTMLParser

PRE_TAGS = ('pre', 'textarea')  # styles and scripts are never minified
# http://www.w3.org/TR/html51/syntax.html#elements-0
NO_CLOSE_TAGS = ('area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img',
                 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track',
                 'wbr')
# http://www.w3.org/TR/html51/index.html#attributes-1
BOOLEAN_ATTRIBUTES = {
  'audio': ('autoplay', 'controls', 'loop', 'muted',),
  'button': ('autofocus', 'disabled', 'formnovalidate',),
  'command': ('checked', 'disabled',),
  'dialog': ('open',),
  'fieldset': ('disabled',),
  'form': ('novalidate',),
  'iframe': ('seamless',),
  'img': ('ismap',),
  'input': ('autofocus', 'checked', 'disabled', 'formnovalidate', 'multiple',
            'readonly', 'required',),
  'keygen': ('autofocus', 'disabled',),
  'object': ('typesmustmatch',),
  'ol': ('reversed',),
  'optgroup': ('disabled',),
  'option': ('disabled', 'selected',),
  'script': ('async', 'defer'),
  'select': ('autofocus', 'disabled', 'multiple', 'required',),
  'style': ('scoped',),
  'textarea': ('autofocus', 'disabled', 'readonly', 'required',),
  'track': ('default',),
  'video': ('autoplay', 'controls', 'loop', 'muted',),
  '*': ('hidden',),
}

leading_whitespace_re = re.compile(r'^\s+')
trailing_whitespace_re = re.compile(r'\s+$')
leading_trailing_whitespace_re = re.compile(r'(^\s+)|(\s+$)')
whitespace_re = re.compile(r'\s+')
whitespace_newline_re = re.compile(r'\s*(\r|\n)+\s*')

# Tag omission rules:
# http://www.w3.org/TR/html51/syntax.html#optional-tags

class HTMLMinError(Exception): pass
class ParseError(HTMLMinError): pass
class OpenTagNotFoundError(ParseError): pass

class HTMLMinParser(HTMLParser):
  def __init__(self,
               remove_comments=False,
               remove_empty_space=False,
               remove_all_empty_space=False,
               reduce_empty_attributes=True,
               reduce_boolean_attributes=False,
               remove_optional_attribute_quotes=True,
               keep_pre=False,
               pre_tags=PRE_TAGS,
               pre_attr='pre'):
    HTMLParser.__init__(self)
    self.keep_pre = keep_pre
    self.pre_tags = pre_tags
    self.remove_comments = remove_comments
    self.remove_empty_space = remove_empty_space
    self.remove_all_empty_space = remove_all_empty_space
    self.reduce_empty_attributes = reduce_empty_attributes
    self.reduce_boolean_attributes = reduce_boolean_attributes
    self.remove_optional_attribute_quotes = remove_optional_attribute_quotes
    self.pre_attr = pre_attr
    self._data_buffer = []
    self._in_pre_tag = 0
    self._in_head = False
    self._in_title = False
    self._after_doctype = False
    self._tag_stack = []
    self._title_newly_opened = False
    self.__title_trailing_whitespace = False

  def _has_pre(self, attrs):
    for k,v in attrs:
      if k == self.pre_attr:
        return True
    return False

  def build_tag(self, tag, attrs, close_tag):
    result = '<{}'.format(escape(tag))
    for k,v in attrs:
      result += ' ' + escape(k)
      if v:
        if self.reduce_boolean_attributes and (
             k in BOOLEAN_ATTRIBUTES.get(tag,[]) or
             k in BOOLEAN_ATTRIBUTES['*']):
          pass
        elif (self.remove_optional_attribute_quotes
              and not any((c in v for c in ('"', "'", ' ', '<', '>', '=')))
              and not v.endswith('/')):
          result += '={}'.format(escape(v, quote=True))
        else:
          result += '="{}"'.format(escape(v, quote=True).replace(''', "'"))
      elif not self.reduce_empty_attributes:
        result += '=""'
    if close_tag:
      return result + '/>'
    return result + '>'

  def handle_decl(self, decl):
    if (len(self._data_buffer) == 1 and
        whitespace_re.match(self._data_buffer[0])):
      self._data_buffer = []
    self._data_buffer.append('<!' + decl + '>\n')
    self._after_doctype = True

  def in_tag(self, *tags):
    for t in self._tag_stack:
      if t[0] in tags:
        return t
    return False

  def _close_tags_up_to(self, tag):
    num_pres = 0
    i = 0
    for i, t in enumerate(self._tag_stack):
      if t[1]:
        num_pres += 1
      if t[0] == tag:
        break

      # Only the html tag can close out everything. Put on the brakes if
      # we encounter a closing tag that we didn't recognize.
      if tag != 'html' and t[0] in ('body', 'html', 'head'):
        raise OpenTagNotFoundError()

    self._tag_stack = self._tag_stack[i+1:]

    return num_pres

  def handle_starttag(self, tag, attrs):
    self._after_doctype = False
    if tag == 'head':
      self._in_head = True
    elif self._in_head and tag == 'title':
      self._in_title = True
      self._title_newly_opened = True

    tag_sets = ( # a list of tags and tags that they are closed by
      (('li',), ('li',)),
      (('dd', 'dt',), ('dd', 'dt',)),
      (('rp', 'rt',), ('rp', 'rt',)),
      (('p',), ('address', 'article', 'aside', 'blockquote', 'dir', 'div',
                'dl', 'fieldset', 'footer', 'form', 'h1', 'h2', 'h3', 'h4',
                'h5', 'h6', 'header', 'hgroup', 'hr', 'menu', 'nav', 'ol', 'p',
                'pre', 'section', 'table', 'ul')),
      (('optgroup',), ('optgroup',)),
      (('option',), ('option', 'optgroup')),
      (('colgroup',), ('*',)),
      (('tbody', 'thead',), ('tbody', 'tfoot')),
      (('tfoot',), ('tbody',)),
      (('tr',), ('tr',)),
      (('td', 'th'), ('td', 'th')),
      )
    for open_tags, closed_by_tags in tag_sets:
      in_tag = self.in_tag(*open_tags)
      if in_tag and (tag in closed_by_tags or '*' in closed_by_tags):
        self._in_pre_tag -= self._close_tags_up_to(in_tag[0])

    start_pre = False
    if (tag in self.pre_tags or
        tag in ('script', 'style') or
        self._has_pre(attrs) or
        self._in_pre_tag > 0):
      self._in_pre_tag += 1
      start_pre = True

    self._tag_stack.insert(0, (tag, start_pre))

    if not self.keep_pre:
      attrs = [(k,v) for k,v in attrs if k != self.pre_attr]

    self._data_buffer.append(self.build_tag(tag, attrs, False))

  def handle_endtag(self, tag):
    # According to the spec,
 tags don't get closed when a parent a
    # tag closes them. Here's some logic that addresses this.
    if tag == 'a':
      contains_p = False
      for i, t in enumerate(self._tag_stack):
        if t[0] == 'p':
          contains_p = True
        elif t[0] == 'a':
          break
      if contains_p: # the p tag, and all its children should be left open
        a_tag = self._tag_stack.pop(i)
        if a_tag[1]:
          self._in_pre_tag -= 1
    else:
      if tag == 'head':
        # TODO: Did we know that we were in an head tag?! If not, we need to
        # reminify everything to remove extra spaces.
        self._in_head = False
      elif tag == 'title':
        self._in_title = False
        self._title_newly_opened = False
      try:
        self._in_pre_tag -= self._close_tags_up_to(tag)
      except OpenTagNotFoundError:
        # Some tags don't require a start tag. Most do. Either way, we leave
        # closing tags along since they affect output. For instance, a '
'
        # results in a '

' in Chrome.
        pass
    if tag not in NO_CLOSE_TAGS:
      self._data_buffer.append('</{}>'.format(escape(tag)))

  def handle_startendtag(self, tag, attrs):
    self._after_doctype = False
    if not self.keep_pre:
      attrs = [(k,v) for k,v in attrs if k != 'pre']
    self._data_buffer.append(self.build_tag(tag, attrs, tag not in NO_CLOSE_TAGS))

  def handle_comment(self, data):
    if not self.remove_comments or data[0] == '!':
      self._data_buffer.append(''.format(
          data[1:] if data[0] == '!' else data))

  def handle_data(self, data):
    if self._in_pre_tag > 0:
      self._data_buffer.append(data)
    else:
      # remove_all_empty_space matches everything. remove_empty_space only
      # matches if there's a newline involved.
      if self.remove_all_empty_space or self._in_head or self._after_doctype:
        match = whitespace_re.match(data)
        if match and match.end(0) == len(data):
          return
      elif self.remove_empty_space:
        match = whitespace_newline_re.match(data)
        if match and match.end(0) == len(data):
          return

      # if we're in the title, remove leading and trailing whitespace.
      # note that the title may be parsed in chunks if entityref's or charrefs
      # are encountered.
      if self._in_title:
        if self.__title_trailing_whitespace:
          self._data_buffer.append(' ')
        self.__title_trailing_whitespace = data[-1].isspace()
        if self._title_newly_opened:
          self._title_newly_opened = False
          data = leading_trailing_whitespace_re.sub('', data)
        else:
          data = trailing_whitespace_re.sub(
            '', leading_whitespace_re.sub(' ', data))

      data = whitespace_re.sub(' ', data)
      if not data:
        return

      if self._in_pre_tag == 0 and self._data_buffer:
        # If we're not in a pre block, its possible that we append two spaces
        # together, which we want to avoid. For instance, if we remove a comment
        # from between two blocks of text: a  c => a  c.
        if data[0] == ' ' and self._data_buffer[-1][-1] == ' ':
          data = data[1:]
          if not data:
            return
      self._data_buffer.append(data)

  def handle_entityref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&{};'.format(data))

  def handle_charref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&#{};'.format(data))

  def handle_pi(self, data):
    self._data_buffer.append('<?' + data + '>')

  def unknown_decl(self, data):
    self._data_buffer.append('<![' + data + ']>')

  def reset(self):
    self._data_buffer = []
    HTMLParser.reset(self)

  @property
  def result(self):
    return ''.join(self._data_buffer)
<commit_msg>Make sure tags don't get closed accidentally.<commit_after>"""
Copyright (c) 2013, Dave Mankoff
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Dave Mankoff nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL DAVE MANKOFF BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from __future__ import unicode_literals
try:
  from html import escape
except ImportError:
  from cgi import escape

import re
try:
  from html.parser import HTMLParser
except ImportError:
  from HTMLParser import HTMLParser

PRE_TAGS = ('pre', 'textarea')  # styles and scripts are never minified
# http://www.w3.org/TR/html51/syntax.html#elements-0
NO_CLOSE_TAGS = ('area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img',
                 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track',
                 'wbr')
# http://www.w3.org/TR/html51/index.html#attributes-1
BOOLEAN_ATTRIBUTES = {
  'audio': ('autoplay', 'controls', 'loop', 'muted',),
  'button': ('autofocus', 'disabled', 'formnovalidate',),
  'command': ('checked', 'disabled',),
  'dialog': ('open',),
  'fieldset': ('disabled',),
  'form': ('novalidate',),
  'iframe': ('seamless',),
  'img': ('ismap',),
  'input': ('autofocus', 'checked', 'disabled', 'formnovalidate', 'multiple',
            'readonly', 'required',),
  'keygen': ('autofocus', 'disabled',),
  'object': ('typesmustmatch',),
  'ol': ('reversed',),
  'optgroup': ('disabled',),
  'option': ('disabled', 'selected',),
  'script': ('async', 'defer'),
  'select': ('autofocus', 'disabled', 'multiple', 'required',),
  'style': ('scoped',),
  'textarea': ('autofocus', 'disabled', 'readonly', 'required',),
  'track': ('default',),
  'video': ('autoplay', 'controls', 'loop', 'muted',),
  '*': ('hidden',),
}

leading_whitespace_re = re.compile(r'^\s+')
trailing_whitespace_re = re.compile(r'\s+$')
leading_trailing_whitespace_re = re.compile(r'(^\s+)|(\s+$)')
whitespace_re = re.compile(r'\s+')
whitespace_newline_re = re.compile(r'\s*(\r|\n)+\s*')

# Tag omission rules:
# http://www.w3.org/TR/html51/syntax.html#optional-tags

class HTMLMinError(Exception): pass
class ParseError(HTMLMinError): pass
class OpenTagNotFoundError(ParseError): pass

class HTMLMinParser(HTMLParser):
  def __init__(self,
               remove_comments=False,
               remove_empty_space=False,
               remove_all_empty_space=False,
               reduce_empty_attributes=True,
               reduce_boolean_attributes=False,
               remove_optional_attribute_quotes=True,
               keep_pre=False,
               pre_tags=PRE_TAGS,
               pre_attr='pre'):
    HTMLParser.__init__(self)
    self.keep_pre = keep_pre
    self.pre_tags = pre_tags
    self.remove_comments = remove_comments
    self.remove_empty_space = remove_empty_space
    self.remove_all_empty_space = remove_all_empty_space
    self.reduce_empty_attributes = reduce_empty_attributes
    self.reduce_boolean_attributes = reduce_boolean_attributes
    self.remove_optional_attribute_quotes = remove_optional_attribute_quotes
    self.pre_attr = pre_attr
    self._data_buffer = []
    self._in_pre_tag = 0
    self._in_head = False
    self._in_title = False
    self._after_doctype = False
    self._tag_stack = []
    self._title_newly_opened = False
    self.__title_trailing_whitespace = False

  def _has_pre(self, attrs):
    for k,v in attrs:
      if k == self.pre_attr:
        return True
    return False

  def build_tag(self, tag, attrs, close_tag):
    result = '<{}'.format(escape(tag))
    needs_closing_space = False
    for k,v in attrs:
      result += ' ' + escape(k)
      if v:
        if self.reduce_boolean_attributes and (
             k in BOOLEAN_ATTRIBUTES.get(tag,[]) or
             k in BOOLEAN_ATTRIBUTES['*']):
          pass
        elif self.remove_optional_attribute_quotes and not any((c in v for c in ('"', "'", ' ', '<', '>', '='))):
          result += '={}'.format(escape(v, quote=True))
          needs_closing_space = v.endswith('/')
        else:
          result += '="{}"'.format(escape(v, quote=True).replace(''', "'"))
      elif not self.reduce_empty_attributes:
        result += '=""'
    if needs_closing_space:
      result += ' '
    if close_tag:
      return result + '/>'
    return result + '>'

  def handle_decl(self, decl):
    if (len(self._data_buffer) == 1 and
        whitespace_re.match(self._data_buffer[0])):
      self._data_buffer = []
    self._data_buffer.append('<!' + decl + '>\n')
    self._after_doctype = True

  def in_tag(self, *tags):
    for t in self._tag_stack:
      if t[0] in tags:
        return t
    return False

  def _close_tags_up_to(self, tag):
    num_pres = 0
    i = 0
    for i, t in enumerate(self._tag_stack):
      if t[1]:
        num_pres += 1
      if t[0] == tag:
        break

      # Only the html tag can close out everything. Put on the brakes if
      # we encounter a closing tag that we didn't recognize.
      if tag != 'html' and t[0] in ('body', 'html', 'head'):
        raise OpenTagNotFoundError()

    self._tag_stack = self._tag_stack[i+1:]

    return num_pres

  def handle_starttag(self, tag, attrs):
    self._after_doctype = False
    if tag == 'head':
      self._in_head = True
    elif self._in_head and tag == 'title':
      self._in_title = True
      self._title_newly_opened = True

    tag_sets = ( # a list of tags and tags that they are closed by
      (('li',), ('li',)),
      (('dd', 'dt',), ('dd', 'dt',)),
      (('rp', 'rt',), ('rp', 'rt',)),
      (('p',), ('address', 'article', 'aside', 'blockquote', 'dir', 'div',
                'dl', 'fieldset', 'footer', 'form', 'h1', 'h2', 'h3', 'h4',
                'h5', 'h6', 'header', 'hgroup', 'hr', 'menu', 'nav', 'ol', 'p',
                'pre', 'section', 'table', 'ul')),
      (('optgroup',), ('optgroup',)),
      (('option',), ('option', 'optgroup')),
      (('colgroup',), ('*',)),
      (('tbody', 'thead',), ('tbody', 'tfoot')),
      (('tfoot',), ('tbody',)),
      (('tr',), ('tr',)),
      (('td', 'th'), ('td', 'th')),
      )
    for open_tags, closed_by_tags in tag_sets:
      in_tag = self.in_tag(*open_tags)
      if in_tag and (tag in closed_by_tags or '*' in closed_by_tags):
        self._in_pre_tag -= self._close_tags_up_to(in_tag[0])

    start_pre = False
    if (tag in self.pre_tags or
        tag in ('script', 'style') or
        self._has_pre(attrs) or
        self._in_pre_tag > 0):
      self._in_pre_tag += 1
      start_pre = True

    self._tag_stack.insert(0, (tag, start_pre))

    if not self.keep_pre:
      attrs = [(k,v) for k,v in attrs if k != self.pre_attr]

    self._data_buffer.append(self.build_tag(tag, attrs, False))

  def handle_endtag(self, tag):
    # According to the spec,
 tags don't get closed when a parent a
    # tag closes them. Here's some logic that addresses this.
    if tag == 'a':
      contains_p = False
      for i, t in enumerate(self._tag_stack):
        if t[0] == 'p':
          contains_p = True
        elif t[0] == 'a':
          break
      if contains_p: # the p tag, and all its children should be left open
        a_tag = self._tag_stack.pop(i)
        if a_tag[1]:
          self._in_pre_tag -= 1
    else:
      if tag == 'head':
        # TODO: Did we know that we were in an head tag?! If not, we need to
        # reminify everything to remove extra spaces.
        self._in_head = False
      elif tag == 'title':
        self._in_title = False
        self._title_newly_opened = False
      try:
        self._in_pre_tag -= self._close_tags_up_to(tag)
      except OpenTagNotFoundError:
        # Some tags don't require a start tag. Most do. Either way, we leave
        # closing tags along since they affect output. For instance, a '
'
        # results in a '

' in Chrome.
        pass
    if tag not in NO_CLOSE_TAGS:
      self._data_buffer.append('</{}>'.format(escape(tag)))

  def handle_startendtag(self, tag, attrs):
    self._after_doctype = False
    if not self.keep_pre:
      attrs = [(k,v) for k,v in attrs if k != 'pre']
    self._data_buffer.append(self.build_tag(tag, attrs, tag not in NO_CLOSE_TAGS))

  def handle_comment(self, data):
    if not self.remove_comments or data[0] == '!':
      self._data_buffer.append(''.format(
          data[1:] if data[0] == '!' else data))

  def handle_data(self, data):
    if self._in_pre_tag > 0:
      self._data_buffer.append(data)
    else:
      # remove_all_empty_space matches everything. remove_empty_space only
      # matches if there's a newline involved.
      if self.remove_all_empty_space or self._in_head or self._after_doctype:
        match = whitespace_re.match(data)
        if match and match.end(0) == len(data):
          return
      elif self.remove_empty_space:
        match = whitespace_newline_re.match(data)
        if match and match.end(0) == len(data):
          return

      # if we're in the title, remove leading and trailing whitespace.
      # note that the title may be parsed in chunks if entityref's or charrefs
      # are encountered.
      if self._in_title:
        if self.__title_trailing_whitespace:
          self._data_buffer.append(' ')
        self.__title_trailing_whitespace = data[-1].isspace()
        if self._title_newly_opened:
          self._title_newly_opened = False
          data = leading_trailing_whitespace_re.sub('', data)
        else:
          data = trailing_whitespace_re.sub(
            '', leading_whitespace_re.sub(' ', data))

      data = whitespace_re.sub(' ', data)
      if not data:
        return

      if self._in_pre_tag == 0 and self._data_buffer:
        # If we're not in a pre block, its possible that we append two spaces
        # together, which we want to avoid. For instance, if we remove a comment
        # from between two blocks of text: a  c => a  c.
        if data[0] == ' ' and self._data_buffer[-1][-1] == ' ':
          data = data[1:]
          if not data:
            return
      self._data_buffer.append(data)

  def handle_entityref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&{};'.format(data))

  def handle_charref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&#{};'.format(data))

  def handle_pi(self, data):
    self._data_buffer.append('<?' + data + '>')

  def unknown_decl(self, data):
    self._data_buffer.append('<![' + data + ']>')

  def reset(self):
    self._data_buffer = []
    HTMLParser.reset(self)

  @property
  def result(self):
    return ''.join(self._data_buffer)
<|endoftext|>


<commit_before>from collections import Counter

import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle, Circle, ConnectionPatch
import numpy as np
from scipy.spatial import Delaunay

def delaunay_figure(box_r, convergence_bins, output_path, triang=None, children=[], parents=[],
                    bins=[], new_bins=[], title="", patches=None, prop1range=(0.0,1.0),
                    prop2range=(0.0,1.0),perturbation_methods=None, show_grid=True,
                    show_triangulation=True, show_hull=True, bin_saturated=10, bin_scores=None,
                    addl_data_set=None):

    # plot visualization
    fig = plt.figure(figsize=(12,12), tight_layout=True)
    ax = fig.add_subplot(1, 1, 1)
    ax.set_xlim(prop1range[0], prop1range[1])
    ax.set_ylim(prop2range[0], prop2range[1])
    ax.set_xlabel("Void Fraction")
    ax.set_ylabel("Methane Loading (V/V)")

    ax.set_xticks(prop1range[1] * np.array([0.0, 0.25, 0.5, 0.75, 1.0]))
    ax.set_yticks(prop2range[1] * np.array([0.0, 0.25, 0.5, 0.75, 1.0]))
    ax.set_xticks(prop1range[1] * np.array(range(0,convergence_bins + 1))/convergence_bins, minor=True)
    ax.set_yticks(prop2range[1] * np.array(range(0,convergence_bins + 1))/convergence_bins, minor=True)

    if show_grid:
        ax.grid(linestyle='-', color='0.8', zorder=0)

    dbinx = prop1range[1] / convergence_bins
    dbiny = prop2range[1] / convergence_bins

    total_materials = bins.sum()
    bin_rects = []
    for b, bcount in np.ndenumerate(bins):
        if bcount > 0.0:
            bin_rects.append(Rectangle((b[0] * dbinx, b[1] * dbiny), dbinx, dbiny, \
                        facecolor=str(max(1-bcount/bin_saturated, 0.0)), edgecolor='0.8'))
    pc = PatchCollection(bin_rects, match_original=True)
    ax.add_collection(pc)

    new_bin_rects = [Rectangle((b[0] * dbinx, b[1] * dbiny), dbinx, dbiny) for b in new_bins]
    pc2 = PatchCollection(new_bin_rects, facecolor='#82b7b7')
    ax.add_collection(pc2)

    if bin_scores is not None:
        for (i,j), score in bin_scores:
            ax.annotate(str(int(score)), ((i + 0.5) * dbinx, (j+0.5) * dbiny), zorder=80, ha="center", va="center", size=9)

    if not triang and (show_hull or show_triangulation):
        triang = Delaunay(box_r)

    # plot all points as triangulation
    if show_triangulation:
        ax.triplot(box_r[:,0], box_r[:,1], triang.simplices.copy(), color='#78a7cc', lw=1)

    # plot hull and labels
    if show_hull:
        hull_point_indices = np.unique(triang.convex_hull.flatten())
        hull_points = np.array([box_r[p] for p in hull_point_indices])
        ax.plot(hull_points[:,0], hull_points[:,1], color='#78a7cc', marker='o', linestyle='None', zorder=10)

    # plot children
    if len(children) > 0:
        ax.plot(children[:,0], children[:,1], color='#81ff6b', marker='o', linestyle='None', zorder=12)

    # plot chosen parents with proportional circles and label
    if len(parents) > 0:
        parent_counter = Counter([tuple(x) for x in parents]) #need tuples because they are hashable
        unique_parents = np.array([[x[0], x[1], num] for x, num in parent_counter.items()])
        ax.scatter(unique_parents[:,0], unique_parents[:,1], s=40*unique_parents[:,2], color='#ffbe6b', marker='o', linestyle='None', zorder=15)
        for p in unique_parents:
            x, y, parent_count = p
            if parent_count > 5:
                ax.annotate(str(int(parent_count)), (x, y), zorder=30, ha="center", va="center", size=9)

    # plot additional data set
    if addl_data_set is not None:
        ax.plot(addl_data_set[:,0], addl_data_set[:,1], color='#FF8C00', marker=',', linestyle='None', zorder=20)

    if patches == "donut":
        ax.add_patch(Circle((0.5, 0.5), 0.125, fill=False, linestyle="dashed", linewidth=1, zorder=50))
        ax.add_patch(Circle((0.5, 0.5), 0.375, fill=False, linestyle="dashed", linewidth=2, zorder=50))

    if perturbation_methods and len(parents) > 0:
        p_cmap = {"all": "black", "lattice": "w", "lattice_nodens": "#848C8E", "density": "#435058", "atom_types": "#0FA3B1", "atom_sites": "#F77936", }
        arrows = [ConnectionPatch(parents[i], children[i], "data", arrowstyle="-|>", \
                    shrinkA=5, shrinkB=5, mutation_scale=20, fc=p_cmap[perturbation_methods[i]], \
                    linestyle="--", zorder=40) \
                    for i in range(len(children))]
        for a in arrows:
            ax.add_patch(a)

    ax.set_title(title)
    fig.savefig(output_path)
    plt.close(fig)
<commit_msg>increase font size on bin plot so it will show at a reasonable size during presentations<commit_after>from collections import Counter

import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Rectangle, Circle, ConnectionPatch
from matplotlib import rc
import numpy as np
from scipy.spatial import Delaunay

font = {'family':'sans-serif',
        'sans-serif':['Helvetica'],
        'weight' : 'normal',
        'size'   : 12}

rc('font', **font)

def delaunay_figure(box_r, convergence_bins, output_path, triang=None, children=[], parents=[],
                    bins=[], new_bins=[], title="", patches=None, prop1range=(0.0,1.0),
                    prop2range=(0.0,1.0),perturbation_methods=None, show_grid=True,
                    show_triangulation=True, show_hull=True, bin_saturated=10, bin_scores=None,
                    addl_data_set=None):

    # plot visualization
    fig = plt.figure(figsize=(4, 4), tight_layout=True)
    ax = fig.add_subplot(1, 1, 1)
    ax.set_xlim(prop1range[0], prop1range[1])
    ax.set_ylim(prop2range[0], prop2range[1])
    ax.set_xlabel("Void Fraction")
    ax.set_ylabel("Methane Loading (V/V)")

    ax.set_xticks(prop1range[1] * np.array([0.0, 0.25, 0.5, 0.75, 1.0]))
    ax.set_yticks(prop2range[1] * np.array([0.0, 0.25, 0.5, 0.75, 1.0]))
    ax.set_xticks(prop1range[1] * np.array(range(0,convergence_bins + 1))/convergence_bins, minor=True)
    ax.set_yticks(prop2range[1] * np.array(range(0,convergence_bins + 1))/convergence_bins, minor=True)

    if show_grid:
        ax.grid(linestyle='-', color='0.8', zorder=0)

    dbinx = prop1range[1] / convergence_bins
    dbiny = prop2range[1] / convergence_bins

    total_materials = bins.sum()
    bin_rects = []
    for b, bcount in np.ndenumerate(bins):
        if bcount > 0.0:
            bin_rects.append(Rectangle((b[0] * dbinx, b[1] * dbiny), dbinx, dbiny, \
                        facecolor=str(max(1-bcount/bin_saturated, 0.0)), edgecolor='0.8', lw=0.5))
    pc = PatchCollection(bin_rects, match_original=True)
    ax.add_collection(pc)

    new_bin_rects = [Rectangle((b[0] * dbinx, b[1] * dbiny), dbinx, dbiny) for b in new_bins]
    pc2 = PatchCollection(new_bin_rects, facecolor='#82b7b7')
    ax.add_collection(pc2)

    if bin_scores is not None:
        for (i,j), score in bin_scores:
            ax.annotate(str(int(score)), ((i + 0.5) * dbinx, (j+0.5) * dbiny), zorder=80, ha="center", va="center", size=9)

    if not triang and (show_hull or show_triangulation):
        triang = Delaunay(box_r)

    # plot all points as triangulation
    if show_triangulation:
        ax.triplot(box_r[:,0], box_r[:,1], triang.simplices.copy(), color='#78a7cc', lw=1)

    # plot hull and labels
    if show_hull:
        hull_point_indices = np.unique(triang.convex_hull.flatten())
        hull_points = np.array([box_r[p] for p in hull_point_indices])
        ax.plot(hull_points[:,0], hull_points[:,1], color='#78a7cc', marker='o', linestyle='None', zorder=10)

    # plot children
    if len(children) > 0:
        ax.plot(children[:,0], children[:,1], color='#81ff6b', marker='o', linestyle='None', zorder=12)

    # plot chosen parents with proportional circles and label
    if len(parents) > 0:
        parent_counter = Counter([tuple(x) for x in parents]) #need tuples because they are hashable
        unique_parents = np.array([[x[0], x[1], num] for x, num in parent_counter.items()])
        ax.scatter(unique_parents[:,0], unique_parents[:,1], s=40*unique_parents[:,2], color='#ffbe6b', marker='o', linestyle='None', zorder=15)
        for p in unique_parents:
            x, y, parent_count = p
            if parent_count > 5:
                ax.annotate(str(int(parent_count)), (x, y), zorder=30, ha="center", va="center", size=9)

    # plot additional data set
    if addl_data_set is not None:
        ax.plot(addl_data_set[:,0], addl_data_set[:,1], color='#FF8C00', marker=',', linestyle='None', zorder=20)

    if patches == "donut":
        ax.add_patch(Circle((0.5, 0.5), 0.125, fill=False, linestyle="dashed", linewidth=1, zorder=50))
        ax.add_patch(Circle((0.5, 0.5), 0.375, fill=False, linestyle="dashed", linewidth=2, zorder=50))

    if perturbation_methods and len(parents) > 0:
        p_cmap = {"all": "black", "lattice": "w", "lattice_nodens": "#848C8E", "density": "#435058", "atom_types": "#0FA3B1", "atom_sites": "#F77936", }
        arrows = [ConnectionPatch(parents[i], children[i], "data", arrowstyle="-|>", \
                    shrinkA=5, shrinkB=5, mutation_scale=20, fc=p_cmap[perturbation_methods[i]], \
                    linestyle="--", zorder=40) \
                    for i in range(len(children))]
        for a in arrows:
            ax.add_patch(a)

    ax.set_title(title)
    fig.savefig(output_path, dpi=600)
    plt.close(fig)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# __BEGIN_LICENSE__
#Copyright (c) 2015, United States Government, as represented by the
#Administrator of the National Aeronautics and Space Administration.
#All rights reserved.
#
#The xGDS platform is licensed under the Apache License, Version 2.0
#(the "License"); you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#http://www.apache.org/licenses/LICENSE-2.0.
#
#Unless required by applicable law or agreed to in writing, software distributed
#under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
#CONDITIONS OF ANY KIND, either express or implied. See the License for the
#specific language governing permissions and limitations under the License.
# __END_LICENSE__

# This Python script will generate a KML network link and a KML file which renders
# placemarks for the air quality at Hawai'i Volcanoes National Park.

import datetime
import pytz
import requests
import json
from xml.sax.saxutils import escape
from fastkml import kml, geometry, styles
from shapely.geometry import Point, Polygon
from polycircles import polycircles
from LatLon import LatLon

USE_MEMCACHE = True
CACHE_KEY = 'HVNP_AIR'

SERVER_URL = None
RAW_DATA_URL = "http://www.hawaiiso2network.com/havo_json.txt" # source of the json for the data
ADVISORY_SUMMARY_URL = 'http://www.hawaiiso2network.com/summary.html'

# Update this with the path to icons on your server
ICON_PATH = '/static/basaltApp/icons/hvnp_air_quality'

NAME_SPACE = '{http://www.opengis.net/kml/2.2}'

# Global variables
CURRENT_DATA = None
CONDITIONS = {}
SITE_POSITIONS = {'Visitor Center': {'lat': 19.4308,
                                     'lon': -155.2578,
                                     'alt': 1215},
                    'Jaggar Museum': {'lat': 19.4203,
                                    'lon': -155.2881,
                                    'alt': 1123}}

WEDGE_POSITIONS = {"Pu'U O'o Crater": {'lat': 19.387979,
                                       'lon': -155.107504,
                                       'alt': 1215},
                  'Halemaumau Crater': {'lat': 19.410000,
                                        'lon': -155.286389,
                                        'alt': 1123}}

PLUME_SOURCE = 'Jaggar Museum'  # update to which is the source of the plume data

#KML colors are aabbggrr in hex binary; colors in this map do not include transparency
ADVISORY_SCALE = {'Unknown':'FFFFFF',
                  'Good':'57C700',
                  'Moderate':'00DEFF',
                  'Unhealthy for Sensitive Groups':'318FF5',
                  'Unhealthy':'2601FF',
                  'Very Unhealthy':'480094',
                  'Hazardous':'260D80'}

WEDGE_LENGTH = 50.0 # in km
WEDGE_ANGLE = 30 # in degrees
CIRCLE_RADIUS = 2.0 #in km
DEGREE_SYMBOL = unichr(176)

def getCurrentStateKml(serverUrl):
    ''' Create the overall KML for the current state of the air quality.
    If we can't get fresh data, return None
    serverUrl = the url of the server providing this KML, so we can have the network link and the icons.
    '''

    # Try to get data
    global CURRENT_DATA
    CURRENT_DATA = getData()
    if not CURRENT_DATA:
        return None

    global SERVER_URL
    SERVER_URL = serverUrl

    # Create the root KML object
    topKml = kml.KML()

    # Create a KML Document and add it to the KML root object
    document = kml.Document(NAME_SPACE, 'hvnp_caq_doc',
                            "Hawai'i Volcanoes National Park",
                            "Approximate direction of the volcanic gas plumes (wedges) from Halema'uma'u and Pu'u 'O'o. Colored circles show the current air quality conditions for sulfur dioxide and particulate matter at each monitoring site.",
                            styles=buildStyles())

    topKml.append(document)

    # Read and interpret the current conditions
    getConditions()

    # Build a top level KML folder
    topFolder = kml.Folder(NAME_SPACE, 'hvnp_caq_doc', 'Current Air Quality', getConditionsString())

    # Build placemarks for all the found sites
    buildSites(topFolder)

    # Put the document together
    document.append(topFolder)

    # Return the KML as a string.
    return document.to_string(prettyprint=True)

def buildNetworkLink(url, name, interval=900):
    ''' Build a Network link to refresh an url every interval seconds
    '''
    return '''
<NetworkLink>
  <name>%(name)s</name>
  <Link>
    <href>%(url)s</href>
    <refreshMode>onInterval</refreshMode>
    <refreshInterval>%(interval)d</refreshInterval>
  </Link>
</NetworkLink>
''' % dict(name=escape(name),
           interval=interval,
           url=escape(url))

def getRawData():
    ''' Get the json data from the hawaiiso2network website and store it
    '''
    try:
        request = requests.get(RAW_DATA_URL, timeout=30)
        return request.json()
    except:
        return None

def getCachedMinutesAgo(datadate):
    try:
        hst = pytz.timezone('US/Hawaii')
        splits = datadate.split()
        lasttime = datetime.datetime.strptime(splits[1] + ' ' + splits[2], '%m/%d/%Y %H:%M')
        lasttime = hst.localize(lasttime)
        now = datetime.datetime.now(hst)
        delta = now - lasttime
        return delta.seconds / 60
    except:
        return 60 #force refresh

def getData():
    ''' Get the json data from cache or raw.
    '''
    if USE_MEMCACHE:
        import memcache
        _cache = memcache.Client(['127.0.0.1:11211'], debug=0)
        cachedData = _cache.get(CACHE_KEY)
        # If there is no cached data at all
        if not cachedData:
            newData = getRawData()
            if newData:
                _cache.set(CACHE_KEY, newData)
            return newData
        else:
            ago = getCachedMinutesAgo(cachedData['datadate'])
            if ago > 15:
                newData = getRawData()
                if newData:
                    _cache.set(CACHE_KEY, newData)
                    return newData
                else:
                    if ago > 30:
                        _cache.set(CACHE_KEY, None)
                        return None
            return cachedData
    else:
        return getRawData()

def getDataValue(keys, dictionary=None):
    ''' Convenience method to get data out of the json '''
    global CURRENT_DATA
    if dictionary is None:
        dictionary = CURRENT_DATA
    if "." in keys:
        key, rest = keys.split(".", 1)
        return getDataValue(rest, dictionary[key])
    else:
        if keys in dictionary:
            return dictionary[keys]
        return None

def getPlumeSourceData():
    ''' Get the data map for the current plume source '''
    for key in CURRENT_DATA:
        data = CURRENT_DATA[key]
        if data['name'] == PLUME_SOURCE:
            return data

def getConditions():
    ''' Interpret the MET block '''
    met = getDataValue('MET')
    CONDITIONS['WIND_SPEED'] = int(met['WS'])
    CONDITIONS['WIND_SPEED_METRIC'] = int(met['WS-metric'])
    CONDITIONS['WIND_DIRECTION'] = int(met['WD']) - 180
    CONDITIONS['TEMPERATURE'] = int(met['AT'])
    CONDITIONS['TEMPERATURE_METRIC'] = int(met['AT-metric'])
    CONDITIONS['LOW_WIND'] = CONDITIONS['WIND_SPEED_METRIC'] <= int(met['WS-threshold'])
    CONDITIONS['HUMIDITY'] = int(met['RH'])
    CONDITIONS['PARKWIDE'] = {'SO2': getDataValue('PARKWIDE.SO2.AQItext'),
                              'PM25': getDataValue('PARKWIDE.PM25.AQItext')}

def getConditionsString():
    ''' Get printable string of the high level conditions '''
    result = ('%s\nWind Speed: %d mph (%d m/s)\nWind Direction: %d\nTemperature: %d %sF (%d %sC)\nHumidity: %d %%\nParkwide SO2: %s\nParkwide PM25: %s')\
        % (getDataValue('datadate'),
           CONDITIONS['WIND_SPEED'],
           CONDITIONS['WIND_SPEED_METRIC'],
           CONDITIONS['WIND_DIRECTION'],
           CONDITIONS['TEMPERATURE'],
           DEGREE_SYMBOL,
           CONDITIONS['TEMPERATURE_METRIC'],
           DEGREE_SYMBOL,
           CONDITIONS['HUMIDITY'],
           CONDITIONS['PARKWIDE']['SO2'],
           CONDITIONS['PARKWIDE']['PM25']
           )
    return result

def makeKey(word):
    return word.replace(' ','_').lower()

def getDotIconId(level):
    ''' Get the clean id for a dot icon and style '''
    if level not in ADVISORY_SCALE:
        level = 'Unknown'
    return makeKey(level) + '_dot'

def buildDotIconUrl(level):
    ''' Build the URL to get the image for a dot icon '''
    return ('%s%s/%s.png') % (SERVER_URL, ICON_PATH, getDotIconId(level))

def getPolyStyleId(level):
    if level not in ADVISORY_SCALE:
        level = 'Unknown'
    ''' Get the clean id for a polygon style '''
    return makeKey(level) + '_poly'

def buildStyles():
    ''' Build the various styles to be used for the placemarks'''
    kmlStyles = []
    for level in ADVISORY_SCALE:
        # build the dot style
        innerIconStyle = styles.IconStyle(NAME_SPACE, scale=.7, icon_href=buildDotIconUrl(level))
        iconStyle = styles.Style(NAME_SPACE, id=getDotIconId(level), styles=[innerIconStyle])
        kmlStyles.append(iconStyle)

        # build the polygon style
        innerPolyStyle = styles.PolyStyle(NAME_SPACE, color='33' + ADVISORY_SCALE[level], fill=1, outline=1)
        innerLineStyle = styles.LineStyle(NAME_SPACE, color='FF' + ADVISORY_SCALE[level], width=2)
        polyStyle = styles.Style(NAME_SPACE, id=getPolyStyleId(level), styles=[innerLineStyle, innerPolyStyle])
        kmlStyles.append(polyStyle)
    return kmlStyles

def buildSites(folder):
    ''' Iterate through the json data and create visible KML elements for each site.
    '''
    siteData = 'start'
    index = 1
    while siteData is not None:
        key = 'SITE' + str(index)
        siteData = getDataValue(key)
        if siteData:
            buildPlacemarkForSite(folder, siteData)
        index += 1

def buildPlacemarkForSite(folder, siteData):
    ''' Build a placemark for a particular site.
        If the site is in PLUME_SOURCES, then show a wedge or circle depending on the wind speed.
        Otherwise show just a point.
        Render a popup or not based on json data.
    '''
    siteName = siteData['name']
    if siteName in SITE_POSITIONS:
        dotPlacemark = kml.Placemark(NAME_SPACE, siteName, siteName, getSiteDescription(siteData))
        dotPlacemark.styleUrl = '#' + getDotIconId(getDataValue('SO2.AQItext', siteData))
        dotPlacemark.geometry = buildPoint(SITE_POSITIONS[siteName], siteName)
        folder.append(dotPlacemark)

        if siteName == PLUME_SOURCE:
            plumeData = getPlumeSourceData()
            for crater in WEDGE_POSITIONS:
                craterPlacemark = kml.Placemark(NAME_SPACE, makeKey(crater), crater, getCraterDescription(plumeData))
                craterPlacemark.styleUrl = '#' + getPolyStyleId(getDataValue('SO2.AQItext', plumeData))
                if CONDITIONS['LOW_WIND']:
                    craterPlacemark.geometry = buildCircle(WEDGE_POSITIONS[crater], crater)
                else:
                    craterPlacemark.geometry = buildWedge(WEDGE_POSITIONS[crater], CONDITIONS['WIND_DIRECTION'], crater)
                folder.append(craterPlacemark);

def getSiteDescription(siteData):
    ''' Get printable string to show when the user clicks on a placemark for a site '''
    result = ('%s\n') % getDataValue('name', siteData)
    if 'SO2' in siteData:
        result += ('SO2: %f ppm\n') % float(getDataValue('SO2.average15', siteData))
    if 'PM25' in siteData:
        result += ('PM25: %d %sg/m3\n') % (int(getDataValue('PM25.average15', siteData)), unichr(956))
    return result

def getCraterDescription(siteData):
    ''' Get printable string of the plume conditions '''
    result = ('%s\nWind Speed: %d mph (%d m/s)\nWind Direction: %d\nTemperature: %d %sF (%d %sC)\nHumidity: %d %%\n%s SO2: %s ppm (%s)\n%s PM25: %s %sg/m3 (%s)')\
        % (getDataValue('datadate'),
           CONDITIONS['WIND_SPEED'],
           CONDITIONS['WIND_SPEED_METRIC'],
           CONDITIONS['WIND_DIRECTION'],
           CONDITIONS['TEMPERATURE'],
           DEGREE_SYMBOL,
           CONDITIONS['TEMPERATURE_METRIC'],
           DEGREE_SYMBOL,
           CONDITIONS['HUMIDITY'],
           PLUME_SOURCE,
           siteData['SO2']['average15'],
           siteData['SO2']['AQItext'],
           PLUME_SOURCE,
           siteData['PM25']['average15'],
           unichr(956),
           siteData['PM25']['AQItext'],
           )
    return result

def buildPoint(location, name):
    pointID = makeKey(name) + '_point'
    pointShape =  Point([(location['lon'], location['lat'], location['alt'])])
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=pointID, geometry=pointShape, extrude=False, tessellate=False, altitude_mode='clampToGround')
    return kmlGeometry

def buildHeading(heading):
    if heading < 0:
        heading += 360
    elif heading > 360:
        heading -= 360
    return heading

def buildWedge(location, heading, name):
    wedgeID = makeKey(name) + '_wedge'
    centerPoint = LatLon(location['lat'], location['lon'])
    point1 = centerPoint.offset(buildHeading(heading - (WEDGE_ANGLE/2.0)), float(WEDGE_LENGTH))
    point2 = centerPoint.offset(buildHeading(heading + (WEDGE_ANGLE/2.0)), float(WEDGE_LENGTH))
    coords = [(centerPoint.lon.decimal_degree, centerPoint.lat.decimal_degree),
              (point1.lon.decimal_degree, point1.lat.decimal_degree),
              (point2.lon.decimal_degree, point2.lat.decimal_degree)]
    wedgeShape = Polygon(coords)
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=wedgeID, geometry=wedgeShape, extrude=False, tessellate=False, altitude_mode='clampToGround')
    return kmlGeometry

def buildCircle(location, name):
    circleID = makeKey(name) + '_circle'
    boundaryCircle = polycircles.Polycircle(latitude=location['lat'],
                                            longitude=location['lon'],
                                            radius=CIRCLE_RADIUS * 1000,
                                            number_of_vertices=36)
    coords = [(coord[1], coord[0]) for coord in boundaryCircle.vertices]
    circleShape = Polygon(coords)
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=circleID, geometry=circleShape, extrude=False, tessellate=True, altitude_mode='clampToGround')
    return kmlGeometry
<commit_msg>return better kml when no data<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# __BEGIN_LICENSE__
#Copyright (c) 2015, United States Government, as represented by the
#Administrator of the National Aeronautics and Space Administration.
#All rights reserved.
#
#The xGDS platform is licensed under the Apache License, Version 2.0
#(the "License"); you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#http://www.apache.org/licenses/LICENSE-2.0.
#
#Unless required by applicable law or agreed to in writing, software distributed
#under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
#CONDITIONS OF ANY KIND, either express or implied. See the License for the
#specific language governing permissions and limitations under the License.
# __END_LICENSE__

# This Python script will generate a KML network link and a KML file which renders
# placemarks for the air quality at Hawai'i Volcanoes National Park.

import datetime
import pytz
import requests
import json
from xml.sax.saxutils import escape
from fastkml import kml, geometry, styles
from shapely.geometry import Point, Polygon
from polycircles import polycircles
from LatLon import LatLon

USE_MEMCACHE = True
CACHE_KEY = 'HVNP_AIR'

SERVER_URL = None
RAW_DATA_URL = "http://www.hawaiiso2network.com/havo_json.txt" # source of the json for the data
ADVISORY_SUMMARY_URL = 'http://www.hawaiiso2network.com/summary.html'

# Update this with the path to icons on your server
ICON_PATH = '/static/basaltApp/icons/hvnp_air_quality'

NAME_SPACE = '{http://www.opengis.net/kml/2.2}'

# Global variables
CURRENT_DATA = None
CONDITIONS = {}
SITE_POSITIONS = {'Visitor Center': {'lat': 19.4308,
                                     'lon': -155.2578,
                                     'alt': 1215},
                    'Jaggar Museum': {'lat': 19.4203,
                                    'lon': -155.2881,
                                    'alt': 1123}}

WEDGE_POSITIONS = {"Pu'U O'o Crater": {'lat': 19.387979,
                                       'lon': -155.107504,
                                       'alt': 1215},
                  'Halemaumau Crater': {'lat': 19.410000,
                                        'lon': -155.286389,
                                        'alt': 1123}}

PLUME_SOURCE = 'Jaggar Museum'  # update to which is the source of the plume data

#KML colors are aabbggrr in hex binary; colors in this map do not include transparency
ADVISORY_SCALE = {'Unknown':'FFFFFF',
                  'Good':'57C700',
                  'Moderate':'00DEFF',
                  'Unhealthy for Sensitive Groups':'318FF5',
                  'Unhealthy':'2601FF',
                  'Very Unhealthy':'480094',
                  'Hazardous':'260D80'}

WEDGE_LENGTH = 50.0 # in km
WEDGE_ANGLE = 30 # in degrees
CIRCLE_RADIUS = 2.0 #in km
DEGREE_SYMBOL = unichr(176)

def getCurrentStateKml(serverUrl):
    ''' Create the overall KML for the current state of the air quality.
    If we can't get fresh data, return None
    serverUrl = the url of the server providing this KML, so we can have the network link and the icons.
    '''

    # Try to get data
    global CURRENT_DATA
    CURRENT_DATA = getData()

    global SERVER_URL
    SERVER_URL = serverUrl

    # Create the root KML object
    topKml = kml.KML()

    # Create a KML Document and add it to the KML root object
    document = kml.Document(NAME_SPACE, 'hvnp_caq_doc',
                            "Hawai'i Volcanoes National Park",
                            "Approximate direction of the volcanic gas plumes (wedges) from Halema'uma'u and Pu'u 'O'o. Colored circles show the current air quality conditions for sulfur dioxide and particulate matter at each monitoring site.",
                            styles=buildStyles())

    topKml.append(document)
    if not CURRENT_DATA:
        document.description = "NO DATA; Cannot reach server"
        return document.to_string(prettyprint=True)

    # Read and interpret the current conditions
    getConditions()

    # Build a top level KML folder
    topFolder = kml.Folder(NAME_SPACE, 'hvnp_caq_doc', 'Current Air Quality', getConditionsString())

    # Build placemarks for all the found sites
    buildSites(topFolder)

    # Put the document together
    document.append(topFolder)

    # Return the KML as a string.
    return document.to_string(prettyprint=True)

def buildNetworkLink(url, name, interval=900):
    ''' Build a Network link to refresh an url every interval seconds
    '''
    return '''
<NetworkLink>
  <name>%(name)s</name>
  <Link>
    <href>%(url)s</href>
    <refreshMode>onInterval</refreshMode>
    <refreshInterval>%(interval)d</refreshInterval>
  </Link>
</NetworkLink>
''' % dict(name=escape(name),
           interval=interval,
           url=escape(url))

def getRawData():
    ''' Get the json data from the hawaiiso2network website and store it
    '''
    try:
        request = requests.get(RAW_DATA_URL, timeout=30)
        return request.json()
    except:
        return None

def getCachedMinutesAgo(datadate):
    try:
        hst = pytz.timezone('US/Hawaii')
        splits = datadate.split()
        lasttime = datetime.datetime.strptime(splits[1] + ' ' + splits[2], '%m/%d/%Y %H:%M')
        lasttime = hst.localize(lasttime)
        now = datetime.datetime.now(hst)
        delta = now - lasttime
        return delta.seconds / 60
    except:
        return 60 #force refresh

def getData():
    ''' Get the json data from cache or raw.
    '''
    if USE_MEMCACHE:
        import memcache
        _cache = memcache.Client(['127.0.0.1:11211'], debug=0)
        cachedData = _cache.get(CACHE_KEY)
        # If there is no cached data at all
        if not cachedData:
            newData = getRawData()
            if newData:
                _cache.set(CACHE_KEY, newData)
            return newData
        else:
            ago = getCachedMinutesAgo(cachedData['datadate'])
            if ago > 15:
                newData = getRawData()
                if newData:
                    _cache.set(CACHE_KEY, newData)
                    return newData
                else:
                    if ago > 30:
                        _cache.set(CACHE_KEY, None)
                        return None
            return cachedData
    else:
        return getRawData()

def getDataValue(keys, dictionary=None):
    ''' Convenience method to get data out of the json '''
    global CURRENT_DATA
    if dictionary is None:
        dictionary = CURRENT_DATA
    if "." in keys:
        key, rest = keys.split(".", 1)
        return getDataValue(rest, dictionary[key])
    else:
        if keys in dictionary:
            return dictionary[keys]
        return None

def getPlumeSourceData():
    ''' Get the data map for the current plume source '''
    for key in CURRENT_DATA:
        data = CURRENT_DATA[key]
        if data['name'] == PLUME_SOURCE:
            return data

def getConditions():
    ''' Interpret the MET block '''
    met = getDataValue('MET')
    CONDITIONS['WIND_SPEED'] = int(met['WS'])
    CONDITIONS['WIND_SPEED_METRIC'] = int(met['WS-metric'])
    CONDITIONS['WIND_DIRECTION'] = int(met['WD']) - 180
    CONDITIONS['TEMPERATURE'] = int(met['AT'])
    CONDITIONS['TEMPERATURE_METRIC'] = int(met['AT-metric'])
    CONDITIONS['LOW_WIND'] = CONDITIONS['WIND_SPEED_METRIC'] <= int(met['WS-threshold'])
    CONDITIONS['HUMIDITY'] = int(met['RH'])
    CONDITIONS['PARKWIDE'] = {'SO2': getDataValue('PARKWIDE.SO2.AQItext'),
                              'PM25': getDataValue('PARKWIDE.PM25.AQItext')}

def getConditionsString():
    ''' Get printable string of the high level conditions '''
    result = ('%s\nWind Speed: %d mph (%d m/s)\nWind Direction: %d\nTemperature: %d %sF (%d %sC)\nHumidity: %d %%\nParkwide SO2: %s\nParkwide PM25: %s')\
        % (getDataValue('datadate'),
           CONDITIONS['WIND_SPEED'],
           CONDITIONS['WIND_SPEED_METRIC'],
           CONDITIONS['WIND_DIRECTION'],
           CONDITIONS['TEMPERATURE'],
           DEGREE_SYMBOL,
           CONDITIONS['TEMPERATURE_METRIC'],
           DEGREE_SYMBOL,
           CONDITIONS['HUMIDITY'],
           CONDITIONS['PARKWIDE']['SO2'],
           CONDITIONS['PARKWIDE']['PM25']
           )
    return result

def makeKey(word):
    return word.replace(' ','_').lower()

def getDotIconId(level):
    ''' Get the clean id for a dot icon and style '''
    if level not in ADVISORY_SCALE:
        level = 'Unknown'
    return makeKey(level) + '_dot'

def buildDotIconUrl(level):
    ''' Build the URL to get the image for a dot icon '''
    return ('%s%s/%s.png') % (SERVER_URL, ICON_PATH, getDotIconId(level))

def getPolyStyleId(level):
    if level not in ADVISORY_SCALE:
        level = 'Unknown'
    ''' Get the clean id for a polygon style '''
    return makeKey(level) + '_poly'

def buildStyles():
    ''' Build the various styles to be used for the placemarks'''
    kmlStyles = []
    for level in ADVISORY_SCALE:
        # build the dot style
        innerIconStyle = styles.IconStyle(NAME_SPACE, scale=.7, icon_href=buildDotIconUrl(level))
        iconStyle = styles.Style(NAME_SPACE, id=getDotIconId(level), styles=[innerIconStyle])
        kmlStyles.append(iconStyle)

        # build the polygon style
        innerPolyStyle = styles.PolyStyle(NAME_SPACE, color='33' + ADVISORY_SCALE[level], fill=1, outline=1)
        innerLineStyle = styles.LineStyle(NAME_SPACE, color='FF' + ADVISORY_SCALE[level], width=2)
        polyStyle = styles.Style(NAME_SPACE, id=getPolyStyleId(level), styles=[innerLineStyle, innerPolyStyle])
        kmlStyles.append(polyStyle)
    return kmlStyles

def buildSites(folder):
    ''' Iterate through the json data and create visible KML elements for each site.
    '''
    siteData = 'start'
    index = 1
    while siteData is not None:
        key = 'SITE' + str(index)
        siteData = getDataValue(key)
        if siteData:
            buildPlacemarkForSite(folder, siteData)
        index += 1

def buildPlacemarkForSite(folder, siteData):
    ''' Build a placemark for a particular site.
        If the site is in PLUME_SOURCES, then show a wedge or circle depending on the wind speed.
        Otherwise show just a point.
        Render a popup or not based on json data.
    '''
    siteName = siteData['name']
    if siteName in SITE_POSITIONS:
        dotPlacemark = kml.Placemark(NAME_SPACE, siteName, siteName, getSiteDescription(siteData))
        dotPlacemark.styleUrl = '#' + getDotIconId(getDataValue('SO2.AQItext', siteData))
        dotPlacemark.geometry = buildPoint(SITE_POSITIONS[siteName], siteName)
        folder.append(dotPlacemark)

        if siteName == PLUME_SOURCE:
            plumeData = getPlumeSourceData()
            for crater in WEDGE_POSITIONS:
                craterPlacemark = kml.Placemark(NAME_SPACE, makeKey(crater), crater, getCraterDescription(plumeData))
                craterPlacemark.styleUrl = '#' + getPolyStyleId(getDataValue('SO2.AQItext', plumeData))
                if CONDITIONS['LOW_WIND']:
                    craterPlacemark.geometry = buildCircle(WEDGE_POSITIONS[crater], crater)
                else:
                    craterPlacemark.geometry = buildWedge(WEDGE_POSITIONS[crater], CONDITIONS['WIND_DIRECTION'], crater)
                folder.append(craterPlacemark);

def getSiteDescription(siteData):
    ''' Get printable string to show when the user clicks on a placemark for a site '''
    result = ('%s\n') % getDataValue('name', siteData)
    if 'SO2' in siteData:
        result += ('SO2: %f ppm\n') % float(getDataValue('SO2.average15', siteData))
    if 'PM25' in siteData:
        result += ('PM25: %d %sg/m3\n') % (int(getDataValue('PM25.average15', siteData)), unichr(956))
    return result

def getCraterDescription(siteData):
    ''' Get printable string of the plume conditions '''
    result = ('%s\nWind Speed: %d mph (%d m/s)\nWind Direction: %d\nTemperature: %d %sF (%d %sC)\nHumidity: %d %%\n%s SO2: %s ppm (%s)\n%s PM25: %s %sg/m3 (%s)')\
        % (getDataValue('datadate'),
           CONDITIONS['WIND_SPEED'],
           CONDITIONS['WIND_SPEED_METRIC'],
           CONDITIONS['WIND_DIRECTION'],
           CONDITIONS['TEMPERATURE'],
           DEGREE_SYMBOL,
           CONDITIONS['TEMPERATURE_METRIC'],
           DEGREE_SYMBOL,
           CONDITIONS['HUMIDITY'],
           PLUME_SOURCE,
           siteData['SO2']['average15'],
           siteData['SO2']['AQItext'],
           PLUME_SOURCE,
           siteData['PM25']['average15'],
           unichr(956),
           siteData['PM25']['AQItext'],
           )
    return result

def buildPoint(location, name):
    pointID = makeKey(name) + '_point'
    pointShape =  Point([(location['lon'], location['lat'], location['alt'])])
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=pointID, geometry=pointShape, extrude=False, tessellate=False, altitude_mode='clampToGround')
    return kmlGeometry

def buildHeading(heading):
    if heading < 0:
        heading += 360
    elif heading > 360:
        heading -= 360
    return heading

def buildWedge(location, heading, name):
    wedgeID = makeKey(name) + '_wedge'
    centerPoint = LatLon(location['lat'], location['lon'])
    point1 = centerPoint.offset(buildHeading(heading - (WEDGE_ANGLE/2.0)), float(WEDGE_LENGTH))
    point2 = centerPoint.offset(buildHeading(heading + (WEDGE_ANGLE/2.0)), float(WEDGE_LENGTH))
    coords = [(centerPoint.lon.decimal_degree, centerPoint.lat.decimal_degree),
              (point1.lon.decimal_degree, point1.lat.decimal_degree),
              (point2.lon.decimal_degree, point2.lat.decimal_degree)]
    wedgeShape = Polygon(coords)
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=wedgeID, geometry=wedgeShape, extrude=False, tessellate=False, altitude_mode='clampToGround')
    return kmlGeometry

def buildCircle(location, name):
    circleID = makeKey(name) + '_circle'
    boundaryCircle = polycircles.Polycircle(latitude=location['lat'],
                                            longitude=location['lon'],
                                            radius=CIRCLE_RADIUS * 1000,
                                            number_of_vertices=36)
    coords = [(coord[1], coord[0]) for coord in boundaryCircle.vertices]
    circleShape = Polygon(coords)
    kmlGeometry = geometry.Geometry(NAME_SPACE, id=circleID, geometry=circleShape, extrude=False, tessellate=True, altitude_mode='clampToGround')
    return kmlGeometry
<|endoftext|>


<commit_before>from subprocess import call
from restnavigator import Navigator

# huckle's imports
from . import config
from . import hutils

import sys
import json
import subprocess
import time
import requests
import urllib

# produces a navigator that starts navigating from the root and with an api display name of apiname
def navigator(root, apiname):
    nav = Navigator.hal(root=root, apiname=apiname)
    return nav

# attempts to traverse through an hcli document with a command line argument
def traverse_argument(nav, arg):
    ilength = 0
    try:
        ilength = len(nav.links()["cli"])
    except Exception as warning:
        #hutils.eprint(warning)
        hutils.eprint(config.cliname + ": unable to navigate HCLI 1.0 compliant semantics.")
        sys.exit(1)

    for j, y in enumerate(nav.links()["cli"]):

        tempnav = nav.links()["cli"][j]

        try:
            if tempnav()["name"] == arg:
                nav = tempnav["cli"][0]
                return nav
        except:
            if arg == "help":
                hcli_to_man(nav)
                sys.exit(0)
            else:
                hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
                if hcli_type == config.hcli_parameter_type:
                    nav = tempnav["cli"][0](hcli_param=urllib.quote(arg))
                    return nav
                else:
                    hutils.eprint(config.cliname + ": " + arg + ": " + "command not found.")
                    sys.exit(2)

        if j == ilength - 1:
            if arg == "help":
                hcli_to_man(nav)
                sys.exit(0)
            else:
                hutils.eprint(config.cliname + ": " + arg + ": " + "command not found.")
                sys.exit(2)

# attempts to traverse through an execution. (only attempted when we've run out of command line arguments to parse)
def traverse_execution(nav):
    for k, z in enumerate(nav.links()["cli"]):
        tempnav = nav.links()["cli"][k]

        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_execution_type:
            method = tempnav()["http"]
            nav = tempnav["cli"][0]
            flexible_executor(nav.uri, method)
            sys.exit(0)

    hutils.eprint(config.cliname + ": " + "unable to execute.")
    for_help()
    sys.exit(2)

# attempts to pull at the root of the hcli to auto configure the cli
def pull(url):
    nav = navigator(root=url, apiname="unknown")
    try:
        version = nav()["hcli_version"]
        if version == "1.0":
            cli = nav()["name"]

            try:
                hcli_to_text(nav)
                config.create_configuration(cli, url)
                config.alias_cli(cli)
                print(cli + " was successfully configured.")
            except Exception as warning:
                hutils.eprint(warning)
    except Exception as warning:
        #hutils.eprint(warning)
        hutils.eprint(config.cliname + ": unable to navigate HCLI 1.0 compliant semantics.")

# displays a man page (file) located on a given path
def display_man_page(path):
    call(["man", path])

# converts an hcli document to a text and displays it
def hcli_to_text(navigator):
    for i, x in enumerate(navigator()["section"]):
        section = navigator()["section"][i]
        if section["name"].upper() == "EXAMPLES":
            print(options_and_commands_to_text(navigator))
        print(section["name"].upper())
        print("       " + section["description"] + "\n")

# generates an OPTIONS and COMMANDS section to add to a text page
def options_and_commands_to_text(navigator):

    # This block outputs an OPTIONS section, in the man page, alongside each available option flag and its description
    options = ""
    option_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_option_type:
            option_count += 1
            options = options + "       " + tempnav()["name"] + "\n"
            options = options + "              " + tempnav()["description"] + "\n"
    if option_count > 0:
        options = "OPTIONS\n" + options

    # This block outputs a COMMANDS section, in the man page, alongside each available command and its description
    commands = ""
    command_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_command_type:
            command_count += 1
            commands = commands + "       " + tempnav()["name"] + "\n"
            commands = commands + "              " + tempnav()["description"] + "\n"
    if command_count > 0:
        commands = "COMMANDS\n" + commands

    return options + commands

# converts an hcli document to a man page and displays it
def hcli_to_man(navigator):
    millis = str(time.time())
    dynamic_doc_path = config.cli_manpage_path + "/" + config.cliname + "." + millis + ".man"
    hutils.create_folder(config.cli_manpage_path)
    hutils.create_file(dynamic_doc_path)
    f = open(dynamic_doc_path, "a+")
    f.write(".TH " + navigator()["name"] + " 1 \n")
    for i, x in enumerate(navigator()["section"]):
        section = navigator()["section"][i]
        if section["name"].upper() == "EXAMPLES":
            f.write(options_and_commands_to_man(navigator))
        f.write(".SH " + section["name"].upper() + "\n")
        f.write(section["description"] + "\n")

    f.close()
    display_man_page(dynamic_doc_path)

# generates an OPTIONS and COMMANDS section to add to a man page
def options_and_commands_to_man(navigator):

    # This block outputs an OPTIONS section, in the man page, alongside each available option flag and its description
    options = ""
    option_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_option_type:
            option_count += 1
            options = options + ".IP " + tempnav()["name"] + "\n"
            options = options + tempnav()["description"] + "\n"
    if option_count > 0:
        options = ".SH OPTIONS\n" + options

    # This block outputs a COMMANDS section, in the man page, alongside each available command and its description
    commands = ""
    command_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_command_type:
            command_count += 1
            commands = commands + ".IP " + tempnav()["name"] + "\n"
            commands = commands + tempnav()["description"] + "\n"
    if command_count > 0:
        commands = ".SH COMMANDS\n" + commands

    return options + commands

# pretty json dump
def pretty_json(json):
    print(json.dumps(json, indent=4, sort_keys=True))

# standard error message to tell users to go check the help pages (man pages)
def for_help():
    hutils.eprint("for help, use:\n")
    hutils.eprint("  " + config.cliname + " help")
    hutils.eprint("  " + config.cliname + " <command> help")

# a flexible executor that can work with the application/octet-stream media-type (per HCLI 1.0 spec)
def flexible_executor(url, method):
    if method == "get":
        r = requests.get(url, stream=True)
        output_chunks(r)
        return
    if method == "post":
        if not sys.stdin.isatty():
            with sys.stdin as f:

                headers = {'content-type': 'application/octet-stream'}
                r = requests.post(url, data=f.read(), headers=headers, stream=True)

                output_chunks(r)
                return
    else:
        r = requests.post(url, data=None, stream=True)
        output_chunks(r)
        return

# outputs the response received from an execution
def output_chunks(response):
    if response.status_code >= 400:
        code = response.status_code
        hutils.eprint(code, requests.status_codes._codes[code][0])
        hutils.eprint(response.headers)
        hutils.eprint(response.content)
        sys.exit(1)
    else:
        with sys.stdout as f:
            for chunk in response.iter_content(16384):
                if chunk:
                    f.write(chunk)
<commit_msg>attempt to handle byte output (rather than string) on stdout<commit_after>from subprocess import call
from restnavigator import Navigator

# huckle's imports
from . import config
from . import hutils

import sys
import json
import subprocess
import time
import requests
import urllib

# produces a navigator that starts navigating from the root and with an api display name of apiname
def navigator(root, apiname):
    nav = Navigator.hal(root=root, apiname=apiname)
    return nav

# attempts to traverse through an hcli document with a command line argument
def traverse_argument(nav, arg):
    ilength = 0
    try:
        ilength = len(nav.links()["cli"])
    except Exception as warning:
        #hutils.eprint(warning)
        hutils.eprint(config.cliname + ": unable to navigate HCLI 1.0 compliant semantics.")
        sys.exit(1)

    for j, y in enumerate(nav.links()["cli"]):

        tempnav = nav.links()["cli"][j]

        try:
            if tempnav()["name"] == arg:
                nav = tempnav["cli"][0]
                return nav
        except:
            if arg == "help":
                hcli_to_man(nav)
                sys.exit(0)
            else:
                hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
                if hcli_type == config.hcli_parameter_type:
                    nav = tempnav["cli"][0](hcli_param=urllib.quote(arg))
                    return nav
                else:
                    hutils.eprint(config.cliname + ": " + arg + ": " + "command not found.")
                    sys.exit(2)

        if j == ilength - 1:
            if arg == "help":
                hcli_to_man(nav)
                sys.exit(0)
            else:
                hutils.eprint(config.cliname + ": " + arg + ": " + "command not found.")
                sys.exit(2)

# attempts to traverse through an execution. (only attempted when we've run out of command line arguments to parse)
def traverse_execution(nav):
    for k, z in enumerate(nav.links()["cli"]):
        tempnav = nav.links()["cli"][k]

        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_execution_type:
            method = tempnav()["http"]
            nav = tempnav["cli"][0]
            flexible_executor(nav.uri, method)
            sys.exit(0)

    hutils.eprint(config.cliname + ": " + "unable to execute.")
    for_help()
    sys.exit(2)

# attempts to pull at the root of the hcli to auto configure the cli
def pull(url):
    nav = navigator(root=url, apiname="unknown")
    try:
        version = nav()["hcli_version"]
        if version == "1.0":
            cli = nav()["name"]

            try:
                hcli_to_text(nav)
                config.create_configuration(cli, url)
                config.alias_cli(cli)
                print(cli + " was successfully configured.")
            except Exception as warning:
                hutils.eprint(warning)
    except Exception as warning:
        #hutils.eprint(warning)
        hutils.eprint(config.cliname + ": unable to navigate HCLI 1.0 compliant semantics.")

# displays a man page (file) located on a given path
def display_man_page(path):
    call(["man", path])

# converts an hcli document to a text and displays it
def hcli_to_text(navigator):
    for i, x in enumerate(navigator()["section"]):
        section = navigator()["section"][i]
        if section["name"].upper() == "EXAMPLES":
            print(options_and_commands_to_text(navigator))
        print(section["name"].upper())
        print("       " + section["description"] + "\n")

# generates an OPTIONS and COMMANDS section to add to a text page
def options_and_commands_to_text(navigator):

    # This block outputs an OPTIONS section, in the man page, alongside each available option flag and its description
    options = ""
    option_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_option_type:
            option_count += 1
            options = options + "       " + tempnav()["name"] + "\n"
            options = options + "              " + tempnav()["description"] + "\n"
    if option_count > 0:
        options = "OPTIONS\n" + options

    # This block outputs a COMMANDS section, in the man page, alongside each available command and its description
    commands = ""
    command_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_command_type:
            command_count += 1
            commands = commands + "       " + tempnav()["name"] + "\n"
            commands = commands + "              " + tempnav()["description"] + "\n"
    if command_count > 0:
        commands = "COMMANDS\n" + commands

    return options + commands

# converts an hcli document to a man page and displays it
def hcli_to_man(navigator):
    millis = str(time.time())
    dynamic_doc_path = config.cli_manpage_path + "/" + config.cliname + "." + millis + ".man"
    hutils.create_folder(config.cli_manpage_path)
    hutils.create_file(dynamic_doc_path)
    f = open(dynamic_doc_path, "a+")
    f.write(".TH " + navigator()["name"] + " 1 \n")
    for i, x in enumerate(navigator()["section"]):
        section = navigator()["section"][i]
        if section["name"].upper() == "EXAMPLES":
            f.write(options_and_commands_to_man(navigator))
        f.write(".SH " + section["name"].upper() + "\n")
        f.write(section["description"] + "\n")

    f.close()
    display_man_page(dynamic_doc_path)

# generates an OPTIONS and COMMANDS section to add to a man page
def options_and_commands_to_man(navigator):

    # This block outputs an OPTIONS section, in the man page, alongside each available option flag and its description
    options = ""
    option_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_option_type:
            option_count += 1
            options = options + ".IP " + tempnav()["name"] + "\n"
            options = options + tempnav()["description"] + "\n"
    if option_count > 0:
        options = ".SH OPTIONS\n" + options

    # This block outputs a COMMANDS section, in the man page, alongside each available command and its description
    commands = ""
    command_count = 0
    for i, x in enumerate(navigator.links()["cli"]):
        tempnav = navigator.links()["cli"][i]
        hcli_type = tempnav.links()["profile"][0].uri.split('#', 1)[1]
        if hcli_type == config.hcli_command_type:
            command_count += 1
            commands = commands + ".IP " + tempnav()["name"] + "\n"
            commands = commands + tempnav()["description"] + "\n"
    if command_count > 0:
        commands = ".SH COMMANDS\n" + commands

    return options + commands

# pretty json dump
def pretty_json(json):
    print(json.dumps(json, indent=4, sort_keys=True))

# standard error message to tell users to go check the help pages (man pages)
def for_help():
    hutils.eprint("for help, use:\n")
    hutils.eprint("  " + config.cliname + " help")
    hutils.eprint("  " + config.cliname + " <command> help")

# a flexible executor that can work with the application/octet-stream media-type (per HCLI 1.0 spec)
def flexible_executor(url, method):
    if method == "get":
        r = requests.get(url, stream=True)
        output_chunks(r)
        return
    if method == "post":
        if not sys.stdin.isatty():
            with sys.stdin as f:

                headers = {'content-type': 'application/octet-stream'}
                r = requests.post(url, data=f.read(), headers=headers, stream=True)

                output_chunks(r)
                return
    else:
        r = requests.post(url, data=None, stream=True)
        output_chunks(r)
        return

# outputs the response received from an execution
def output_chunks(response):
    if response.status_code >= 400:
        code = response.status_code
        hutils.eprint(code, requests.status_codes._codes[code][0])
        hutils.eprint(response.headers)
        hutils.eprint(response.content)
        sys.exit(1)
    else:
        with sys.stdout.buffer as f:
            for chunk in response.iter_content(16384):
                if chunk:
                    f.write(chunk)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Simple DNS server comparison benchmarking tool.

Designed to assist system administrators in selection and prioritization.
"""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

import datetime
import sys
import tempfile
from lib import benchmark
from lib import config
from lib import history_parser
from lib import nameserver_list
from lib import conn_quality

VERSION = '0.9.1'

class NameBenchCli(object):
  def __init__(self, options, primary_ns, secondary_ns):
    self.options = options
    self.primary_ns = primary_ns
    self.secondary_ns = secondary_ns

  def msg(self, msg, count=None, total=None, error=False):
    if error:
      print
      print '* ERROR: %s' % msg
      sys.exit(2)
    elif not total:
      print '- %s' % msg
    elif count == 1:
      sys.stdout.write('- %s: %s/%s' % (msg, count, total))
    elif count == total:
      sys.stdout.write('%s/%s\n' % (count, total))
    else:
      sys.stdout.write('.')
    sys.stdout.flush()

  def PrepareNameservers(self):
    include_internal = True
    if self.options.only:
      include_internal = False
      if not self.primary_ns:
        print 'If you use --only, you must provide nameservers to use.'
        sys.exit(1)

    nameservers = nameserver_list.NameServers(
        self.primary_ns, self.secondary_ns,
        num_servers=self.options.num_servers,
        include_internal=include_internal,
        timeout=self.options.timeout,
        health_timeout=self.options.health_timeout,
        status_callback=self.msg
    )
    nameservers.CheckHealth()
    print ''
    print 'Final list of nameservers considered:'
    print '-' * 78
    for n in nameservers.SortByFastest():
      print '%-15.15s %-16.16s %-4.0fms | %s' % (n.ip, n.name, n.check_duration,
                                               n.warnings_string)
    print ''
    return nameservers

  def PrepareBenchmark(self, nameservers):
    if self.options.import_file:
      importer = history_parser.HistoryParser()
      history = importer.Parse(self.options.import_file)
      if history:
        print '- Imported %s records from %s' % (len(history), self.options.import_file)
      else:
        print '- Could not import anything from %s' % self.options.import_file
        sys.exit(2)
    else:
      history = None

    bmark = benchmark.Benchmark(nameservers,
                                run_count=self.options.run_count,
                                test_count=self.options.test_count,
                                status_callback=self.msg)
    if history:
      bmark.CreateTests(history, select_mode=self.options.select_mode)
    else:
      bmark.CreateTestsFromFile(self.options.data_file, select_mode=self.options.select_mode)

    return bmark

  def Execute(self):
    print('namebench %s - %s (%s) on %s' %
          (VERSION, self.options.import_file or self.options.data_file, self.options.select_mode,
           datetime.datetime.now()))
    print ('threads=%s tests=%s runs=%s timeout=%s health_timeout=%s servers=%s' %
           (self.options.thread_count, self.options.test_count, self.options.run_count, self.options.timeout,
            self.options.health_timeout, self.options.num_servers))
    print '-' * 78

    nameservers = self.PrepareNameservers()
    bmark = self.PrepareBenchmark(nameservers)
    bmark.Run()
    print ''
    print bmark.CreateReport(format='ascii')
    if self.options.output_file:
      f = open(self.options.output_file, 'w')
      print '* Saving report to %s (%s)' % (self.options.output_file,
                                            self.options.output_format)
      f.write(bmark.CreateReport(format=self.options.output_format))
      f.close()

    if self.options.csv_file:
      print ''
      print '* Saving request details to %s' % self.options.csv_file
      bmark.SaveResultsToCsv(self.options.output_file)

if __name__ == '__main__':
  (options, primary, secondary) = config.GetConfiguration()
  cli = NameBenchCli(options, primary, secondary)
  cli.Execute()
<commit_msg>Oops, the thread_count was not being passed by the CLI<commit_after>#!/usr/bin/env python
# Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Simple DNS server comparison benchmarking tool.

Designed to assist system administrators in selection and prioritization.
"""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

import datetime
import sys
import tempfile
from lib import benchmark
from lib import config
from lib import history_parser
from lib import nameserver_list
from lib import conn_quality

VERSION = '0.9.1'

class NameBenchCli(object):
  def __init__(self, options, primary_ns, secondary_ns):
    self.options = options
    self.primary_ns = primary_ns
    self.secondary_ns = secondary_ns

  def msg(self, msg, count=None, total=None, error=False):
    if error:
      print
      print '* ERROR: %s' % msg
      sys.exit(2)
    elif not total:
      print '- %s' % msg
    elif count == 1:
      sys.stdout.write('- %s: %s/%s' % (msg, count, total))
    elif count == total:
      sys.stdout.write('%s/%s\n' % (count, total))
    else:
      sys.stdout.write('.')
    sys.stdout.flush()

  def PrepareNameservers(self):
    include_internal = True
    if self.options.only:
      include_internal = False
      if not self.primary_ns:
        print 'If you use --only, you must provide nameservers to use.'
        sys.exit(1)

    nameservers = nameserver_list.NameServers(
        self.primary_ns, self.secondary_ns,
        num_servers=self.options.num_servers,
        include_internal=include_internal,
        timeout=self.options.timeout,
        health_timeout=self.options.health_timeout,
        status_callback=self.msg,
        threads=self.options.thread_count
    )
    nameservers.CheckHealth()
    print ''
    print 'Final list of nameservers considered:'
    print '-' * 78
    for n in nameservers.SortByFastest():
      print '%-15.15s %-16.16s %-4.0fms | %s' % (n.ip, n.name, n.check_duration,
                                               n.warnings_string)
    print ''
    return nameservers

  def PrepareBenchmark(self, nameservers):
    if self.options.import_file:
      importer = history_parser.HistoryParser()
      history = importer.Parse(self.options.import_file)
      if history:
        print '- Imported %s records from %s' % (len(history), self.options.import_file)
      else:
        print '- Could not import anything from %s' % self.options.import_file
        sys.exit(2)
    else:
      history = None

    bmark = benchmark.Benchmark(nameservers,
                                run_count=self.options.run_count,
                                test_count=self.options.test_count,
                                status_callback=self.msg)
    if history:
      bmark.CreateTests(history, select_mode=self.options.select_mode)
    else:
      bmark.CreateTestsFromFile(self.options.data_file, select_mode=self.options.select_mode)

    return bmark

  def Execute(self):
    print('namebench %s - %s (%s) on %s' %
          (VERSION, self.options.import_file or self.options.data_file, self.options.select_mode,
           datetime.datetime.now()))
    print ('threads=%s tests=%s runs=%s timeout=%s health_timeout=%s servers=%s' %
           (self.options.thread_count, self.options.test_count, self.options.run_count, self.options.timeout,
            self.options.health_timeout, self.options.num_servers))
    print '-' * 78

    nameservers = self.PrepareNameservers()
    bmark = self.PrepareBenchmark(nameservers)
    bmark.Run()
    print ''
    print bmark.CreateReport(format='ascii')
    if self.options.output_file:
      f = open(self.options.output_file, 'w')
      print '* Saving report to %s (%s)' % (self.options.output_file,
                                            self.options.output_format)
      f.write(bmark.CreateReport(format=self.options.output_format))
      f.close()

    if self.options.csv_file:
      print ''
      print '* Saving request details to %s' % self.options.csv_file
      bmark.SaveResultsToCsv(self.options.output_file)

if __name__ == '__main__':
  (options, primary, secondary) = config.GetConfiguration()
  cli = NameBenchCli(options, primary, secondary)
  cli.Execute()
<|endoftext|>