<commit_before>#!/usr/bin/env python
# img.py

import sys, cgi, re, urllib, urllib2, urlparse, collections, os
sys.path.append(os.path.dirname(__file__))
import config
imgdirname = os.path.join(config.cachedir, 'img')

scrapeagent = 'Mozilla/5.0 (Linux x86_64) Gecko/20100101 Firefox/31.0'
scrapeurl = 'https://www.google.com/search?tbm=isch&sa=X&biw=1440&bih=828'
scraper = re.compile(r'href="(?:[^"/]*//[^"/]*)?/imgres?[^"]*imgurl=([^&"]*)')

class HeadRequest(urllib2.Request):
  def get_method(self):
    return "HEAD"

# Some sites redirect image urls to http urls, or don't like hotlinking.
# Blacklist these as we find them.
blacklist = set([
   'atom.smasher.org',
   'pixabay.com',
   'www.public-domain-image.com'
])
cross_origin_ok = set([
   'upload.wikimedia.org'
])

def uri_without_query(uri):
  return uri.split('?', 1)[0]

def splituri(uri):
  return [p for p in uri_without_query(uri).split('/') if p]

def filename_from_uri(uri):
  return '/'.join(splituri(uri)[1:])

keepcharacters = ('-','.',' ','/','_',',',';')
def normname(name):
  safe = re.sub('[- /_,;]+', '-',
     "".join(c for c in name if c.isalnum() or c in keepcharacters)
     ).strip('-')
  safe = re.sub('^s-', '', safe)
  safe = re.sub('^((?:[^-]+-)?)t(?:ransparent)?-', '\\1trans-', safe)
  safe = re.sub('\.jpg$', '.jpeg', safe)
  return safe.lower()

def application(env, start_response):
  redirect_url = None
  try:
    form = cgi.FieldStorage(fp=env['wsgi.input'], environ=env)
    request_uri = env['REQUEST_URI']
    host_name = env['HTTP_HOST']
    if env.has_key('UWSGI_SCHEME'):
      scheme = env['UWSGI_SCHEME']
    elif env.has_key('HTTPS'):
      scheme = 'https'
    else:
      scheme = 'http'
    referer = env.get('HTTP_REFERER', scheme + '://' + host_name + '/')
    agent = env.get('HTTP_USER_AGENT', scrapeagent)
    origin = env.get('HTTP_ORIGIN', None)
    cache_control = env.get('HTTP_CACHE_CONTROL', '')
    pragma = env.get('HTTP_PRAGMA', '')
    userip = env['REMOTE_ADDR']
    filename = normname(urllib.unquote(filename_from_uri(request_uri)))
    m = []

    if m.append(re.match(r'(?i)(https?:)//?([^/]+/.*)$', filename)) or any(m):
      # http://...  is a redirect.  (Note server reduces repeated slashes.)
      p = m.pop()
      redirect_url = p.group(1) + '//' + p.group(2)

    elif filename in cache and not 'no-cache' in cache_control + pragma:
      # Use a cache to a avoid hitting imagesearch repeatedly.
      redirect_url = cache[filename]

    else:
      # TODO: look up a matching filename in the cache.
      fullname = os.path.join(imgdirname, filename)
      if os.path.isfile(fullname):
        redirect_url = file(fullname, 'rb').read()

    if redirect_url is None:
      redirect_url = 'http://pencilcode.net/image/vpencil-20-64.png'

      # Anything else is an imagesearch.
      # Labelled for noncommercial reuse with modification.
      tbs = 'sur:fm'
      terms = filename

      # path with /a/, /i/, /s/, /m/, /l/, or /1024x768/ sizes image
      isz = 's'
      if m.append(re.match(r'([aisml])[-/](.*)$', terms)) or any(m):
        p = m.pop()
        if p.group(1) == 'a':
          isz = None
        else:
          isz = p.group(1)
        terms = p.group(2)
      elif m.append(re.match(r'(\d+)x(\d+)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        isz = 'ex,iszw:' + p.group(1) + ',iszh:' + p.group(2)
        terms = p.group(3)
      if isz is not None:
        tbs += ',isz:' + isz

      # path with /color/, /gray/, /red/, /orange/, etc, selects color
      if m.append(re.match(r'(color|gray|trans)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',ic:' + p.group(1)
        terms = p.group(2)
      elif m.append(re.match(r'(red|orange|yellow|green|teal|blue|purple|' +
          r'pink|white|gray|black|brown)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',ic:specific,isc:' + p.group(1)
        terms = p.group(2)

      # path with /face/, /photo/, /clipart/, /lineart/
      if m.append(re.match(r'(face|photo|clipart|lineart|animated)' +
          r'[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',itp:' + p.group(1)
        terms = p.group(2)

      # ends with .gif, .jpeg, etc.
      if m.append(re.match(r'(.*)\.(gif|jpeg|png|bmp|svg|ico|webp)$',
           terms)) or any(m):
        p = m.pop()
        tbs += ',ift:' + p.group(2)
        terms = p.group(1)

      url = scrapeurl + '&safe=active&tbs=' + tbs

      headers = { 'User-Agent': agent, 'Referer': referer }
      if userip:
        # url += '&userip=' + userip
        headers['X-Forwarded-For'] = userip

      query = re.sub(r'[-\s\.\+%&=]+', '+', terms)
      url += '&q=' + query
      request = urllib2.Request(url, None, headers)
      response = urllib2.urlopen(request)
      text = response.read()
      for link in scraper.findall(text):
        candidate = urllib.unquote(link)
        u = urlparse.urlparse(candidate)
        if u.hostname in blacklist:
          continue
        try:
          probe = urllib2.urlopen(HeadRequest(candidate))
        except urllib2.HTTPError, e:
          print 'for', filename, 'probe of', candidate, 'returned', e.code
          continue
        redirect_url = candidate
        cache[filename] = redirect_url
        file(fullname, 'wb').write(redirect_url)
        break

  except Exception as e:
    print e
    pass
  finally:
    if (origin is None or
        urlparse.urlparse(redirect_url).hostname in cross_origin_ok or
        redirect_url.startswith('/proxy/')):
      pass
    else:
      redirect_url = '/proxy/' +  redirect_url

    start_response('302 Redirect', [('Location', redirect_url.encode('utf-8'))])
  return []

class Cache(collections.OrderedDict):
  '''A quick implementation of an LRU cache based on OrderedDict'''

  def __init__(self, nslots=3):
    super(Cache, self).__init__()
    self.nslots = nslots

  if not hasattr(collections.OrderedDict, 'move_to_end'):
    def move_to_end(self, key, last=True):
      if not last:
        raise NotImplementedError(
          'valuers different from True of the "last" parameter are '
          'not supported in this implementation')

      # do not use super().pop because the implemetation uses self[key]
      value = super(Cache, self).__getitem__(key)
      del self[key]
      super(Cache, self).__setitem__(key, value)

  def __getitem__(self, key):
    value = super(Cache, self).__getitem__(key)
    self.move_to_end(key)
    return value

  def __setitem__(self, key, value):
    if self.nslots < 1:
      return

    if len(self) >= self.nslots:
      super(Cache, self).popitem(last=False)

    super(Cache, self).__setitem__(key, value)

  def get(self, k, d=None):
    if k in self:
      self.move_to_end(k)

    return super(Cache, self).get(k, d)

  def __contains__(self, key):
    try:
      super(Cache, self).__getitem__(key)
    except KeyError:
      return False
    else:
      return True

  _internal_getitem = collections.OrderedDict.__getitem__

  class _ValuesView(collections.ValuesView):
    def __contains__(self, value):
      for key in self._mapping:
        if value == self._mapping._internal_getitem(key):
          return True
      return False

    def __iter__(self):
      for key in self._mapping:
        yield self._mapping._internal_getitem(key)

  class _ItemsView(collections.ItemsView):
    def __contains__(self, item):
      key, value = item
      try:
        v = self._mapping._internal_getitem(key)
      except KeyError:
        return False
      else:
        return v == value

    def __iter__(self):
      for key in self._mapping:
        yield (key, self._mapping._internal_getitem(key))

  def values(self):
    return self._ValuesView(self)

  def items(self):
    return self._ItemsView(self)

cache = Cache(nslots=100)
<commit_msg>Remove wikimedia from the cross-origin list.<commit_after>#!/usr/bin/env python
# img.py

import sys, cgi, re, urllib, urllib2, urlparse, collections, os
sys.path.append(os.path.dirname(__file__))
import config
imgdirname = os.path.join(config.cachedir, 'img')

scrapeagent = 'Mozilla/5.0 (Linux x86_64) Gecko/20100101 Firefox/31.0'
scrapeurl = 'https://www.google.com/search?tbm=isch&sa=X&biw=1440&bih=828'
scraper = re.compile(r'href="(?:[^"/]*//[^"/]*)?/imgres?[^"]*imgurl=([^&"]*)')

class HeadRequest(urllib2.Request):
  def get_method(self):
    return "HEAD"

# Some sites redirect image urls to http urls, or don't like hotlinking.
# Blacklist these as we find them.
blacklist = set([
   'atom.smasher.org',
   'pixabay.com',
   'www.public-domain-image.com'
])
cross_origin_ok = set([
#   Wikimedia no longer allows cross-origin!
#   'upload.wikimedia.org'
])

def uri_without_query(uri):
  return uri.split('?', 1)[0]

def splituri(uri):
  return [p for p in uri_without_query(uri).split('/') if p]

def filename_from_uri(uri):
  return '/'.join(splituri(uri)[1:])

keepcharacters = ('-','.',' ','/','_',',',';')
def normname(name):
  safe = re.sub('[- /_,;]+', '-',
     "".join(c for c in name if c.isalnum() or c in keepcharacters)
     ).strip('-')
  safe = re.sub('^s-', '', safe)
  safe = re.sub('^((?:[^-]+-)?)t(?:ransparent)?-', '\\1trans-', safe)
  safe = re.sub('\.jpg$', '.jpeg', safe)
  return safe.lower()

def application(env, start_response):
  redirect_url = None
  try:
    form = cgi.FieldStorage(fp=env['wsgi.input'], environ=env)
    request_uri = env['REQUEST_URI']
    host_name = env['HTTP_HOST']
    if env.has_key('UWSGI_SCHEME'):
      scheme = env['UWSGI_SCHEME']
    elif env.has_key('HTTPS'):
      scheme = 'https'
    else:
      scheme = 'http'
    referer = env.get('HTTP_REFERER', scheme + '://' + host_name + '/')
    agent = env.get('HTTP_USER_AGENT', scrapeagent)
    origin = env.get('HTTP_ORIGIN', None)
    cache_control = env.get('HTTP_CACHE_CONTROL', '')
    pragma = env.get('HTTP_PRAGMA', '')
    userip = env['REMOTE_ADDR']
    filename = normname(urllib.unquote(filename_from_uri(request_uri)))
    m = []

    if m.append(re.match(r'(?i)(https?:)//?([^/]+/.*)$', filename)) or any(m):
      # http://...  is a redirect.  (Note server reduces repeated slashes.)
      p = m.pop()
      redirect_url = p.group(1) + '//' + p.group(2)

    elif filename in cache and not 'no-cache' in cache_control + pragma:
      # Use a cache to a avoid hitting imagesearch repeatedly.
      redirect_url = cache[filename]

    else:
      # TODO: look up a matching filename in the cache.
      fullname = os.path.join(imgdirname, filename)
      if os.path.isfile(fullname):
        redirect_url = file(fullname, 'rb').read()

    if redirect_url is None:
      redirect_url = 'http://pencilcode.net/image/vpencil-20-64.png'

      # Anything else is an imagesearch.
      # Labelled for noncommercial reuse with modification.
      tbs = 'sur:fm'
      terms = filename

      # path with /a/, /i/, /s/, /m/, /l/, or /1024x768/ sizes image
      isz = 's'
      if m.append(re.match(r'([aisml])[-/](.*)$', terms)) or any(m):
        p = m.pop()
        if p.group(1) == 'a':
          isz = None
        else:
          isz = p.group(1)
        terms = p.group(2)
      elif m.append(re.match(r'(\d+)x(\d+)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        isz = 'ex,iszw:' + p.group(1) + ',iszh:' + p.group(2)
        terms = p.group(3)
      if isz is not None:
        tbs += ',isz:' + isz

      # path with /color/, /gray/, /red/, /orange/, etc, selects color
      if m.append(re.match(r'(color|gray|trans)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',ic:' + p.group(1)
        terms = p.group(2)
      elif m.append(re.match(r'(red|orange|yellow|green|teal|blue|purple|' +
          r'pink|white|gray|black|brown)[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',ic:specific,isc:' + p.group(1)
        terms = p.group(2)

      # path with /face/, /photo/, /clipart/, /lineart/
      if m.append(re.match(r'(face|photo|clipart|lineart|animated)' +
          r'[-/](.*)$', terms)) or any(m):
        p = m.pop()
        tbs += ',itp:' + p.group(1)
        terms = p.group(2)

      # ends with .gif, .jpeg, etc.
      if m.append(re.match(r'(.*)\.(gif|jpeg|png|bmp|svg|ico|webp)$',
           terms)) or any(m):
        p = m.pop()
        tbs += ',ift:' + p.group(2)
        terms = p.group(1)

      url = scrapeurl + '&safe=active&tbs=' + tbs

      headers = { 'User-Agent': agent, 'Referer': referer }
      if userip:
        # url += '&userip=' + userip
        headers['X-Forwarded-For'] = userip

      query = re.sub(r'[-\s\.\+%&=]+', '+', terms)
      url += '&q=' + query
      request = urllib2.Request(url, None, headers)
      response = urllib2.urlopen(request)
      text = response.read()
      for link in scraper.findall(text):
        candidate = urllib.unquote(link)
        u = urlparse.urlparse(candidate)
        if u.hostname in blacklist:
          continue
        try:
          probe = urllib2.urlopen(HeadRequest(candidate))
        except urllib2.HTTPError, e:
          print 'for', filename, 'probe of', candidate, 'returned', e.code
          continue
        redirect_url = candidate
        cache[filename] = redirect_url
        file(fullname, 'wb').write(redirect_url)
        break

  except Exception as e:
    print e
    pass
  finally:
    if (origin is None or
        urlparse.urlparse(redirect_url).hostname in cross_origin_ok or
        redirect_url.startswith('/proxy/')):
      pass
    else:
      redirect_url = '/proxy/' +  redirect_url

    start_response('302 Redirect', [('Location', redirect_url.encode('utf-8'))])
  return []

class Cache(collections.OrderedDict):
  '''A quick implementation of an LRU cache based on OrderedDict'''

  def __init__(self, nslots=3):
    super(Cache, self).__init__()
    self.nslots = nslots

  if not hasattr(collections.OrderedDict, 'move_to_end'):
    def move_to_end(self, key, last=True):
      if not last:
        raise NotImplementedError(
          'valuers different from True of the "last" parameter are '
          'not supported in this implementation')

      # do not use super().pop because the implemetation uses self[key]
      value = super(Cache, self).__getitem__(key)
      del self[key]
      super(Cache, self).__setitem__(key, value)

  def __getitem__(self, key):
    value = super(Cache, self).__getitem__(key)
    self.move_to_end(key)
    return value

  def __setitem__(self, key, value):
    if self.nslots < 1:
      return

    if len(self) >= self.nslots:
      super(Cache, self).popitem(last=False)

    super(Cache, self).__setitem__(key, value)

  def get(self, k, d=None):
    if k in self:
      self.move_to_end(k)

    return super(Cache, self).get(k, d)

  def __contains__(self, key):
    try:
      super(Cache, self).__getitem__(key)
    except KeyError:
      return False
    else:
      return True

  _internal_getitem = collections.OrderedDict.__getitem__

  class _ValuesView(collections.ValuesView):
    def __contains__(self, value):
      for key in self._mapping:
        if value == self._mapping._internal_getitem(key):
          return True
      return False

    def __iter__(self):
      for key in self._mapping:
        yield self._mapping._internal_getitem(key)

  class _ItemsView(collections.ItemsView):
    def __contains__(self, item):
      key, value = item
      try:
        v = self._mapping._internal_getitem(key)
      except KeyError:
        return False
      else:
        return v == value

    def __iter__(self):
      for key in self._mapping:
        yield (key, self._mapping._internal_getitem(key))

  def values(self):
    return self._ValuesView(self)

  def items(self):
    return self._ItemsView(self)

cache = Cache(nslots=100)
<|endoftext|>


<commit_before>"""
hydrofunctions.usgs_rdb
~~~~~~~~~~~~~~~~~~~~~~~

This module is for working with the various USGS dataservices that use the rdb
text format. These include the statistics service, the field measurements
service, the rating curve service, and the peak discharge service.
"""
import pandas as pd
import requests
from io import StringIO
from IPython.core import display

class hydroRDB:
    """A class for holding the information from USGS rdb files.

    Args:
        header (str):
            A multi-line string from the header of the rdb file. The header
            often contain important metadata and user warnings.
        table (pandas dataframe):
            This is a dataframe made from the rdb file.
        columns (str):
            A string from the rdb file that lists the column names.
        dtypes (str):
            A string from the rdb file that gives the data type and length of
            each column.

    Note:
        The args to create this object are supplied by hf.read_rdb().
    """
    def __init__(self, header, table, columns, dtypes):
        self.header = header
        self.table = table
        self.columns = columns
        self.dtypes = dtypes

    def __repr__(self):
        return f'hydroRDB(header={self.header},\ntable={self.table}'

    def _repr_html_(self):
        html_header = '
' + self.header.replace('\n', '
') + '
'
        #return html_header + self.df._repr_html_()
        return f'
hydroRDB(header=
{html_header}

table=
{self.table._repr_html_()})
'

def read_rdb(text):
    """Read strings that are in rdb format.

    Args:
        text (str):
            A long string containing the contents of a rdb file. A common way
            to obtain these would be from the .text property of a requests
            response, as in the example usage below.
    Returns:
        header (list of strings):
            Every commented line at the top of the rdb file is marked with a
            '#' symbol. Each of these lines is stored in this output.
        outputDF (pandas.DataFrame):
            A dataframe containing the information in the rdb file. 'site_no'
            and parameter_cd
            are interpreted as a string, but every other number is interpreted
            as a float or int; missing values as an np.nan; strings for
            everything else.
        columns (list of strings):
            The column names, taken from the rdb header row.
        dtypes (list of strings):
            The second header row from the rdb file. These mostly tell the
            column width, and typically record everything as string data ('s')
            type. The exception to this are dates, which are listed with a 'd'.
    """
    headerlines = []
    datalines = []
    count = 0
    for line in text.splitlines():
        if line[0] == '#':
            headerlines.append(line)
        elif count == 0:
            columns = line.split()
            count = count + 1
        elif count == 1:
            dtypes = line.split()
            count = count + 1
        else:
            datalines.append(line)
    data = "\n".join(datalines)
    header = "\n".join(headerlines)

    outputDF = pd.read_csv(StringIO(data),
                           sep='\t',
                           comment='#',
                           header=None,
                           names=columns,
                           dtype={'site_no': str, 'parameter_cd': str},
                           )

    #outputDF.outputDF.filter(like='_cd').columns
    #TODO: code columns ('*._cd') should be interpreted as strings.

    return header, outputDF, columns, dtypes

def field_meas(site):
    """Load USGS field measurements of stream discharge into a Pandas dataframe

    Args:
        site (str):
            The gage ID number for the site.

    Returns:
        a dataframe. Each row represents an observation on a given date of
        river conditions at the gage by USGS personnel. Values are stored in
        columns, and include the measured stream discharge, channel width,
        channel area, depth, and velocity. These data can be used to create a
        rating curve, to estimate the gage height for a discharge of zero, and
        to get readings of water velocity.

    NOTES:
        To plot a rating curve, use:
            `output.plot(x='gage_height_va', y='discharge_va', kind='scatter')`

        Rating curves are typically plotted with the indepedent variable,
        gage_height, plotted on the Y axis.

    Discussion:
        The USGS operates over 8,000 stream gages around the United States and
        territories. Each of these sensors records the depth, or 'stage' of the
        water. In order to translate this stage data into stream discharge, the
        USGS staff creates an empirical relationship called a 'rating curve'
        between the river stage and stream discharge. To construct this curve,
        the USGS personnel visit all of the gage every one to eight weeks, and
        measure the stage and the discharge of the river manually.

        The `field_meas()` function returns all of the field-collected data for
        this site. You can use these data to create your own rating curve or to
        read the notes about local conditions.

        The `rating_curve()` function returns the most recent 'expanded shift-
        adjusted' rating curve constructed for this site.

    """
    url = 'https://waterdata.usgs.gov/pa/nwis/measurements?site_no=' + site + \
          '&agency_cd=USGS&format=rdb_expanded'
    headers = {'Accept-encoding': 'gzip'}

    print("Retrieving field measurements for site #", site, " from ", url)
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    # It may be desireable to keep the original na_values, like 'unkn' for many
    # of the columns. However, it is still a good idea to replace for the gage
    # depth and discharge values, since these variables get used in plotting
    # functions.
    header, outputDF, columns, dtype, = read_rdb(response.text)
    outputDF.measurement_dt = pd.to_datetime(outputDF.measurement_dt)

    # An attempt to use the tz_cd column to make measurement_dt timezone aware.
    #outputDF.tz_cd.replace({np.nan: 'UTC'}, inplace=True)
    #def f(x, y):
    #    return x.tz_localize(y)
    #outputDF['datetime'] = outputDF[['measurement_dt', 'tz_cd']].apply(lambda x: f(*x), axis=1)

    outputDF.set_index('measurement_dt', inplace=True)

    output_obj = hydroRDB(header, outputDF, columns, dtype)

    return output_obj

def peaks(site):
    """Return a series of annual peak discharges.

    Args:
        site(str):
            The gauge ID number for the site.

    Returns:
        a dataframe with the annual peak discharge series.

        a header of meta-data supplied by the USGS with the data series.
    """
    url = 'https://nwis.waterdata.usgs.gov/nwis/peak?site_no=' + site + \
    '&agency_cd=USGS&format=rdb'

    headers = {'Accept-encoding': 'gzip'}

    print("Retrieving annual peak discharges for site #", site, " from ", url)
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    header, outputDF, columns, dtype = read_rdb(response.text)
    outputDF.peak_dt = pd.to_datetime(outputDF.peak_dt)

    outputDF.set_index('peak_dt', inplace=True)

    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj

def rating_curve(site):
    """Return the most recent USGS expanded-shift-adjusted rating curve for a
    given stream gage into a dataframe.

    Args:
        site (str):
            The gage ID number for the site.

    Returns:
        a dataframe with the most recent rating curve.

    Note:
        Rating curves change over time
    """
    url = "https://waterdata.usgs.gov/nwisweb/data/ratings/exsa/USGS." + \
          site + ".exsa.rdb"
    headers = {'Accept-encoding': 'gzip'}
    print("Retrieving rating curve for site #", site, " from ", url)
    response = requests.get(url, headers=headers)

    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    header, outputDF, columns, dtype = read_rdb(response.text)
    outputDF.columns = ['stage', 'shift', 'discharge', 'stor']
    """
    outputDF = pd.read_csv(StringIO(response.text),
                     sep='\t',
                     comment='#',
                     header=1,
                     names=['stage', 'shift', 'discharge', 'stor'],
                     skiprows=2
                     )
    """
    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj

def stats(site, statReportType='daily', **kwargs):
    """Return statistics from the USGS Stats Service as a dataframe.

    Args:
        site (str):
            The gage ID number for the site, or a series of gage IDs separated
            by commas, like this: '01546500,01548000'.

        statReportType ('annual'|'monthly'|'daily'):
            There are three different types of report that you can request.
            - 'daily' (default): this

    Returns:
        a dataframe with the requested statistics.

    Raises:
        HTTPError when a non-200 http status code is returned.

    Note:
        This function is based on the USGS statistics service, described here:
        https://waterservices.usgs.gov/rest/Statistics-Service.html

        The USGS Statistics Service allows you to specify a wide array of
        additional parameters in your request. You can provide these parameters
        as keyword arguments, like in this example:
        `hf.stats('01452500', parameterCD='00060')`  This will only request
        statistics for discharge, which is specified with the '00060'
        parameter code.

        Additional useful parameters include:

            - `parameterCD='00060,00065'` Limit the request for statistics to
              only one parameter or to a list of parameters. The default behavior
              is to provide statistics for every parameter that has been measured
              at this site. In this example, statistics for discharge ('00060')
              and stage ('00065') are requested.

            - `statYearType='water'` Calculate annual statistics based on the
              water year, which runs from October 1st to September 31st. This
              parameter is only for use with annual reports. If not specified,
              the default behavior will use calendar years for reporting.

            - `missingData='on'`  Calculate statistics even when there are some
              missing values. If not specified, the default behavior is to drop
              years that have fewer than 365 values from annual reports, and to
              drop months that have fewer than 30 values in monthly reports. The
              number of values used to calculate a statistic is reported in the
              'count_nu' column.

            - You can read about other useful parameters here: https://waterservices.usgs.gov/rest/Statistics-Service.html#statistical_Controls

    """
    url = "https://waterservices.usgs.gov/nwis/stat/"

    headers = {
            'Accept-encoding': 'gzip',
            }
    # Set default values for parameters that are too obscure to put into call
    # signature.
    params = {
            'statReportType': statReportType,
            'statType': 'all',
            'sites': site,
            'format': 'rdb',
            }
    # Overwrite defaults if they are specified.
    params.update(kwargs)

    response = requests.get(url, headers=headers, params=params)
    print(f"Retrieving {params['statReportType']} statistics for site #{params['sites']} from {response.url}")

    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    else:
        header, outputDF, columns, dtype = read_rdb(response.text)

    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj
<commit_msg>Make hydroRDB return a tuple<commit_after>"""
hydrofunctions.usgs_rdb
~~~~~~~~~~~~~~~~~~~~~~~

This module is for working with the various USGS dataservices that use the rdb
text format. These include the statistics service, the field measurements
service, the rating curve service, and the peak discharge service.
"""
import pandas as pd
import requests
from io import StringIO
from IPython.core import display

class hydroRDB:
    """A class for holding the information from USGS rdb files.

    Args:
        header (str):
            A multi-line string from the header of the rdb file. The header
            often contain important metadata and user warnings.
        table (pandas dataframe):
            This is a dataframe made from the rdb file.
        columns (str):
            A string from the rdb file that lists the column names.
        dtypes (str):
            A string from the rdb file that gives the data type and length of
            each column.

    Note:
        The args to create this object are supplied by hf.read_rdb().
    """
    def __init__(self, header, table, columns, dtypes):
        self.header = header
        self.table = table
        self.columns = columns
        self.dtypes = dtypes

    def __iter__(self):
        return iter((self.header, self.table))

    def __repr__(self):
        return f'hydroRDB(header={self.header},\ntable={self.table}'

    def _repr_html_(self):
        html_header = '
' + self.header.replace('\n', '
') + '
'
        #return html_header + self.df._repr_html_()
        return f'
hydroRDB(header=
{html_header}

table=
{self.table._repr_html_()})
'

def read_rdb(text):
    """Read strings that are in rdb format.

    Args:
        text (str):
            A long string containing the contents of a rdb file. A common way
            to obtain these would be from the .text property of a requests
            response, as in the example usage below.
    Returns:
        header (list of strings):
            Every commented line at the top of the rdb file is marked with a
            '#' symbol. Each of these lines is stored in this output.
        outputDF (pandas.DataFrame):
            A dataframe containing the information in the rdb file. 'site_no'
            and parameter_cd
            are interpreted as a string, but every other number is interpreted
            as a float or int; missing values as an np.nan; strings for
            everything else.
        columns (list of strings):
            The column names, taken from the rdb header row.
        dtypes (list of strings):
            The second header row from the rdb file. These mostly tell the
            column width, and typically record everything as string data ('s')
            type. The exception to this are dates, which are listed with a 'd'.
    """
    headerlines = []
    datalines = []
    count = 0
    for line in text.splitlines():
        if line[0] == '#':
            headerlines.append(line)
        elif count == 0:
            columns = line.split()
            count = count + 1
        elif count == 1:
            dtypes = line.split()
            count = count + 1
        else:
            datalines.append(line)
    data = "\n".join(datalines)
    header = "\n".join(headerlines)

    outputDF = pd.read_csv(StringIO(data),
                           sep='\t',
                           comment='#',
                           header=None,
                           names=columns,
                           dtype={'site_no': str, 'parameter_cd': str},
                           )

    #outputDF.outputDF.filter(like='_cd').columns
    #TODO: code columns ('*._cd') should be interpreted as strings.

    return header, outputDF, columns, dtypes

def field_meas(site):
    """Load USGS field measurements of stream discharge into a Pandas dataframe

    Args:
        site (str):
            The gage ID number for the site.

    Returns:
        a dataframe. Each row represents an observation on a given date of
        river conditions at the gage by USGS personnel. Values are stored in
        columns, and include the measured stream discharge, channel width,
        channel area, depth, and velocity. These data can be used to create a
        rating curve, to estimate the gage height for a discharge of zero, and
        to get readings of water velocity.

    NOTES:
        To plot a rating curve, use:
            `output.plot(x='gage_height_va', y='discharge_va', kind='scatter')`

        Rating curves are typically plotted with the indepedent variable,
        gage_height, plotted on the Y axis.

    Discussion:
        The USGS operates over 8,000 stream gages around the United States and
        territories. Each of these sensors records the depth, or 'stage' of the
        water. In order to translate this stage data into stream discharge, the
        USGS staff creates an empirical relationship called a 'rating curve'
        between the river stage and stream discharge. To construct this curve,
        the USGS personnel visit all of the gage every one to eight weeks, and
        measure the stage and the discharge of the river manually.

        The `field_meas()` function returns all of the field-collected data for
        this site. You can use these data to create your own rating curve or to
        read the notes about local conditions.

        The `rating_curve()` function returns the most recent 'expanded shift-
        adjusted' rating curve constructed for this site.

    """
    url = 'https://waterdata.usgs.gov/pa/nwis/measurements?site_no=' + site + \
          '&agency_cd=USGS&format=rdb_expanded'
    headers = {'Accept-encoding': 'gzip'}

    print("Retrieving field measurements for site #", site, " from ", url)
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    # It may be desireable to keep the original na_values, like 'unkn' for many
    # of the columns. However, it is still a good idea to replace for the gage
    # depth and discharge values, since these variables get used in plotting
    # functions.
    header, outputDF, columns, dtype, = read_rdb(response.text)
    outputDF.measurement_dt = pd.to_datetime(outputDF.measurement_dt)

    # An attempt to use the tz_cd column to make measurement_dt timezone aware.
    #outputDF.tz_cd.replace({np.nan: 'UTC'}, inplace=True)
    #def f(x, y):
    #    return x.tz_localize(y)
    #outputDF['datetime'] = outputDF[['measurement_dt', 'tz_cd']].apply(lambda x: f(*x), axis=1)

    outputDF.set_index('measurement_dt', inplace=True)

    output_obj = hydroRDB(header, outputDF, columns, dtype)

    return output_obj

def peaks(site):
    """Return a series of annual peak discharges.

    Args:
        site(str):
            The gauge ID number for the site.

    Returns:
        a dataframe with the annual peak discharge series.

        a header of meta-data supplied by the USGS with the data series.
    """
    url = 'https://nwis.waterdata.usgs.gov/nwis/peak?site_no=' + site + \
    '&agency_cd=USGS&format=rdb'

    headers = {'Accept-encoding': 'gzip'}

    print("Retrieving annual peak discharges for site #", site, " from ", url)
    response = requests.get(url, headers=headers)
    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    header, outputDF, columns, dtype = read_rdb(response.text)
    outputDF.peak_dt = pd.to_datetime(outputDF.peak_dt)

    outputDF.set_index('peak_dt', inplace=True)

    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj

def rating_curve(site):
    """Return the most recent USGS expanded-shift-adjusted rating curve for a
    given stream gage into a dataframe.

    Args:
        site (str):
            The gage ID number for the site.

    Returns:
        a dataframe with the most recent rating curve.

    Note:
        Rating curves change over time
    """
    url = "https://waterdata.usgs.gov/nwisweb/data/ratings/exsa/USGS." + \
          site + ".exsa.rdb"
    headers = {'Accept-encoding': 'gzip'}
    print("Retrieving rating curve for site #", site, " from ", url)
    response = requests.get(url, headers=headers)

    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    header, outputDF, columns, dtype = read_rdb(response.text)
    outputDF.columns = ['stage', 'shift', 'discharge', 'stor']
    """
    outputDF = pd.read_csv(StringIO(response.text),
                     sep='\t',
                     comment='#',
                     header=1,
                     names=['stage', 'shift', 'discharge', 'stor'],
                     skiprows=2
                     )
    """
    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj

def stats(site, statReportType='daily', **kwargs):
    """Return statistics from the USGS Stats Service as a dataframe.

    Args:
        site (str):
            The gage ID number for the site, or a series of gage IDs separated
            by commas, like this: '01546500,01548000'.

        statReportType ('annual'|'monthly'|'daily'):
            There are three different types of report that you can request.
            - 'daily' (default): this

    Returns:
        a dataframe with the requested statistics.

    Raises:
        HTTPError when a non-200 http status code is returned.

    Note:
        This function is based on the USGS statistics service, described here:
        https://waterservices.usgs.gov/rest/Statistics-Service.html

        The USGS Statistics Service allows you to specify a wide array of
        additional parameters in your request. You can provide these parameters
        as keyword arguments, like in this example:
        `hf.stats('01452500', parameterCD='00060')`  This will only request
        statistics for discharge, which is specified with the '00060'
        parameter code.

        Additional useful parameters include:

            - `parameterCD='00060,00065'` Limit the request for statistics to
              only one parameter or to a list of parameters. The default behavior
              is to provide statistics for every parameter that has been measured
              at this site. In this example, statistics for discharge ('00060')
              and stage ('00065') are requested.

            - `statYearType='water'` Calculate annual statistics based on the
              water year, which runs from October 1st to September 31st. This
              parameter is only for use with annual reports. If not specified,
              the default behavior will use calendar years for reporting.

            - `missingData='on'`  Calculate statistics even when there are some
              missing values. If not specified, the default behavior is to drop
              years that have fewer than 365 values from annual reports, and to
              drop months that have fewer than 30 values in monthly reports. The
              number of values used to calculate a statistic is reported in the
              'count_nu' column.

            - You can read about other useful parameters here: https://waterservices.usgs.gov/rest/Statistics-Service.html#statistical_Controls

    """
    url = "https://waterservices.usgs.gov/nwis/stat/"

    headers = {
            'Accept-encoding': 'gzip',
            }
    # Set default values for parameters that are too obscure to put into call
    # signature.
    params = {
            'statReportType': statReportType,
            'statType': 'all',
            'sites': site,
            'format': 'rdb',
            }
    # Overwrite defaults if they are specified.
    params.update(kwargs)

    response = requests.get(url, headers=headers, params=params)
    print(f"Retrieving {params['statReportType']} statistics for site #{params['sites']} from {response.url}")

    if response.status_code != 200:
        print(f'The USGS has returned an error code of {response.status_code}')
        # If this code is being run inside of a notebook, the USGS error page
        # will be displayed.
        display.display(display.HTML(response.text))
        # raise an exception
        response.raise_for_status()
        # or raise some sort of Hydro http error based on requests http error.
        return response
    else:
        header, outputDF, columns, dtype = read_rdb(response.text)

    output_obj = hydroRDB(header, outputDF, columns, dtype)
    return output_obj
<|endoftext|>


<commit_before>import django_filters

from rest_framework import viewsets, permissions, response, decorators, filters, exceptions

from django.conf import settings
from django.views.static import serve
from nodeconductor.core.filters import DjangoMappingFilterBackend

from nodeconductor.structure.filters import GenericRoleFilter
from nodeconductor.billing.serializers import InvoiceSerializer, InvoiceDetailedSerializer
from nodeconductor.billing.models import PriceList, Invoice

class BillingViewSet(viewsets.GenericViewSet):

    @decorators.list_route()
    def pricelist(self, request):
        return response.Response({pl.name: pl.price for pl in PriceList.objects.all()})

class InvoiceFilter(django_filters.FilterSet):
    customer = django_filters.CharFilter(name='customer__uuid', distinct=True)
    customer_name = django_filters.CharFilter(
        name='customer__name',
        lookup_type='icontains',
        distinct=True
    )
    customer_native_name = django_filters.CharFilter(
        name='customer__native_name',
        lookup_type='icontains',
        distinct=True
    )
    customer_abbreviation = django_filters.CharFilter(
        name='customer__abbreviation',
        lookup_type='icontains',
        distinct=True
    )
    month = django_filters.CharFilter(name='date', lookup_type='month')
    year = django_filters.CharFilter(name='date', lookup_type='year')

    class Meta(object):
        model = Invoice
        fields = [
            'customer', 'customer_name', 'customer_native_name', 'customer_abbreviation',
            'year', 'month',
            'amount',
            'status',
        ]
        order_by = [
            'date',
            '-date',
            'amount',
            '-amount',
            'status',
            '-status',
            'customer__name',
            '-customer__name',
            'customer__abbreviation',
            '-customer__abbreviation',
            'customer__native_name',
            '-customer__native_name',
        ]
        order_by_mapping = {
            # Proper field naming
            'customer_name': 'customer__name',
            'customer_abbreviation': 'customer__abbreviation',
            'customer_native_name': 'customer__native_name',
        }

class InvoiceViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Invoice.objects.all()
    filter_class = InvoiceFilter
    filter_backends = (GenericRoleFilter, DjangoMappingFilterBackend)
    lookup_field = 'uuid'
    permission_classes = (
        permissions.IsAuthenticated,
        permissions.DjangoObjectPermissions,
    )

    # Invoice items are being fetched directly from backend
    # thus we expose them in detailed view only
    # TODO: Move invoice items to DB and use single serializer
    def get_serializer_class(self):
        return InvoiceDetailedSerializer if self.action == 'retrieve' else InvoiceSerializer

    @decorators.detail_route()
    def pdf(self, request, uuid=None):
        invoice = self.get_object()
        if not invoice.pdf:
            raise exceptions.NotFound("There's no PDF for this invoice")

        response = serve(request, invoice.pdf.name, document_root=settings.MEDIA_ROOT)
        if request.query_params.get('download'):
            response['Content-Type'] = 'application/pdf'
            response['Content-Disposition'] = 'attachment; filename="invoice.pdf"'

        return response
<commit_msg>Extend invoice view with filter by date<commit_after>import django_filters

from rest_framework import viewsets, permissions, response, decorators, filters, exceptions

from django.conf import settings
from django.views.static import serve
from nodeconductor.core.filters import DjangoMappingFilterBackend

from nodeconductor.structure.filters import GenericRoleFilter
from nodeconductor.billing.serializers import InvoiceSerializer, InvoiceDetailedSerializer
from nodeconductor.billing.models import PriceList, Invoice

class BillingViewSet(viewsets.GenericViewSet):

    @decorators.list_route()
    def pricelist(self, request):
        return response.Response({pl.name: pl.price for pl in PriceList.objects.all()})

class InvoiceFilter(django_filters.FilterSet):
    customer = django_filters.CharFilter(name='customer__uuid', distinct=True)
    customer_name = django_filters.CharFilter(
        name='customer__name',
        lookup_type='icontains',
        distinct=True
    )
    customer_native_name = django_filters.CharFilter(
        name='customer__native_name',
        lookup_type='icontains',
        distinct=True
    )
    customer_abbreviation = django_filters.CharFilter(
        name='customer__abbreviation',
        lookup_type='icontains',
        distinct=True
    )
    month = django_filters.CharFilter(name='date', lookup_type='month')
    year = django_filters.CharFilter(name='date', lookup_type='year')

    class Meta(object):
        model = Invoice
        fields = [
            'customer', 'customer_name', 'customer_native_name', 'customer_abbreviation',
            'year', 'month',
            'amount',
            'status',
            'date',
        ]
        order_by = [
            'date',
            '-date',
            'amount',
            '-amount',
            'status',
            '-status',
            'customer__name',
            '-customer__name',
            'customer__abbreviation',
            '-customer__abbreviation',
            'customer__native_name',
            '-customer__native_name',
        ]
        order_by_mapping = {
            # Proper field naming
            'customer_name': 'customer__name',
            'customer_abbreviation': 'customer__abbreviation',
            'customer_native_name': 'customer__native_name',
        }

class InvoiceViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Invoice.objects.all()
    filter_class = InvoiceFilter
    filter_backends = (GenericRoleFilter, DjangoMappingFilterBackend)
    lookup_field = 'uuid'
    permission_classes = (
        permissions.IsAuthenticated,
        permissions.DjangoObjectPermissions,
    )

    # Invoice items are being fetched directly from backend
    # thus we expose them in detailed view only
    # TODO: Move invoice items to DB and use single serializer
    def get_serializer_class(self):
        return InvoiceDetailedSerializer if self.action == 'retrieve' else InvoiceSerializer

    @decorators.detail_route()
    def pdf(self, request, uuid=None):
        invoice = self.get_object()
        if not invoice.pdf:
            raise exceptions.NotFound("There's no PDF for this invoice")

        response = serve(request, invoice.pdf.name, document_root=settings.MEDIA_ROOT)
        if request.query_params.get('download'):
            response['Content-Type'] = 'application/pdf'
            response['Content-Disposition'] = 'attachment; filename="invoice.pdf"'

        return response
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
This module provides a test runner for setup.py copied over from
https://bitbucket.org/pytest-dev/pytest-runner/
in order to make some improvements.

MIT License:

Copyright (c) 2011 Jason R. Coombs <jaraco@jaraco.com>

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

from __future__ import division, print_function, absolute_import

import sys
import os
import shlex

import pkg_resources
from setuptools.command.test import test as TestCommand

__author__ = "Jason R. Coombs, Florian Wilhelm"
__copyright__ = "Jason R. Coombs"
__license__ = "MIT"

class PyTest(TestCommand):
    user_options = [
        ('extras', None, "Install (all) setuptools extras when running tests"),
        ('index-url=', None, "Specify an index url from which to retrieve "
            "dependencies"),
        ('allow-hosts=', None, "Whitelist of comma-separated hosts to allow "
            "when retrieving dependencies"),
        ('addopts=', 'a', "Additional options to be passed verbatim to the "
            "pytest runner")
    ]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.extras = False
        self.index_url = None
        self.allow_hosts = None
        self.addopts = []

    def finalize_options(self):
        if self.addopts:
            self.addopts = shlex.split(self.addopts)

    @staticmethod
    def marker_passes(marker):
        """
        Given an environment marker, return True if the marker is valid
        and matches this environment.
        """
        return (
            marker
            and not pkg_resources.invalid_marker(marker)
            and pkg_resources.evaluate_marker(marker)
        )

    def run(self):
        """
        Override run to ensure requirements are available in this session (but
        don't install them anywhere).
        """
        self._build_egg_fetcher()
        if self.distribution.install_requires:
            self.distribution.fetch_build_eggs(self.distribution.install_requires)
        if self.distribution.tests_require:
            self.distribution.fetch_build_eggs(self.distribution.tests_require)
        extras_require = self.distribution.extras_require or {}
        for spec, reqs in extras_require.items():
            name, sep, marker = spec.partition(':')
            if marker and not self.marker_passes(marker):
                continue
            # always include unnamed extras
            if not name or self.extras:
                self.distribution.fetch_build_eggs(reqs)
        if self.dry_run:
            self.announce('skipping tests (dry run)')
            return
        self.with_project_on_sys_path(self.run_tests)
        if self.result_code:
            raise SystemExit(self.result_code)
        return self.result_code

    def _build_egg_fetcher(self):
        """Build an egg fetcher that respects index_url and allow_hosts"""
        # modified from setuptools.dist:Distribution.fetch_build_egg
        from setuptools.command.easy_install import easy_install
        main_dist = self.distribution
        # construct a fake distribution to store the args for easy_install
        dist = main_dist.__class__({'script_args': ['easy_install']})
        dist.parse_config_files()
        opts = dist.get_option_dict('easy_install')
        keep = (
            'find_links', 'site_dirs', 'index_url', 'optimize',
            'site_dirs', 'allow_hosts'
        )
        for key in opts.keys():
            if key not in keep:
                del opts[key]   # don't use any other settings
        if main_dist.dependency_links:
            links = main_dist.dependency_links[:]
            if 'find_links' in opts:
                links = opts['find_links'][1].split() + links
            opts['find_links'] = ('setup', links)
        if self.allow_hosts:
            opts['allow_hosts'] = ('test', self.allow_hosts)
        if self.index_url:
            opts['index_url'] = ('test', self.index_url)
        install_dir_func = getattr(dist, 'get_egg_cache_dir', os.getcwd)
        install_dir = install_dir_func()
        cmd = easy_install(
            dist, args=["x"], install_dir=install_dir, exclude_scripts=True,
            always_copy=False, build_directory=None, editable=False,
            upgrade=False, multi_version=True, no_report = True
        )
        cmd.ensure_finalized()
        main_dist._egg_fetcher = cmd

    def run_tests(self):
        import pytest
        errno = pytest.main(self.addopts)
        sys.exit(errno)
<commit_msg>Added comment to pytest_runner.py<commit_after># -*- coding: utf-8 -*-

"""
This module provides a test runner for setup.py copied over from
https://bitbucket.org/pytest-dev/pytest-runner/
in order to make some improvements.

Follow the discussion under:
https://bitbucket.org/pytest-dev/pytest-runner/issues/7/support-all-pytest-commands

This file is MIT licensed:

Copyright (c) 2011 Jason R. Coombs <jaraco@jaraco.com>

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

from __future__ import division, print_function, absolute_import

import sys
import os
import shlex

import pkg_resources
from setuptools.command.test import test as TestCommand

__author__ = "Jason R. Coombs, Florian Wilhelm"
__copyright__ = "Jason R. Coombs"
__license__ = "MIT"

class PyTest(TestCommand):
    user_options = [
        ('extras', None, "Install (all) setuptools extras when running tests"),
        ('index-url=', None, "Specify an index url from which to retrieve "
            "dependencies"),
        ('allow-hosts=', None, "Whitelist of comma-separated hosts to allow "
            "when retrieving dependencies"),
        ('addopts=', 'a', "Additional options to be passed verbatim to the "
            "pytest runner")
    ]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        self.extras = False
        self.index_url = None
        self.allow_hosts = None
        self.addopts = []

    def finalize_options(self):
        if self.addopts:
            self.addopts = shlex.split(self.addopts)

    @staticmethod
    def marker_passes(marker):
        """
        Given an environment marker, return True if the marker is valid
        and matches this environment.
        """
        return (
            marker
            and not pkg_resources.invalid_marker(marker)
            and pkg_resources.evaluate_marker(marker)
        )

    def run(self):
        """
        Override run to ensure requirements are available in this session (but
        don't install them anywhere).
        """
        self._build_egg_fetcher()
        if self.distribution.install_requires:
            self.distribution.fetch_build_eggs(self.distribution.install_requires)
        if self.distribution.tests_require:
            self.distribution.fetch_build_eggs(self.distribution.tests_require)
        extras_require = self.distribution.extras_require or {}
        for spec, reqs in extras_require.items():
            name, sep, marker = spec.partition(':')
            if marker and not self.marker_passes(marker):
                continue
            # always include unnamed extras
            if not name or self.extras:
                self.distribution.fetch_build_eggs(reqs)
        if self.dry_run:
            self.announce('skipping tests (dry run)')
            return
        self.with_project_on_sys_path(self.run_tests)
        if self.result_code:
            raise SystemExit(self.result_code)
        return self.result_code

    def _build_egg_fetcher(self):
        """Build an egg fetcher that respects index_url and allow_hosts"""
        # modified from setuptools.dist:Distribution.fetch_build_egg
        from setuptools.command.easy_install import easy_install
        main_dist = self.distribution
        # construct a fake distribution to store the args for easy_install
        dist = main_dist.__class__({'script_args': ['easy_install']})
        dist.parse_config_files()
        opts = dist.get_option_dict('easy_install')
        keep = (
            'find_links', 'site_dirs', 'index_url', 'optimize',
            'site_dirs', 'allow_hosts'
        )
        for key in opts.keys():
            if key not in keep:
                del opts[key]   # don't use any other settings
        if main_dist.dependency_links:
            links = main_dist.dependency_links[:]
            if 'find_links' in opts:
                links = opts['find_links'][1].split() + links
            opts['find_links'] = ('setup', links)
        if self.allow_hosts:
            opts['allow_hosts'] = ('test', self.allow_hosts)
        if self.index_url:
            opts['index_url'] = ('test', self.index_url)
        install_dir_func = getattr(dist, 'get_egg_cache_dir', os.getcwd)
        install_dir = install_dir_func()
        cmd = easy_install(
            dist, args=["x"], install_dir=install_dir, exclude_scripts=True,
            always_copy=False, build_directory=None, editable=False,
            upgrade=False, multi_version=True, no_report = True
        )
        cmd.ensure_finalized()
        main_dist._egg_fetcher = cmd

    def run_tests(self):
        import pytest
        errno = pytest.main(self.addopts)
        sys.exit(errno)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
hyper/http20/connection
~~~~~~~~~~~~~~~~~~~~~~~

Objects that build hyper's connection-level HTTP/2.0 abstraction.
"""
from .hpack import Encoder, Decoder
from .stream import Stream
from .tls import wrap_socket
from .frame import DataFrame, HeadersFrame, SettingsFrame, Frame

import socket

class HTTP20Connection(object):
    """
    An object representing a single HTTP/2.0 connection to a server.

    This object behaves similarly to the Python standard library's
    HTTPConnection object, with a few critical differences.
    """
    def __init__(self, host, port=None, **kwargs):
        """
        Creates an HTTP/2.0 connection to a specific server.

        Most of the standard library's arguments to the constructor are
        irrelevant for HTTP/2.0 or not supported by hyper.
        """
        if port is None:
            try:
                self.host, self.port = host.split(':')
                self.port = int(self.port)
            except ValueError:
                self.host, self.port = host, 443
        else:
            self.host, self.port = host, port

        # Streams are stored in a dictionary keyed off their stream IDs. We
        # also save the most recent one for easy access without having to walk
        # the dictionary.
        self.streams = {}
        self.recent_stream = None
        self.next_stream_id = 1

        # Header encoding/decoding is at the connection scope, so we embed a
        # header encoder and a decoder. These get passed to child stream
        # objects.
        self.encoder = Encoder()
        self.decoder = Decoder()

        # The socket used to send data.
        self._sock = None

        # The inbound and outbound flow control windows.
        self._out_flow_control_window = 65535
        self._in_flow_control_window = 65535

        return

    def request(self, method, url, body=None, headers={}):
        """
        This will send a request to the server using the HTTP request method
        ``method`` and the selector ``url``. If the ``body`` argument is
        present, it should be string or bytes object of data to send after the
        headers are finished. Strings are encoded as UTF-8. To use other
        encodings, pass a bytes object. The Content-Length header is set to the
        length of the body field.

        Returns a stream ID for the request.
        """
        pass

    def getresponse(self, stream_id=None):
        """
        Should be called after a request is sent to get a response from the
        server. If sending multiple parallel requests, pass the stream ID of
        the request whose response you want. Returns a HTTPResponse instance.
        If you pass no stream_id, you will receive the oldest HTTPResponse
        still outstanding.
        """
        pass

    def connect(self):
        """
        Connect to the server specified when the object was created. This is a
        no-op if we're already connected.
        """
        if self._sock is None:
            sock = socket.create_connection((self.host, self.port), 5)
            sock = wrap_socket(sock, self.host)
            self._sock = sock

            # We need to send a Settings frame immediately on this connection.
            f = SettingsFrame(0)
            f.settings[SettingsFrame.ENABLE_PUSH] = 0
            self._send_cb(f)

        return

    def close(self):
        """
        Close the connection to the server.
        """
        pass

    def putrequest(self, method, selector, **kwargs):
        """
        This should be the first call for sending a given HTTP request to a
        server. It returns a stream ID for the given connection that should be
        passed to all subsequent request building calls.
        """
        # Create a new stream.
        s = self._new_stream()

        # To this stream we need to immediately add a few headers that are
        # HTTP/2.0 specific. These are: ":method", ":scheme", ":authority" and
        # ":path". We can set all of these now.
        s.add_header(":method", method)
        s.add_header(":scheme", "https")  # We only support HTTPS at this time.
        s.add_header(":authority", self.host)
        s.add_header(":path", selector)

        # Save the stream.
        self.streams[s.stream_id] = s
        self.recent_stream = s

        return s.stream_id

    def putheader(self, header, argument, stream_id=None):
        """
        Sends an HTTP header to the server, with name ``header`` and value
        ``argument``.

        Unlike the httplib version of this function, this version does not
        actually send anything when called. Instead, it queues the headers up
        to be sent when you call ``endheaders``.
        """
        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        stream.add_header(header, argument)

        return

    def endheaders(self, message_body=None, final=False, stream_id=None):
        """
        Sends the prepared headers to the server. If the ``message_body``
        argument is provided it will also be sent to the server as the body of
        the request, and the stream will immediately be closed. If the
        ``final`` argument is set to True, the stream will also immediately
        be closed: otherwise, the stream will be left open and subsequent calls
        to ``send()`` will be required.
        """
        self.connect()

        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        # Close this if we've been told no more data is coming and we don't
        # have any to send.
        stream.open(final and message_body is None)

        # Send whatever data we have.
        if message_body is not None:
            stream.send_data(message_body, final)

        return

    def send(self, data, final=False, stream_id=None):
        """
        Sends some data to the server. This data will be sent immediately
        (excluding the normal HTTP/2.0 flow control rules). If this is the last
        data that will be sent as part of this request, the ``final`` argument
        should be set to ``True``. This will cause the stream to be closed.
        """
        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        stream.send_data(data, final)

        return

    def _new_stream(self):
        """
        Returns a new stream object for this connection.
        """
        s = Stream(
            self.next_stream_id, self._send_cb, self._recv_cb, self.encoder,
            self.decoder
        )
        self.next_stream_id += 2

        return s

    def _send_cb(self, frame):
        """
        This is the callback used by streams to send data on the connection.

        It expects to receive a single frame, and then to serialize that frame
        and send it on the connection. It does so obeying the connection-level
        flow-control principles of HTTP/2.0.
        """
        # Maintain our outgoing flow-control window.
        if (isinstance(frame, DataFrame) and
            not isinstance(frame, HeadersFrame)):
            if self._out_flow_control_window < len(frame.data):
                raise RuntimeError("Flow control not yet implemented.")

            self._out_flow_control_window -= len(frame.data)

        data = frame.serialize()

        self._sock.send(data)

    def _recv_cb(self):
        """
        This is the callback used by streams to read data from the connection.

        It expects to read a single frame, and then to deserialize that frame
        and pass it to the relevant stream. This is generally called by a
        stream, not by the connection itself, and it's likely that streams will
        read a frame that doesn't belong to them. That's ok: streams need to
        make a decision to spin around again.
        """
        # Begin by reading 8 bytes from the socket.
        header = self._sock.recv(8)

        # Parse the header.
        frame, length = Frame.parse_frame_header(header)

        # Read the remaining data from the socket.
        data = self._sock.recv(length)
        frame.parse_body(data)

        # Work out to whom this frame should go.
        if frame.stream_id != 0:
            self.streams[frame.stream_id].receive_frame(frame)
        else:
            self.receive_frame(frame)
<commit_msg>Connection-level flow control.<commit_after># -*- coding: utf-8 -*-
"""
hyper/http20/connection
~~~~~~~~~~~~~~~~~~~~~~~

Objects that build hyper's connection-level HTTP/2.0 abstraction.
"""
from .hpack import Encoder, Decoder
from .stream import Stream
from .tls import wrap_socket
from .frame import DataFrame, HeadersFrame, SettingsFrame, Frame

import socket

class HTTP20Connection(object):
    """
    An object representing a single HTTP/2.0 connection to a server.

    This object behaves similarly to the Python standard library's
    HTTPConnection object, with a few critical differences.
    """
    def __init__(self, host, port=None, **kwargs):
        """
        Creates an HTTP/2.0 connection to a specific server.

        Most of the standard library's arguments to the constructor are
        irrelevant for HTTP/2.0 or not supported by hyper.
        """
        if port is None:
            try:
                self.host, self.port = host.split(':')
                self.port = int(self.port)
            except ValueError:
                self.host, self.port = host, 443
        else:
            self.host, self.port = host, port

        # Streams are stored in a dictionary keyed off their stream IDs. We
        # also save the most recent one for easy access without having to walk
        # the dictionary.
        self.streams = {}
        self.recent_stream = None
        self.next_stream_id = 1

        # Header encoding/decoding is at the connection scope, so we embed a
        # header encoder and a decoder. These get passed to child stream
        # objects.
        self.encoder = Encoder()
        self.decoder = Decoder()

        # The socket used to send data.
        self._sock = None

        # The inbound and outbound flow control windows.
        self._out_flow_control_window = 65535
        self._in_flow_control_window = 65535

        return

    def request(self, method, url, body=None, headers={}):
        """
        This will send a request to the server using the HTTP request method
        ``method`` and the selector ``url``. If the ``body`` argument is
        present, it should be string or bytes object of data to send after the
        headers are finished. Strings are encoded as UTF-8. To use other
        encodings, pass a bytes object. The Content-Length header is set to the
        length of the body field.

        Returns a stream ID for the request.
        """
        pass

    def getresponse(self, stream_id=None):
        """
        Should be called after a request is sent to get a response from the
        server. If sending multiple parallel requests, pass the stream ID of
        the request whose response you want. Returns a HTTPResponse instance.
        If you pass no stream_id, you will receive the oldest HTTPResponse
        still outstanding.
        """
        pass

    def connect(self):
        """
        Connect to the server specified when the object was created. This is a
        no-op if we're already connected.
        """
        if self._sock is None:
            sock = socket.create_connection((self.host, self.port), 5)
            sock = wrap_socket(sock, self.host)
            self._sock = sock

            # We need to send a Settings frame immediately on this connection.
            f = SettingsFrame(0)
            f.settings[SettingsFrame.ENABLE_PUSH] = 0
            self._send_cb(f)

        return

    def close(self):
        """
        Close the connection to the server.
        """
        pass

    def putrequest(self, method, selector, **kwargs):
        """
        This should be the first call for sending a given HTTP request to a
        server. It returns a stream ID for the given connection that should be
        passed to all subsequent request building calls.
        """
        # Create a new stream.
        s = self._new_stream()

        # To this stream we need to immediately add a few headers that are
        # HTTP/2.0 specific. These are: ":method", ":scheme", ":authority" and
        # ":path". We can set all of these now.
        s.add_header(":method", method)
        s.add_header(":scheme", "https")  # We only support HTTPS at this time.
        s.add_header(":authority", self.host)
        s.add_header(":path", selector)

        # Save the stream.
        self.streams[s.stream_id] = s
        self.recent_stream = s

        return s.stream_id

    def putheader(self, header, argument, stream_id=None):
        """
        Sends an HTTP header to the server, with name ``header`` and value
        ``argument``.

        Unlike the httplib version of this function, this version does not
        actually send anything when called. Instead, it queues the headers up
        to be sent when you call ``endheaders``.
        """
        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        stream.add_header(header, argument)

        return

    def endheaders(self, message_body=None, final=False, stream_id=None):
        """
        Sends the prepared headers to the server. If the ``message_body``
        argument is provided it will also be sent to the server as the body of
        the request, and the stream will immediately be closed. If the
        ``final`` argument is set to True, the stream will also immediately
        be closed: otherwise, the stream will be left open and subsequent calls
        to ``send()`` will be required.
        """
        self.connect()

        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        # Close this if we've been told no more data is coming and we don't
        # have any to send.
        stream.open(final and message_body is None)

        # Send whatever data we have.
        if message_body is not None:
            stream.send_data(message_body, final)

        return

    def send(self, data, final=False, stream_id=None):
        """
        Sends some data to the server. This data will be sent immediately
        (excluding the normal HTTP/2.0 flow control rules). If this is the last
        data that will be sent as part of this request, the ``final`` argument
        should be set to ``True``. This will cause the stream to be closed.
        """
        stream = (self.streams[stream_id] if stream_id is not None
                  else self.recent_stream)

        stream.send_data(data, final)

        return

    def _new_stream(self):
        """
        Returns a new stream object for this connection.
        """
        s = Stream(
            self.next_stream_id, self._send_cb, self._recv_cb, self.encoder,
            self.decoder
        )
        self.next_stream_id += 2

        return s

    def _send_cb(self, frame):
        """
        This is the callback used by streams to send data on the connection.

        It expects to receive a single frame, and then to serialize that frame
        and send it on the connection. It does so obeying the connection-level
        flow-control principles of HTTP/2.0.
        """
        # Maintain our outgoing flow-control window.
        if (isinstance(frame, DataFrame) and
            not isinstance(frame, HeadersFrame)):

            # If we don't have room in the flow control window, we need to look
            # for a Window Update frame.
            while self._out_flow_control_window < len(frame.data):
                self._recv_cb()

            self._out_flow_control_window -= len(frame.data)

        data = frame.serialize()

        self._sock.send(data)

    def _recv_cb(self):
        """
        This is the callback used by streams to read data from the connection.

        It expects to read a single frame, and then to deserialize that frame
        and pass it to the relevant stream. This is generally called by a
        stream, not by the connection itself, and it's likely that streams will
        read a frame that doesn't belong to them. That's ok: streams need to
        make a decision to spin around again.
        """
        # Begin by reading 8 bytes from the socket.
        header = self._sock.recv(8)

        # Parse the header.
        frame, length = Frame.parse_frame_header(header)

        # Read the remaining data from the socket.
        data = self._sock.recv(length)
        frame.parse_body(data)

        # Work out to whom this frame should go.
        if frame.stream_id != 0:
            self.streams[frame.stream_id].receive_frame(frame)
        else:
            self.receive_frame(frame)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import importlib

from openquake.baselib import sap
from openquake.commonlib import __version__
from openquake import commands

# force cluster users to use `oq engine` so that we have centralized logs
if os.environ['OQ_DISTRIBUTE'] == 'celery' and 'run' in sys.argv:
    print('You are on a cluster and you are using oq run?? '
          'Use oq engine --run instead!')

def oq():
    modnames = ['openquake.commands.%s' % mod[:-3]
                for mod in os.listdir(commands.__path__[0])
                if mod.endswith('.py') and not mod.startswith('_')]
    for modname in modnames:
        importlib.import_module(modname)
    parser = sap.compose(sap.Script.registry.values(),
                         prog='oq', version=__version__)
    parser.callfunc()

if __name__ == '__main__':
    oq()
<commit_msg>Version check [skip hazardlib][demos]<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import importlib

from openquake.baselib import sap
from openquake.commonlib import __version__
from openquake import commands

# check for Python version
if sys.version < '3.5':
    sys.exit('Python 3.5 is required, you are using %s', sys.executable)

# force cluster users to use `oq engine` so that we have centralized logs
if os.environ['OQ_DISTRIBUTE'] == 'celery' and 'run' in sys.argv:
    print('You are on a cluster and you are using oq run?? '
          'Use oq engine --run instead!')

def oq():
    modnames = ['openquake.commands.%s' % mod[:-3]
                for mod in os.listdir(commands.__path__[0])
                if mod.endswith('.py') and not mod.startswith('_')]
    for modname in modnames:
        importlib.import_module(modname)
    parser = sap.compose(sap.Script.registry.values(),
                         prog='oq', version=__version__)
    parser.callfunc()

if __name__ == '__main__':
    oq()
<|endoftext|>


<commit_before>from abc import abstractmethod, ABCMeta
from raco.algebra import DEFAULT_CARDINALITY
from raco.relation_key import RelationKey
from raco.scheme import Scheme
from ast import literal_eval
import os

class Relation(object):
    def __init__(self, name, sch):
        self.name = name
        self._scheme = sch

    def __eq__(self, other):
        return self.name == other.name
        # and self.scheme == other.scheme

    def scheme(self):
        return self._scheme

class FileRelation(Relation):
    pass

class ASCIIFile(FileRelation):
    pass

class Catalog(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def get_num_servers(self):
        """ Return number of servers in myria deployment """

    @abstractmethod
    def get_scheme(self, rel_key):
        """ Return scheme of tuples of rel_key """

    @abstractmethod
    def num_tuples(self, rel_key):
        """ Return number of tuples of rel_key """

# Some useful Catalog implementations

class FakeCatalog(Catalog):
    """ fake catalog, should only be used in test """
    def __init__(self, num_servers, child_sizes=None):
        self.num_servers = num_servers
        # default sizes
        self.cached = {}
        # overwrite default sizes if necessary
        if child_sizes:
            for child, size in child_sizes.items():
                self.cached[RelationKey(child)] = size

    def get_num_servers(self):
        return self.num_servers

    def num_tuples(self, rel_key):
        if rel_key in self.cached:
            return self.cached[rel_key]
        return DEFAULT_CARDINALITY

    def get_scheme(self, rel_key):
        raise NotImplementedError()

class FromFileCatalog(Catalog):
    """ Catalog that is created from a python file.
    Format of file is a dictionary of schemas.

    {'relation1' : [('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')],
     'relation2' : [('y', 'STRING_TYPE'), ('z', 'DATETIME_TYPE')]}

     Or there can be an optional cardinality for any relation
    {'relation1' : ([('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')], 10),
     'relation2' : [('y', 'STRING_TYPE'), ('z', 'DATETIME_TYPE')]}

     Or it can be a single relation, using filename as basename
     [('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')]

     see raco.types for allowed types
    """

    def __init__(self, cat, fname):
        self.catalog = {}

        def error():
            assert False, """Unexpected catalog file format. \
                    See raco.catalog.FromFileCatalog"""

        if isinstance(cat, dict):
            def parse(v):
                if isinstance(v, tuple):
                    return v
                elif isinstance(v, list):
                    return v, DEFAULT_CARDINALITY
                else:
                    error()

            self.catalog = dict([(k, parse(v)) for k, v in cat.iteritems()])
        elif isinstance(cat, list):
            name = os.path.splitext(os.path.basename(fname))[0]
            self.catalog = {'public:adhoc:{0}'.format(name): (cat, DEFAULT_CARDINALITY,)}
        else:
            error()

    def get_scheme(self, rel_key):
        return Scheme(self.__get_catalog_entry__(rel_key)[0])

    def get_keys(self):
        return self.catalog.keys()

    @classmethod
    def print_cat(cls, ffc1, ffc2):
        cpy = ffc1.catalog.copy()
        for k,v in ffc2.catalog:
            cpy[k] = v
        print cpy

    @classmethod
    def load_from_file(cls, path):
        with open(path) as fh:
            return cls(literal_eval(fh.read()), path)

    def get_num_servers(self):
        return 1

    def __get_catalog_entry__(self, rel_key):
        e = self.catalog.get(str(rel_key))
        if e is None:
            raise Exception(
                "relation {r} not found in catalog".format(r=rel_key))

        return e

    def num_tuples(self, rel_key):
        return self.__get_catalog_entry__(rel_key)[1]
<commit_msg>fix dictionary iter bug in fromfilecatalog<commit_after>from abc import abstractmethod, ABCMeta
from raco.algebra import DEFAULT_CARDINALITY
from raco.relation_key import RelationKey
from raco.scheme import Scheme
from ast import literal_eval
import os

class Relation(object):
    def __init__(self, name, sch):
        self.name = name
        self._scheme = sch

    def __eq__(self, other):
        return self.name == other.name
        # and self.scheme == other.scheme

    def scheme(self):
        return self._scheme

class FileRelation(Relation):
    pass

class ASCIIFile(FileRelation):
    pass

class Catalog(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def get_num_servers(self):
        """ Return number of servers in myria deployment """

    @abstractmethod
    def get_scheme(self, rel_key):
        """ Return scheme of tuples of rel_key """

    @abstractmethod
    def num_tuples(self, rel_key):
        """ Return number of tuples of rel_key """

# Some useful Catalog implementations

class FakeCatalog(Catalog):
    """ fake catalog, should only be used in test """
    def __init__(self, num_servers, child_sizes=None):
        self.num_servers = num_servers
        # default sizes
        self.cached = {}
        # overwrite default sizes if necessary
        if child_sizes:
            for child, size in child_sizes.items():
                self.cached[RelationKey(child)] = size

    def get_num_servers(self):
        return self.num_servers

    def num_tuples(self, rel_key):
        if rel_key in self.cached:
            return self.cached[rel_key]
        return DEFAULT_CARDINALITY

    def get_scheme(self, rel_key):
        raise NotImplementedError()

class FromFileCatalog(Catalog):
    """ Catalog that is created from a python file.
    Format of file is a dictionary of schemas.

    {'relation1' : [('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')],
     'relation2' : [('y', 'STRING_TYPE'), ('z', 'DATETIME_TYPE')]}

     Or there can be an optional cardinality for any relation
    {'relation1' : ([('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')], 10),
     'relation2' : [('y', 'STRING_TYPE'), ('z', 'DATETIME_TYPE')]}

     Or it can be a single relation, using filename as basename
     [('a', 'LONG_TYPE'), ('b', 'STRING_TYPE')]

     see raco.types for allowed types
    """

    def __init__(self, cat, fname):
        self.catalog = {}

        def error():
            assert False, """Unexpected catalog file format. \
                    See raco.catalog.FromFileCatalog"""

        if isinstance(cat, dict):
            def parse(v):
                if isinstance(v, tuple):
                    return v
                elif isinstance(v, list):
                    return v, DEFAULT_CARDINALITY
                else:
                    error()

            self.catalog = dict([(k, parse(v)) for k, v in cat.iteritems()])
        elif isinstance(cat, list):
            name = os.path.splitext(os.path.basename(fname))[0]
            self.catalog = {'public:adhoc:{0}'.format(name): (cat, DEFAULT_CARDINALITY,)}
        else:
            error()

    def get_scheme(self, rel_key):
        return Scheme(self.__get_catalog_entry__(rel_key)[0])

    def get_keys(self):
        return self.catalog.keys()

    @classmethod
    def print_cat(cls, ffc1, ffc2):
        cpy = ffc1.catalog.copy()
        for k,v in ffc2.catalog.iteritems():
            cpy[k] = v
        print cpy

    @classmethod
    def load_from_file(cls, path):
        with open(path) as fh:
            return cls(literal_eval(fh.read()), path)

    def get_num_servers(self):
        return 1

    def __get_catalog_entry__(self, rel_key):
        e = self.catalog.get(str(rel_key))
        if e is None:
            raise Exception(
                "relation {r} not found in catalog".format(r=rel_key))

        return e

    def num_tuples(self, rel_key):
        return self.__get_catalog_entry__(rel_key)[1]
<|endoftext|>


<commit_before>import ConfigParser
import os
from lxml import etree

from openquake.commonlib import nrml
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib.readinput import get_imtls

def _collect_source_model_paths(smlt):
    """
    Given a path to a source model logic tree or a file-like, collect all of
    the soft-linked path names to the source models it contains and return them
    as a uniquified list (no duplicates).
    """
    src_paths = []
    tree = etree.parse(smlt)
    for branch_set in tree.xpath('//nrml:logicTreeBranchSet',
                                 namespaces=nrml.PARSE_NS_MAP):

        if branch_set.get('uncertaintyType') == 'sourceModel':
            for branch in branch_set.xpath(
                    './nrml:logicTreeBranch/nrml:uncertaintyModel',
                    namespaces=nrml.PARSE_NS_MAP):
                src_paths.append(branch.text)
    return sorted(set(src_paths))

def parse_config(source, hazard_calculation_id=None, hazard_output_id=None):
    """
    Parse a dictionary of parameters from an INI-style config file.

    :param source:
        File-like object containing the config parameters.
    :param hazard_job_id:
        The ID of a previous calculation (or None)
    :param hazard_ouput_id:
        The output of a previous job (or None)
    :returns:
        An :class:`openquake.commonlib.oqvalidation.OqParam` instance
        containing the validate and casted parameters/values parsed from
        the job.ini file as well as a subdictionary 'inputs' containing
        absolute paths to all of the files referenced in the job.ini, keyed by
        the parameter name.
    """
    cp = ConfigParser.ConfigParser()
    cp.readfp(source)

    base_path = os.path.dirname(
        os.path.join(os.path.abspath('.'), source.name))
    params = dict(base_path=base_path, inputs={},
                  hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id)

    # Directory containing the config file we're parsing.
    base_path = os.path.dirname(os.path.abspath(source.name))

    for sect in cp.sections():
        for key, value in cp.items(sect):
            if key.endswith(('_file', '_csv')):
                input_type, _ext = key.rsplit('_', 1)
                path = value if os.path.isabs(value) else os.path.join(
                    base_path, value)
                params['inputs'][input_type] = path
            else:
                params[key] = value

    # load source inputs (the paths are the source_model_logic_tree)
    smlt = params['inputs'].get('source_model_logic_tree')
    if smlt:
        params['inputs']['source'] = [
            os.path.join(base_path, src_path)
            for src_path in _collect_source_model_paths(smlt)]

    # check for obsolete calculation_mode
    is_risk = hazard_calculation_id or hazard_output_id
    cmode = params['calculation_mode']
    if is_risk and cmode in ('classical', 'event_based', 'scenario'):
        raise ValueError('Please change calculation_mode=%s into %s_risk '
                         'in the .ini file' % (cmode, cmode))

    oqparam = OqParam(**params)
    if 'quantile_loss_curves' not in params:
        oqparam.quantile_loss_curves = []
    if 'poes_disagg' not in params:
        oqparam.poes_disagg = []
    if 'sites_disagg' not in params:
        oqparam.sites_disagg = []
    if 'specific_assets' not in params:
        oqparam.specific_assets = []
    if 'conditional_loss_poes' not in params:
        oqparam.conditional_loss_poes = []

    # define the parameter `intensity measure types and levels` always
    oqparam.intensity_measure_types_and_levels = get_imtls(oqparam)

    # remove the redundant parameter `intensity_measure_types`
    if hasattr(oqparam, 'intensity_measure_types'):
        delattr(oqparam, 'intensity_measure_types')

    return oqparam
<commit_msg>Moved some code into engine.py<commit_after>import ConfigParser
import os
from lxml import etree

from openquake.commonlib import nrml
from openquake.commonlib.oqvalidation import OqParam
from openquake.commonlib.readinput import get_imtls

def _collect_source_model_paths(smlt):
    """
    Given a path to a source model logic tree or a file-like, collect all of
    the soft-linked path names to the source models it contains and return them
    as a uniquified list (no duplicates).
    """
    src_paths = []
    tree = etree.parse(smlt)
    for branch_set in tree.xpath('//nrml:logicTreeBranchSet',
                                 namespaces=nrml.PARSE_NS_MAP):

        if branch_set.get('uncertaintyType') == 'sourceModel':
            for branch in branch_set.xpath(
                    './nrml:logicTreeBranch/nrml:uncertaintyModel',
                    namespaces=nrml.PARSE_NS_MAP):
                src_paths.append(branch.text)
    return sorted(set(src_paths))

def parse_config(source, hazard_calculation_id=None, hazard_output_id=None):
    """
    Parse a dictionary of parameters from an INI-style config file.

    :param source:
        File-like object containing the config parameters.
    :param hazard_job_id:
        The ID of a previous calculation (or None)
    :param hazard_ouput_id:
        The output of a previous job (or None)
    :returns:
        An :class:`openquake.commonlib.oqvalidation.OqParam` instance
        containing the validate and casted parameters/values parsed from
        the job.ini file as well as a subdictionary 'inputs' containing
        absolute paths to all of the files referenced in the job.ini, keyed by
        the parameter name.
    """
    cp = ConfigParser.ConfigParser()
    cp.readfp(source)

    base_path = os.path.dirname(
        os.path.join(os.path.abspath('.'), source.name))
    params = dict(base_path=base_path, inputs={},
                  hazard_calculation_id=hazard_calculation_id,
                  hazard_output_id=hazard_output_id)

    # Directory containing the config file we're parsing.
    base_path = os.path.dirname(os.path.abspath(source.name))

    for sect in cp.sections():
        for key, value in cp.items(sect):
            if key.endswith(('_file', '_csv')):
                input_type, _ext = key.rsplit('_', 1)
                path = value if os.path.isabs(value) else os.path.join(
                    base_path, value)
                params['inputs'][input_type] = path
            else:
                params[key] = value

    # load source inputs (the paths are the source_model_logic_tree)
    smlt = params['inputs'].get('source_model_logic_tree')
    if smlt:
        params['inputs']['source'] = [
            os.path.join(base_path, src_path)
            for src_path in _collect_source_model_paths(smlt)]

    # check for obsolete calculation_mode
    is_risk = hazard_calculation_id or hazard_output_id
    cmode = params['calculation_mode']
    if is_risk and cmode in ('classical', 'event_based', 'scenario'):
        raise ValueError('Please change calculation_mode=%s into %s_risk '
                         'in the .ini file' % (cmode, cmode))

    oqparam = OqParam(**params)

    # define the parameter `intensity measure types and levels` always
    oqparam.intensity_measure_types_and_levels = get_imtls(oqparam)

    # remove the redundant parameter `intensity_measure_types`
    if hasattr(oqparam, 'intensity_measure_types'):
        delattr(oqparam, 'intensity_measure_types')

    return oqparam
<|endoftext|>


<commit_before>import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration

from .base import *  # noqa
from .base import env

SECRET_KEY = env("SECRET_KEY")
# CELERY_BROKER_URL = env("CELERY_BROKER_URL")
USE_SCHEDULER = False

# if FRONTEND_URL == "http://testfrontend.it/":
#     raise ImproperlyConfigured("Please configure FRONTEND_URL for production")

SENTRY_DSN = env("SENTRY_DSN", default="")

if SENTRY_DSN:
    sentry_sdk.init(dsn=SENTRY_DSN, integrations=[DjangoIntegration()])

SLACK_INCOMING_WEBHOOK_URL = env("SLACK_INCOMING_WEBHOOK_URL")

DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"

AWS_STORAGE_BUCKET_NAME = env("AWS_MEDIA_BUCKET", None)
AWS_S3_REGION_NAME = env("AWS_REGION_NAME", None)
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", None)
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY", None)
<commit_msg>Add better default for s3 region<commit_after>import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration

from .base import *  # noqa
from .base import env

SECRET_KEY = env("SECRET_KEY")
# CELERY_BROKER_URL = env("CELERY_BROKER_URL")
USE_SCHEDULER = False

# if FRONTEND_URL == "http://testfrontend.it/":
#     raise ImproperlyConfigured("Please configure FRONTEND_URL for production")

SENTRY_DSN = env("SENTRY_DSN", default="")

if SENTRY_DSN:
    sentry_sdk.init(dsn=SENTRY_DSN, integrations=[DjangoIntegration()])

SLACK_INCOMING_WEBHOOK_URL = env("SLACK_INCOMING_WEBHOOK_URL")

DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"

AWS_STORAGE_BUCKET_NAME = env("AWS_MEDIA_BUCKET", None)
AWS_S3_REGION_NAME = env("AWS_REGION_NAME", "eu-central-1")
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID", None)
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY", None)
<|endoftext|>