<commit_before>''' Script to create and populate new run directory.
Created on 25 Jun 2010

@author: Ian Huston
'''

import os
import os.path
import logging
import sys
import time
from optparse import OptionParser

#Version information
from sys import version as python_version
from numpy import __version__ as numpy_version
from scipy import __version__ as scipy_version
from tables import __version__ as tables_version

#Local modules from pyflation package
from pyflation import configuration, helpers
from pyflation.setup import setup, setup_args

provenance_template = """Provenance document for this Pyflation run
------------------------------------------

Bazaar Revision Control Information (if available)
-------------------------------------------------
Branch name: %(nick)s
Branch revision number: %(revno)s
Branch revision id: %(revid)s

Code Directory Information
--------------------------
Original code directory: %(codedir)s
New run directory: %(newrundir)s
Date run directory was created: %(now)s

Version information at time of run creation
-------------------------------------------
Python version: %(python_version)s
Numpy version: %(numpy_version)s
Scipy version: %(scipy_version)s
PyTables version: %(tables_version)s

This information added on: %(now)s.
-----------------------------------------------

"""

def create_run_directory(newrundir, codedir, bzr_checkout=False):
    """Create the run directory using `newdir` as directory name."""
    if os.path.isdir(newrundir):
        raise IOError("New run directory already exists!")

    try:
        helpers.ensurepath(newrundir)
        os.makedirs(newrundir)
    except OSError:
        logging.error("Creating new run directory failed.")
        raise

    resultsdir = os.path.join(newrundir, configuration.RESULTSDIRNAME)
    logdir = os.path.join(newrundir, configuration.LOGDIRNAME)
    qsublogsdir = os.path.join(newrundir, configuration.QSUBLOGSDIRNAME)
    qsubscriptsdir = os.path.join(newrundir, configuration.QSUBSCRIPTSDIRNAME)
    #debug info
    logging.debug("resultsdir=%s, logdir=%s, qsublogsdir=%s, qsubscriptsdir=%s",
                  resultsdir, logdir, qsublogsdir, qsubscriptsdir)

    try:
        os.makedirs(resultsdir)
        os.makedirs(logdir)
        os.makedirs(qsublogsdir)
        os.makedirs(qsubscriptsdir)
    except OSError:
        logging.error("Creating subdirectories in new run directory failed.")
        raise

    #Try to do Bazaar checkout of code
    try:
        import bzrlib.export, bzrlib.workingtree
        bzr_available = True
    except ImportError:
        bzr_available = False

    logging.debug("bzr_available=%s", bzr_available)

    if bzr_available:
        mytree =  bzrlib.workingtree.WorkingTree.open(codedir)
        if bzr_checkout:
            newtree = mytree.branch.create_checkout(os.path.join(newrundir,
                            configuration.CODEDIRNAME), lightweight=True)
        else:
            bzrlib.export.export(mytree, os.path.join(newrundir, configuration.CODEDIRNAME))

    else:
        raise NotImplementedError("Bazaar is needed to copy code directory. Please do this manually.")

    #Try to run setup to create .so files
    try:
        olddir = os.getcwd()
        os.chdir(os.path.join(newrundir, configuration.CODEDIRNAME))
        logging.info("Preparing to compile non-python files.")
        setup(script_args=["build_ext", "-i"], **setup_args)
        os.chdir(olddir)
    except:
        logging.exception("Compiling additional modules did not work. Please do so by hand!")

    #Create provenance file detailing revision and branch used
    provenance_dict = dict(python_version=python_version,
                           numpy_version=numpy_version,
                           scipy_version=scipy_version,
                           tables_version=tables_version,
                           codedir=codedir,
                           newrundir=newrundir,
                           now=time.strftime("%Y/%m/%d %H:%M:%S %Z"))
    if bzr_available:
        provenance_dict["nick"] = mytree.branch.nick
        provenance_dict["revno"] = mytree.branch.revno()
        provenance_dict["revid"] = mytree.branch.last_revision()
    else:
        provenance_dict["nick"] = "Unavailable"
        provenance_dict["revno"] = "Unavailable"
        provenance_dict["revid"] = "Unavailable"
    provenance_file = os.path.join(newrundir, configuration.LOGDIRNAME,
                                   configuration.provenancefilename)
    with open(provenance_file, "w") as f:
        f.write(provenance_template % provenance_dict)
        logging.info("Created provenance file %s." % provenance_file)

    return

def main(argv = None):
    """Check command line options and start directory creation."""
    if not argv:
        argv = sys.argv
    #Parse command line options
    parser = OptionParser()

    parser.set_defaults(loglevel=configuration.LOGLEVEL)

    parser.add_option("-d", "--dir", dest="dir", default=os.getcwd(),
                  help="create run directory in DIR, default is current directory", metavar="DIR")
    parser.add_option("-n", "--name", dest="dirname",
                      help="new run directory name")
    parser.add_option("-c", "--codedir", dest="codedir",
                  help="copy code from CODEDIR", metavar="CODEDIR")
    parser.add_option("-q", "--quiet",
                  action="store_const", const=logging.FATAL, dest="loglevel",
                  help="only print fatal error messages")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=logging.INFO, dest="loglevel",
                  help="print informative messages")
    parser.add_option("--debug",
                  action="store_const", const=logging.DEBUG, dest="loglevel",
                  help="print lots of debugging information")
    parser.add_option("--checkout", action="store_true", dest="bzr_checkout",
                      default=True, help="create a bzr checkout instead of export")

    (options, args) = parser.parse_args(args=argv[1:])

    logging.basicConfig(level=options.loglevel)

    if not os.path.isdir(options.dir):
        raise IOError("Please check that parent directory %s exists." % options.dir)

    if options.dirname:
        newdir = os.path.join(options.dir, options.dirname)
        logging.debug("Variable newdir specified with value %s.", newdir)
    else:
        newdir = os.path.join(options.dir, time.strftime("%Y%m%d%H%M%S"))
        logging.debug("Variable newdir created with value %s", newdir)

    if options.codedir:
        codedir = options.codedir
        logging.debug("Option codedir specified with value %s.", options.codedir)
    else:
        codedir = os.path.join(configuration.CODEDIR)
        logging.debug("Variable codedir created with value %s.", codedir)

    try:
        create_run_directory(newdir, codedir, options.bzr_checkout)
    except Exception, e:
        logging.critical("Something went wrong! Quitting.")
        sys.exit(e)

    logging.info("New run directory created successfully.")
    return

if __name__ == '__main__':
    main()
    <commit_msg>Add error message if pyflation package is not found.<commit_after>''' Script to create and populate new run directory.
Created on 25 Jun 2010

@author: Ian Huston
'''

import os
import os.path
import logging
import sys
import time
from optparse import OptionParser

#Version information
from sys import version as python_version
from numpy import __version__ as numpy_version
from scipy import __version__ as scipy_version
from tables import __version__ as tables_version

try:
    #Local modules from pyflation package
    from pyflation import configuration, helpers
    from pyflation.setup import setup, setup_args
except ImportError:
    if __name__ == "__main__":
        print """Pyflation module needs to be available.
        Either run this script from the base directory as bin/newrun.py or add directory enclosing pyflation package to PYTHONPATH."""
    else:
        raise

provenance_template = """Provenance document for this Pyflation run
------------------------------------------

Bazaar Revision Control Information (if available)
-------------------------------------------------
Branch name: %(nick)s
Branch revision number: %(revno)s
Branch revision id: %(revid)s

Code Directory Information
--------------------------
Original code directory: %(codedir)s
New run directory: %(newrundir)s
Date run directory was created: %(now)s

Version information at time of run creation
-------------------------------------------
Python version: %(python_version)s
Numpy version: %(numpy_version)s
Scipy version: %(scipy_version)s
PyTables version: %(tables_version)s

This information added on: %(now)s.
-----------------------------------------------

"""

def create_run_directory(newrundir, codedir, bzr_checkout=False):
    """Create the run directory using `newdir` as directory name."""
    if os.path.isdir(newrundir):
        raise IOError("New run directory already exists!")

    try:
        helpers.ensurepath(newrundir)
        os.makedirs(newrundir)
    except OSError:
        logging.error("Creating new run directory failed.")
        raise

    resultsdir = os.path.join(newrundir, configuration.RESULTSDIRNAME)
    logdir = os.path.join(newrundir, configuration.LOGDIRNAME)
    qsublogsdir = os.path.join(newrundir, configuration.QSUBLOGSDIRNAME)
    qsubscriptsdir = os.path.join(newrundir, configuration.QSUBSCRIPTSDIRNAME)
    #debug info
    logging.debug("resultsdir=%s, logdir=%s, qsublogsdir=%s, qsubscriptsdir=%s",
                  resultsdir, logdir, qsublogsdir, qsubscriptsdir)

    try:
        os.makedirs(resultsdir)
        os.makedirs(logdir)
        os.makedirs(qsublogsdir)
        os.makedirs(qsubscriptsdir)
    except OSError:
        logging.error("Creating subdirectories in new run directory failed.")
        raise

    #Try to do Bazaar checkout of code
    try:
        import bzrlib.export, bzrlib.workingtree
        bzr_available = True
    except ImportError:
        bzr_available = False

    logging.debug("bzr_available=%s", bzr_available)

    if bzr_available:
        mytree =  bzrlib.workingtree.WorkingTree.open(codedir)
        if bzr_checkout:
            newtree = mytree.branch.create_checkout(os.path.join(newrundir,
                            configuration.CODEDIRNAME), lightweight=True)
        else:
            bzrlib.export.export(mytree, os.path.join(newrundir, configuration.CODEDIRNAME))

    else:
        raise NotImplementedError("Bazaar is needed to copy code directory. Please do this manually.")

    #Try to run setup to create .so files
    try:
        olddir = os.getcwd()
        os.chdir(os.path.join(newrundir, configuration.CODEDIRNAME))
        logging.info("Preparing to compile non-python files.")
        setup(script_args=["build_ext", "-i"], **setup_args)
        os.chdir(olddir)
    except:
        logging.exception("Compiling additional modules did not work. Please do so by hand!")

    #Create provenance file detailing revision and branch used
    provenance_dict = dict(python_version=python_version,
                           numpy_version=numpy_version,
                           scipy_version=scipy_version,
                           tables_version=tables_version,
                           codedir=codedir,
                           newrundir=newrundir,
                           now=time.strftime("%Y/%m/%d %H:%M:%S %Z"))
    if bzr_available:
        provenance_dict["nick"] = mytree.branch.nick
        provenance_dict["revno"] = mytree.branch.revno()
        provenance_dict["revid"] = mytree.branch.last_revision()
    else:
        provenance_dict["nick"] = "Unavailable"
        provenance_dict["revno"] = "Unavailable"
        provenance_dict["revid"] = "Unavailable"
    provenance_file = os.path.join(newrundir, configuration.LOGDIRNAME,
                                   configuration.provenancefilename)
    with open(provenance_file, "w") as f:
        f.write(provenance_template % provenance_dict)
        logging.info("Created provenance file %s." % provenance_file)

    return

def main(argv = None):
    """Check command line options and start directory creation."""
    if not argv:
        argv = sys.argv
    #Parse command line options
    parser = OptionParser()

    parser.set_defaults(loglevel=configuration.LOGLEVEL)

    parser.add_option("-d", "--dir", dest="dir", default=os.getcwd(),
                  help="create run directory in DIR, default is current directory", metavar="DIR")
    parser.add_option("-n", "--name", dest="dirname",
                      help="new run directory name")
    parser.add_option("-c", "--codedir", dest="codedir",
                  help="copy code from CODEDIR", metavar="CODEDIR")
    parser.add_option("-q", "--quiet",
                  action="store_const", const=logging.FATAL, dest="loglevel",
                  help="only print fatal error messages")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=logging.INFO, dest="loglevel",
                  help="print informative messages")
    parser.add_option("--debug",
                  action="store_const", const=logging.DEBUG, dest="loglevel",
                  help="print lots of debugging information")
    parser.add_option("--checkout", action="store_true", dest="bzr_checkout",
                      default=True, help="create a bzr checkout instead of export")

    (options, args) = parser.parse_args(args=argv[1:])

    logging.basicConfig(level=options.loglevel)

    if not os.path.isdir(options.dir):
        raise IOError("Please check that parent directory %s exists." % options.dir)

    if options.dirname:
        newdir = os.path.join(options.dir, options.dirname)
        logging.debug("Variable newdir specified with value %s.", newdir)
    else:
        newdir = os.path.join(options.dir, time.strftime("%Y%m%d%H%M%S"))
        logging.debug("Variable newdir created with value %s", newdir)

    if options.codedir:
        codedir = options.codedir
        logging.debug("Option codedir specified with value %s.", options.codedir)
    else:
        codedir = os.path.join(configuration.CODEDIR)
        logging.debug("Variable codedir created with value %s.", codedir)

    try:
        create_run_directory(newdir, codedir, options.bzr_checkout)
    except Exception, e:
        logging.critical("Something went wrong! Quitting.")
        sys.exit(e)

    logging.info("New run directory created successfully.")
    return

if __name__ == '__main__':
    main()
    <|endoftext|>


<commit_before>#!/usr/bin/python
# vim: set ai sw=4 sta et fo=croql

# The structure of this data is: one site per line.
# The line is separated into two halves by a colon.
# The first half is a comma-separated list containing:
#     the ***permanent*** site number,
#     name of the site,
#     county of the site,
#     the hostname of the computer hosting the sensors.
# The second half is a comma-separated list of sensor model and serials.
rths_sites = """
0,Dutch Apple,Albany,s14:windair-02,DEPTH-1,voltage-s14,ppal-01
1,Beacon,Dutchess,s7:windair-04,ppal1-02
2,Stillwater,Saratoga,s8:windair-10,voltage-s8,pdepth2-2,pbar2-3,ppal1-03
3,Freemans Bridge,Schenectady,s10:windair-05,pdepth-01,ppal1-01
4,Old Main test,Saint Lawrence,r17:pbar2-16,pH-10,windair-35,COND-10,DO2-10,voltage-r17,pdepth2-19
5,West Point,Orange,s17:pdepth2-4,pbar-02
6,Newburgh,Orange,s25:pdepth1-14,pbar-04
7,North Creek,Warren,r9:windair-11,voltage-r9,pdepth1-02,pbar1-1,ppal1-06
8,Russ Lawn,Saint Lawrence,s4:voltage-s4
9,Old Main back lawn,Saint Lawrence,s6:windair-01,ppal2-0,pbar2-x
12,Corinth,Saratoga,r1:windair-40,pbar2-08,pdepth1-19,ppal1-08
14,St. Regis,Saint Lawrence,r4:COND-3,pdepth1-20,pbar2-10,DO-3,ppal1-09,OBS-10,windair-48,pH-3,voltage-r10
15,Hinckley,Herkimer,r16:pbar2-4,pH-1,windair-41,OBS-9,COND-1,DO-1,ppal1-07,pdepth2-8,fl3-414
16,Grasse,Saint Lawrence,r11:pH-0,COND-1,DO-1,ppal1-07,pdepth1-18,ppal1-14,windair-01,pbar2-7,voltage-r11
17,Racquette,Saint Lawrence,r7:COND-1,ppal1-15,DO-1,pH-1,pbar2-5,windair-18,pdepth2-3,voltage-r7
18,Lock 8,Schenectady,r19:pbar2-17,ppal1-17,pdepth2-11,windair-25,pdepth2-7,voltage-r19
19,Schodack,Rensselaer,r21:pbar2-13,pdepth2-17,pdepth2-9,ppal1-16,windair-38,voltage-r21
20,Indian Lake,Hamilton,r22:pbar2-18,pdepth2-19,pdepth2-15,voltage-r22
21,Piermont,Rockland,r17:pbar2-16,pdepth2-20,pdepth2-16,windair-35,voltage-r17
22,RATES Barn,St. Lawrence,r37:voltage-r37
"""
#13,St. Regis,Saint Lawrence,r1:windair-40,pbar2-8,pdepth1-19
#11,St. Regis,Saint Lawrence,r4:COND-0,pdepth1-20,pbar2-01,DO-0,ppal1-09,OBS-8,windair-23,pH-0

other_sites = """
4,Old Main back lawn,Saint Lawrence,s24:windair-09
Old Main,s23:MBIRDS-5
10,Evans and Whites,Saint Lawrence,s10:MBIRDS-2
"""

rths_sites = [(a[0].split(","), a[1].split(",")) for a in [a.split(":") for a in rths_sites.split("\n")[1:-1]]]

# The structure of this data is: one sensor per line
#   Except for first and last blank lines,
#   and indented lines are folded and have a semicolon inserted
# The sensors are ordered in the same order as the columns in the data file.
# The line is separated into two halves by a colon.
# The first half is the sensor model name.
# The second half is split into measurements by a semicolon.
# Each measurement has these parts: the name of the parameter being measured,
#   the codename of the parameter,
#   the units of the parameter,
#   the sample medium of the parameter (Air or Surface Water),
#   the database name of the parameter.
#   insert into variables (VariableCode, VariableName, VariableUnitsID, SampleMedium, ValueType, IsRegular, TimeSupport, TimeUnitsID, DataType, GeneralCategory, NoDataValue) values ('precipdaily', 'Precipitation', 305, 'Precipitation', 'Field Observation', 1, 0, 100, 'Continuous', 'Instrumentation', -999);
#
rths_sensors = """
windair:
    Wind speed,windspeed,119,Air,
    Wind direction,winddir,2,Air,
    Temperature,airtemp,96,Air,
    Relative Humidity,humidity,1,Air,
ppal:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
ppal1:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
ppal2:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
pdepth:
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Temperature,temperature,96,Surface Water,
pdepth1:
    ,,,,
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Gage height,largerangepressure,47,Surface Water,Large range
    Temperature,temperature,96,Surface Water,
pdepth2:
    ,,,,
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Gage height,largerangepressure,47,Surface Water,Large range
    Temperature,temperature,96,Surface Water,
    Temperature,boardtemperature,96,Other,PCB temperature
pbar:
    Temperature,cabinettemperature,96,Other,Cabinet temperature
    Barometric Pressure,baro,90,Air,
pbar1:
    Temperature,cabinettemperature,96,Other,Cabinet temperature
    Barometric Pressure,baro,90,Air,
    ,,,,
pbar2:
    Temperature,cabinettemperature,96,Other,
    Barometric Pressure,baro,90,Air,
DEPTH:
    Gage height,depth,47,Surface Water,
DEPTH2:
    Gage height,depth2highaccuracy,47,Surface Water,High accuracy
    Gage height,depth2largerange,47,Surface Water,Large range
MBIRDS:
    Temperature,mbwatertemp,96,Surface Water,Infrared sensing
    Temperature,mbairtemp,96,Air,Unshielded sensor
    Gage height,mbdepth,47,Surface Water,Ultrasonic distance ranging
COND:
    Electrical conductivity,lowcond,192,Surface Water,Low range sensor
    Electrical conductivity,mediumcond,192,Surface Water,Medium range sensor
    Electrical conductivity,highcond,192,Surface Water,High range sensor
fl3:
    ,,,,
    Chlorophyll a,chlorophyll,206,Surface Water,Optical sensor
    ,,,,
    ,,,,
    ,,,,
    Colored Dissolved Organic Matter,cdom,206,Surface Water,Optical sensor
    ,,,,
OBS:
    ,,,,
    ,,,,
    ,,,,
    Turbidity,optical-red,221,Surface Water,Red
    Turbidity,reference,257,Surface Water,Reference sensor
    ,,,,
    Turbidity,optical-green,221,Surface Water,Green
    Turbidity,optical-blue,221,Surface Water,Blue
pH:
    pH,ph,309,Surface Water,ion exchange
DO:
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
DO2:
    Oxygen%2C dissolved,dopercent,1,Surface Water,galvanic response
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
DO3:
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
voltage:
    Battery Voltage,batteryvoltage,168,Not Relevant,
    Current,chargecurrent,378,Not Relevant,Solar Charge
    Current,loadcurrent,378,Not Relevant,Load
optode:
    Oxygen%2C dissolved,do,199,Surface Water,optode
    Oxygen%2C dissolved,dopercent,1,Surface Water,optode
    Temperature,temperature,96,Surface Water,optode
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
"""

rths_sensors = rths_sensors .replace(":\n    ",":").replace("\n    ",";")

rths_sensors = dict([(a[0], [[d.replace("%2C",",") for d in b.split(",")] for b in a[1].split(";")]) for a in [c.split(":") for c in rths_sensors.split("\n")[1:-1]]])

# The structure of this data is: one sensor per line.
# The line is separated into two halves by a colon.
# The first half is the sensor model name.
# The second half is a description of the sensor.
rths_descriptions = """windair:A shielded air temperature and relative humidity sensor mounted on the same pole as an anememometer and wind vane.
ppal:A water collection tube with depth gauge.
ppal1:A water collection tube with depth gauge.
ppal2:A water collection tube with embedded depth gauge.
pdepth:An encapsulated differential pressure sensor measuring water column height.
pdepth1:An encapsulated differential pressure sensor measuring water column height and water temperature.
pdepth2:An encapsulated differential pressure sensor measuring water column height and water temperature.
pbar:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar1:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar2:A barometer and temperature gauge (which is inside our electronics enclosure).
DEPTH:An unencapsulated differential pressure sensor measuring water column height.
DEPTH2:An unencapsulated differential pressure sensor measuring water column height at two resolutions and ranges.
MBIRDS:An ultrasonic distance measure to the water surface below, an infrared temperature sensor, and an unshielded air temperature sensor.
COND:A direct measurement of the conductivity sampled at three ranges.
OBS:Optical Back Scatter measurement of the reflectance of the water at three wavelengths: visible red, green, and blue.
pH:An ion exchange pH probe.
DO:Dissolved Oxygen
voltage:Battery voltage"""
rths_descriptions = dict([ [a[0], a[1]] for a in [c.split(":") for c in rths_descriptions.split("\n")]])

# select distinct(VariableUnitsID), UnitsAbbreviation from seriescatalog, units where seriescatalog.VariableUnitsID = units.UnitsID;
units = { 54:"mm", 47:"cm", 168:"V", 199:"mg/l", 309:"pH", 100:"s", 52:"m", 1:"%",2:"deg",90:"mbar",96:"degC",119:"m/s", 192:"uS/cm",
          221:"NTU", 257:"#", 121:'mm/hr', 378:'A', 305:"mm/day" }

<commit_msg>pbar3 and 4 differ by the hardware, not the data format.<commit_after>#!/usr/bin/python
# vim: set ai sw=4 sta et fo=croql

# The structure of this data is: one site per line.
# The line is separated into two halves by a colon.
# The first half is a comma-separated list containing:
#     the ***permanent*** site number,
#     name of the site,
#     county of the site,
#     the hostname of the computer hosting the sensors.
# The second half is a comma-separated list of sensor model and serials.
rths_sites = """
0,Dutch Apple,Albany,s14:windair-02,DEPTH-1,voltage-s14,ppal-01
1,Beacon,Dutchess,s7:windair-04,ppal1-02
2,Stillwater,Saratoga,s8:windair-10,voltage-s8,pdepth2-2,pbar2-3,ppal1-03
3,Freemans Bridge,Schenectady,s10:windair-05,pdepth-01,ppal1-01
4,Old Main test,Saint Lawrence,r17:pbar2-16,pH-10,windair-35,COND-10,DO2-10,voltage-r17,pdepth2-19
5,West Point,Orange,s17:pdepth2-4,pbar-02
6,Newburgh,Orange,s25:pdepth1-14,pbar-04
7,North Creek,Warren,r9:windair-11,voltage-r9,pdepth1-02,pbar1-1,ppal1-06
8,Russ Lawn,Saint Lawrence,s4:voltage-s4
9,Old Main back lawn,Saint Lawrence,s6:windair-01,ppal2-0,pbar2-x
12,Corinth,Saratoga,r1:windair-40,pbar2-08,pdepth1-19,ppal1-08
14,St. Regis,Saint Lawrence,r4:COND-3,pdepth1-20,pbar2-10,DO-3,ppal1-09,OBS-10,windair-48,pH-3,voltage-r10
15,Hinckley,Herkimer,r16:pbar2-4,pH-1,windair-41,OBS-9,COND-1,DO-1,ppal1-07,pdepth2-8,fl3-414
16,Grasse,Saint Lawrence,r11:pH-0,COND-1,DO-1,ppal1-07,pdepth1-18,ppal1-14,windair-01,pbar2-7,voltage-r11
17,Racquette,Saint Lawrence,r7:COND-1,ppal1-15,DO-1,pH-1,pbar2-5,windair-18,pdepth2-3,voltage-r7
18,Lock 8,Schenectady,r19:pbar2-17,ppal1-17,pdepth2-11,windair-25,pdepth2-7,voltage-r19
19,Schodack,Rensselaer,r21:pbar2-13,pdepth2-17,pdepth2-9,ppal1-16,windair-38,voltage-r21
20,Indian Lake,Hamilton,r22:pbar2-18,pdepth2-19,pdepth2-15,voltage-r22
21,Piermont,Rockland,r17:pbar2-16,pdepth2-20,pdepth2-16,windair-35,voltage-r17
22,RATES Barn,St. Lawrence,r37:voltage-r37
"""
#13,St. Regis,Saint Lawrence,r1:windair-40,pbar2-8,pdepth1-19
#11,St. Regis,Saint Lawrence,r4:COND-0,pdepth1-20,pbar2-01,DO-0,ppal1-09,OBS-8,windair-23,pH-0

other_sites = """
4,Old Main back lawn,Saint Lawrence,s24:windair-09
Old Main,s23:MBIRDS-5
10,Evans and Whites,Saint Lawrence,s10:MBIRDS-2
"""

rths_sites = [(a[0].split(","), a[1].split(",")) for a in [a.split(":") for a in rths_sites.split("\n")[1:-1]]]

# The structure of this data is: one sensor per line
#   Except for first and last blank lines,
#   and indented lines are folded and have a semicolon inserted
# The sensors are ordered in the same order as the columns in the data file.
# The line is separated into two halves by a colon.
# The first half is the sensor model name.
# The second half is split into measurements by a semicolon.
# Each measurement has these parts: the name of the parameter being measured,
#   the codename of the parameter,
#   the units of the parameter,
#   the sample medium of the parameter (Air or Surface Water),
#   the database name of the parameter.
#   insert into variables (VariableCode, VariableName, VariableUnitsID, SampleMedium, ValueType, IsRegular, TimeSupport, TimeUnitsID, DataType, GeneralCategory, NoDataValue) values ('precipdaily', 'Precipitation', 305, 'Precipitation', 'Field Observation', 1, 0, 100, 'Continuous', 'Instrumentation', -999);
#
rths_sensors = """
windair:
    Wind speed,windspeed,119,Air,
    Wind direction,winddir,2,Air,
    Temperature,airtemp,96,Air,
    Relative Humidity,humidity,1,Air,
ppal:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
ppal1:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
ppal2:
    Precipitation,precip,121,Precipitation,
    Water level,bucketdepth,54,Precipitation,Rain gauge depth
    Temperature,buckettemp,96,Precipitation,
pdepth:
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Temperature,temperature,96,Surface Water,
pdepth1:
    ,,,,
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Gage height,largerangepressure,47,Surface Water,Large range
    Temperature,temperature,96,Surface Water,
pdepth2:
    ,,,,
    Gage height,highaccuracypressure,47,Surface Water,High accuracy
    Gage height,largerangepressure,47,Surface Water,Large range
    Temperature,temperature,96,Surface Water,
    Temperature,boardtemperature,96,Other,PCB temperature
pbar:
    Temperature,cabinettemperature,96,Other,Cabinet temperature
    Barometric Pressure,baro,90,Air,
pbar1:
    Temperature,cabinettemperature,96,Other,Cabinet temperature
    Barometric Pressure,baro,90,Air,
    ,,,,
pbar2:
    Temperature,cabinettemperature,96,Other,
    Barometric Pressure,baro,90,Air,
pbar3:
    Temperature,cabinettemperature,96,Other,
    Barometric Pressure,baro,90,Air,
pbar4:
    Temperature,cabinettemperature,96,Other,
    Barometric Pressure,baro,90,Air,
DEPTH:
    Gage height,depth,47,Surface Water,
DEPTH2:
    Gage height,depth2highaccuracy,47,Surface Water,High accuracy
    Gage height,depth2largerange,47,Surface Water,Large range
MBIRDS:
    Temperature,mbwatertemp,96,Surface Water,Infrared sensing
    Temperature,mbairtemp,96,Air,Unshielded sensor
    Gage height,mbdepth,47,Surface Water,Ultrasonic distance ranging
COND:
    Electrical conductivity,lowcond,192,Surface Water,Low range sensor
    Electrical conductivity,mediumcond,192,Surface Water,Medium range sensor
    Electrical conductivity,highcond,192,Surface Water,High range sensor
fl3:
    ,,,,
    Chlorophyll a,chlorophyll,206,Surface Water,Optical sensor
    ,,,,
    ,,,,
    ,,,,
    Colored Dissolved Organic Matter,cdom,206,Surface Water,Optical sensor
    ,,,,
OBS:
    ,,,,
    ,,,,
    ,,,,
    Turbidity,optical-red,221,Surface Water,Red
    Turbidity,reference,257,Surface Water,Reference sensor
    ,,,,
    Turbidity,optical-green,221,Surface Water,Green
    Turbidity,optical-blue,221,Surface Water,Blue
pH:
    pH,ph,309,Surface Water,ion exchange
DO:
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
DO2:
    Oxygen%2C dissolved,dopercent,1,Surface Water,galvanic response
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
DO3:
    Oxygen%2C dissolved,do,199,Surface Water,galvanic response
voltage:
    Battery Voltage,batteryvoltage,168,Not Relevant,
    Current,chargecurrent,378,Not Relevant,Solar Charge
    Current,loadcurrent,378,Not Relevant,Load
optode:
    Oxygen%2C dissolved,do,199,Surface Water,optode
    Oxygen%2C dissolved,dopercent,1,Surface Water,optode
    Temperature,temperature,96,Surface Water,optode
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
    ,,,,
"""

rths_sensors = rths_sensors .replace(":\n    ",":").replace("\n    ",";")

rths_sensors = dict([(a[0], [[d.replace("%2C",",") for d in b.split(",")] for b in a[1].split(";")]) for a in [c.split(":") for c in rths_sensors.split("\n")[1:-1]]])

# The structure of this data is: one sensor per line.
# The line is separated into two halves by a colon.
# The first half is the sensor model name.
# The second half is a description of the sensor.
rths_descriptions = """windair:A shielded air temperature and relative humidity sensor mounted on the same pole as an anememometer and wind vane.
ppal:A water collection tube with depth gauge.
ppal1:A water collection tube with depth gauge.
ppal2:A water collection tube with embedded depth gauge.
pdepth:An encapsulated differential pressure sensor measuring water column height.
pdepth1:An encapsulated differential pressure sensor measuring water column height and water temperature.
pdepth2:An encapsulated differential pressure sensor measuring water column height and water temperature.
pbar:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar1:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar2:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar3:A barometer and temperature gauge (which is inside our electronics enclosure).
pbar4:A barometer and temperature gauge (which is inside our electronics enclosure).
DEPTH:An unencapsulated differential pressure sensor measuring water column height.
DEPTH2:An unencapsulated differential pressure sensor measuring water column height at two resolutions and ranges.
MBIRDS:An ultrasonic distance measure to the water surface below, an infrared temperature sensor, and an unshielded air temperature sensor.
COND:A direct measurement of the conductivity sampled at three ranges.
OBS:Optical Back Scatter measurement of the reflectance of the water at three wavelengths: visible red, green, and blue.
pH:An ion exchange pH probe.
DO:Dissolved Oxygen
voltage:Battery voltage"""
rths_descriptions = dict([ [a[0], a[1]] for a in [c.split(":") for c in rths_descriptions.split("\n")]])

# select distinct(VariableUnitsID), UnitsAbbreviation from seriescatalog, units where seriescatalog.VariableUnitsID = units.UnitsID;
units = { 54:"mm", 47:"cm", 168:"V", 199:"mg/l", 309:"pH", 100:"s", 52:"m", 1:"%",2:"deg",90:"mbar",96:"degC",119:"m/s", 192:"uS/cm",
          221:"NTU", 257:"#", 121:'mm/hr', 378:'A', 305:"mm/day" }

<|endoftext|>


<commit_before>''' Script to create and populate new run directory.
Created on 25 Jun 2010

@author: Ian Huston
'''
from pyflation import configuration
import os
import os.path
import helpers
import logging
import sys
import time
from optparse import OptionParser
from setup import setup, setup_args

#Version information
from sys import version as python_version
from numpy import __version__ as numpy_version
from scipy import __version__ as scipy_version
from tables import __version__ as tables_version

provenance_template = """Provenance document for this Pyflation run
------------------------------------------

Bazaar Revision Control Information (if available)
-------------------------------------------------
Branch name: %(nick)s
Branch revision number: %(revno)s
Branch revision id: %(revid)s

Code Directory Information
--------------------------
Original code directory: %(codedir)s
New run directory: %(newrundir)s
Date run directory was created: %(now)s

Version information at time of run creation
-------------------------------------------
Python version: %(python_version)s
Numpy version: %(numpy_version)s
Scipy version: %(scipy_version)s
PyTables version: %(tables_version)s

This information added on: %(now)s.
-----------------------------------------------

"""

def create_run_directory(newrundir, codedir, bzr_checkout=False):
    """Create the run directory using `newdir` as directory name."""
    if os.path.isdir(newrundir):
        raise IOError("New run directory already exists!")

    try:
        helpers.ensurepath(newrundir)
        os.makedirs(newrundir)
    except OSError:
        logging.error("Creating new run directory failed.")
        raise

    resultsdir = os.path.join(newrundir, configuration.RESULTSDIRNAME)
    logdir = os.path.join(newrundir, configuration.LOGDIRNAME)
    qsublogsdir = os.path.join(newrundir, configuration.QSUBLOGSDIRNAME)
    qsubscriptsdir = os.path.join(newrundir, configuration.QSUBSCRIPTSDIRNAME)
    #debug info
    logging.debug("resultsdir=%s, logdir=%s, qsublogsdir=%s, qsubscriptsdir=%s",
                  resultsdir, logdir, qsublogsdir, qsubscriptsdir)

    try:
        os.makedirs(resultsdir)
        os.makedirs(logdir)
        os.makedirs(qsublogsdir)
        os.makedirs(qsubscriptsdir)
    except OSError:
        logging.error("Creating subdirectories in new run directory failed.")
        raise

    #Try to do Bazaar checkout of code
    try:
        import bzrlib.export, bzrlib.workingtree
        bzr_available = True
    except ImportError:
        bzr_available = False

    logging.debug("bzr_available=%s", bzr_available)

    if bzr_available:
        mytree =  bzrlib.workingtree.WorkingTree.open(codedir)
        if bzr_checkout:
            newtree = mytree.branch.create_checkout(os.path.join(newrundir,
                            configuration.CODEDIRNAME), lightweight=True)
        else:
            bzrlib.export.export(mytree, os.path.join(newrundir, configuration.CODEDIRNAME))

    else:
        raise NotImplementedError("Bazaar is needed to copy code directory. Please do this manually.")

    #Try to run setup to create .so files
    try:
        olddir = os.getcwd()
        os.chdir(os.path.join(newrundir, configuration.CODEDIRNAME))
        logging.info("Preparing to compile non-python files.")
        setup(script_args=["build_ext", "-i"], **setup_args)
        os.chdir(olddir)
    except:
        logging.exception("Compiling additional modules did not work. Please do so by hand!")

    #Create provenance file detailing revision and branch used
    provenance_dict = dict(python_version=python_version,
                           numpy_version=numpy_version,
                           scipy_version=scipy_version,
                           tables_version=tables_version,
                           codedir=codedir,
                           newrundir=newrundir,
                           now=time.strftime("%Y/%m/%d %H:%M:%S %Z"))
    if bzr_available:
        provenance_dict["nick"] = mytree.branch.nick
        provenance_dict["revno"] = mytree.branch.revno()
        provenance_dict["revid"] = mytree.branch.last_revision()
    else:
        provenance_dict["nick"] = "Unavailable"
        provenance_dict["revno"] = "Unavailable"
        provenance_dict["revid"] = "Unavailable"
    provenance_file = os.path.join(newrundir, configuration.LOGDIRNAME,
                                   configuration.provenancefilename)
    with open(provenance_file, "w") as f:
        f.write(provenance_template % provenance_dict)
        logging.info("Created provenance file %s." % provenance_file)

    return

def main(argv = None):
    """Check command line options and start directory creation."""
    if not argv:
        argv = sys.argv
    #Parse command line options
    parser = OptionParser()

    parser.set_defaults(loglevel=configuration.LOGLEVEL)

    parser.add_option("-d", "--dir", dest="dir", default=os.getcwd(),
                  help="create run directory in DIR, default is current directory", metavar="DIR")
    parser.add_option("-n", "--name", dest="dirname",
                      help="new run directory name")
    parser.add_option("-c", "--codedir", dest="codedir",
                  help="copy code from CODEDIR", metavar="CODEDIR")
    parser.add_option("-q", "--quiet",
                  action="store_const", const=logging.FATAL, dest="loglevel",
                  help="only print fatal error messages")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=logging.INFO, dest="loglevel",
                  help="print informative messages")
    parser.add_option("--debug",
                  action="store_const", const=logging.DEBUG, dest="loglevel",
                  help="print lots of debugging information")
    parser.add_option("--checkout", action="store_true", dest="bzr_checkout",
                      default=True, help="create a bzr checkout instead of export")

    (options, args) = parser.parse_args(args=argv[1:])

    logging.basicConfig(level=options.loglevel)

    if not os.path.isdir(options.dir):
        raise IOError("Please check that parent directory %s exists." % options.dir)

    if options.dirname:
        newdir = os.path.join(options.dir, options.dirname)
        logging.debug("Variable newdir specified with value %s.", newdir)
    else:
        newdir = os.path.join(options.dir, time.strftime("%Y%m%d%H%M%S"))
        logging.debug("Variable newdir created with value %s", newdir)

    if options.codedir:
        codedir = options.codedir
        logging.debug("Option codedir specified with value %s.", options.codedir)
    else:
        codedir = os.path.join(configuration.CODEDIR)
        logging.debug("Variable codedir created with value %s.", codedir)

    try:
        create_run_directory(newdir, codedir, options.bzr_checkout)
    except Exception, e:
        logging.critical("Something went wrong! Quitting.")
        sys.exit(e)

    logging.info("New run directory created successfully.")
    return

if __name__ == '__main__':
    main()
    <commit_msg>Change import statements to import from pyflation.<commit_after>''' Script to create and populate new run directory.
Created on 25 Jun 2010

@author: Ian Huston
'''

import os
import os.path
import logging
import sys
import time
from optparse import OptionParser

#Version information
from sys import version as python_version
from numpy import __version__ as numpy_version
from scipy import __version__ as scipy_version
from tables import __version__ as tables_version

#Local modules from pyflation package
from pyflation import configuration, helpers
from pyflation.setup import setup, setup_args

provenance_template = """Provenance document for this Pyflation run
------------------------------------------

Bazaar Revision Control Information (if available)
-------------------------------------------------
Branch name: %(nick)s
Branch revision number: %(revno)s
Branch revision id: %(revid)s

Code Directory Information
--------------------------
Original code directory: %(codedir)s
New run directory: %(newrundir)s
Date run directory was created: %(now)s

Version information at time of run creation
-------------------------------------------
Python version: %(python_version)s
Numpy version: %(numpy_version)s
Scipy version: %(scipy_version)s
PyTables version: %(tables_version)s

This information added on: %(now)s.
-----------------------------------------------

"""

def create_run_directory(newrundir, codedir, bzr_checkout=False):
    """Create the run directory using `newdir` as directory name."""
    if os.path.isdir(newrundir):
        raise IOError("New run directory already exists!")

    try:
        helpers.ensurepath(newrundir)
        os.makedirs(newrundir)
    except OSError:
        logging.error("Creating new run directory failed.")
        raise

    resultsdir = os.path.join(newrundir, configuration.RESULTSDIRNAME)
    logdir = os.path.join(newrundir, configuration.LOGDIRNAME)
    qsublogsdir = os.path.join(newrundir, configuration.QSUBLOGSDIRNAME)
    qsubscriptsdir = os.path.join(newrundir, configuration.QSUBSCRIPTSDIRNAME)
    #debug info
    logging.debug("resultsdir=%s, logdir=%s, qsublogsdir=%s, qsubscriptsdir=%s",
                  resultsdir, logdir, qsublogsdir, qsubscriptsdir)

    try:
        os.makedirs(resultsdir)
        os.makedirs(logdir)
        os.makedirs(qsublogsdir)
        os.makedirs(qsubscriptsdir)
    except OSError:
        logging.error("Creating subdirectories in new run directory failed.")
        raise

    #Try to do Bazaar checkout of code
    try:
        import bzrlib.export, bzrlib.workingtree
        bzr_available = True
    except ImportError:
        bzr_available = False

    logging.debug("bzr_available=%s", bzr_available)

    if bzr_available:
        mytree =  bzrlib.workingtree.WorkingTree.open(codedir)
        if bzr_checkout:
            newtree = mytree.branch.create_checkout(os.path.join(newrundir,
                            configuration.CODEDIRNAME), lightweight=True)
        else:
            bzrlib.export.export(mytree, os.path.join(newrundir, configuration.CODEDIRNAME))

    else:
        raise NotImplementedError("Bazaar is needed to copy code directory. Please do this manually.")

    #Try to run setup to create .so files
    try:
        olddir = os.getcwd()
        os.chdir(os.path.join(newrundir, configuration.CODEDIRNAME))
        logging.info("Preparing to compile non-python files.")
        setup(script_args=["build_ext", "-i"], **setup_args)
        os.chdir(olddir)
    except:
        logging.exception("Compiling additional modules did not work. Please do so by hand!")

    #Create provenance file detailing revision and branch used
    provenance_dict = dict(python_version=python_version,
                           numpy_version=numpy_version,
                           scipy_version=scipy_version,
                           tables_version=tables_version,
                           codedir=codedir,
                           newrundir=newrundir,
                           now=time.strftime("%Y/%m/%d %H:%M:%S %Z"))
    if bzr_available:
        provenance_dict["nick"] = mytree.branch.nick
        provenance_dict["revno"] = mytree.branch.revno()
        provenance_dict["revid"] = mytree.branch.last_revision()
    else:
        provenance_dict["nick"] = "Unavailable"
        provenance_dict["revno"] = "Unavailable"
        provenance_dict["revid"] = "Unavailable"
    provenance_file = os.path.join(newrundir, configuration.LOGDIRNAME,
                                   configuration.provenancefilename)
    with open(provenance_file, "w") as f:
        f.write(provenance_template % provenance_dict)
        logging.info("Created provenance file %s." % provenance_file)

    return

def main(argv = None):
    """Check command line options and start directory creation."""
    if not argv:
        argv = sys.argv
    #Parse command line options
    parser = OptionParser()

    parser.set_defaults(loglevel=configuration.LOGLEVEL)

    parser.add_option("-d", "--dir", dest="dir", default=os.getcwd(),
                  help="create run directory in DIR, default is current directory", metavar="DIR")
    parser.add_option("-n", "--name", dest="dirname",
                      help="new run directory name")
    parser.add_option("-c", "--codedir", dest="codedir",
                  help="copy code from CODEDIR", metavar="CODEDIR")
    parser.add_option("-q", "--quiet",
                  action="store_const", const=logging.FATAL, dest="loglevel",
                  help="only print fatal error messages")
    parser.add_option("-v", "--verbose",
                  action="store_const", const=logging.INFO, dest="loglevel",
                  help="print informative messages")
    parser.add_option("--debug",
                  action="store_const", const=logging.DEBUG, dest="loglevel",
                  help="print lots of debugging information")
    parser.add_option("--checkout", action="store_true", dest="bzr_checkout",
                      default=True, help="create a bzr checkout instead of export")

    (options, args) = parser.parse_args(args=argv[1:])

    logging.basicConfig(level=options.loglevel)

    if not os.path.isdir(options.dir):
        raise IOError("Please check that parent directory %s exists." % options.dir)

    if options.dirname:
        newdir = os.path.join(options.dir, options.dirname)
        logging.debug("Variable newdir specified with value %s.", newdir)
    else:
        newdir = os.path.join(options.dir, time.strftime("%Y%m%d%H%M%S"))
        logging.debug("Variable newdir created with value %s", newdir)

    if options.codedir:
        codedir = options.codedir
        logging.debug("Option codedir specified with value %s.", options.codedir)
    else:
        codedir = os.path.join(configuration.CODEDIR)
        logging.debug("Variable codedir created with value %s.", codedir)

    try:
        create_run_directory(newdir, codedir, options.bzr_checkout)
    except Exception, e:
        logging.critical("Something went wrong! Quitting.")
        sys.exit(e)

    logging.info("New run directory created successfully.")
    return

if __name__ == '__main__':
    main()
    <|endoftext|>


<commit_before>#!/usr/local/bin/python

import cgi, copy, datetime, getopt, os, stat, sys, time
if os.getenv('REQUEST_METHOD'):  # is this apache?  # pragma: no cover
    import cgitb; cgitb.enable()

import config
import dbhand
import logger
import render
import secure
import useful

# The file environ.py modifies the environment upon first import.
# It sets PYTHON_EGG_CACHE; it adds /usr/local/bin to PATH;
# and it sets DOCUMENT_ROOT and SERVER_NAME if this is being run
# from the command line.  Since those change based on environment,
# environ.py has not be checked into github.
import environ

crawlers = [  # precluded from normal url tracking
    'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)',
    'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
    'Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)',
    'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    'Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)',
    'Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)',
    'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
    'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
    'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0;  http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    'ia_archiver',
    'tbot-nutch/Nutch-1.10',
]

class BaseForm:
    def __init__(self, cgi_form, args):
        '''Reads the cgi form and puts it into this object.'''
        form = dict()
        if 'REQUEST_METHOD' in os.environ:  # is this apache?
            for key in cgi_form.keys():
		field = cgi_form[key]
                if isinstance(field, list):
                    form.setdefault(key, list())
		    form[key].extend([elem.value for elem in field])
                elif key.endswith('.x'):
                    key_root = key[:-2]
                    if key_root + '.y' in cgi_form:
                        form[key_root] = (field.value, cgi_form[key_root + '.y'].value)
                    else:
                        form[key] = field.value
                elif key.endswith('.y'):
                    if key[:-2] + '.x' not in cgi_form:
                        form[key] = field.value
                elif 'filename' in field.__dict__:
                    form[key + '.name'] = field.filename
                    form[key] = field.value
                else:
                    form[key] = field.value

	else:
            for fl in args:
                if '=' in fl:
                    spl = fl.split('=')
                    form[spl[0]] = spl[1]
	self.form = form

    def __repr__(self):
	return self.form.__repr__()

    def __str__(self):
	return self.form.__str__()

    def set_val(self, key, val):
        self.form[key] = val

    def default(self, key, val):
        self.form.setdefault(key, val)

    def delete(self, key):
        if key in self.form:
            del self.form[key]

    def has(self, key):
        return key in self.form

    def get_form(self):
	return self.form

    def get(self, key, defval=None):
	return self.form.get(key, defval)

    def get_int(self, key, defval=0):
        try:
            return int(self.form[key])
        except:
            return int(defval)

    def get_bool(self, key, defval=False):
        try:
            return bool(self.form[key])
        except:
            return bool(defval)

    def get_str(self, key, defval=''):
        try:
            return str(self.form[key])
        except:
            return str(defval)

    def get_list(self, key=None, start=None, defval=None):
	ret = list()
	if key:
	    val = self.form.get(key, defval)
	    if isinstance(val, list):
		ret.extend(val)
	    elif val is not None:
		ret.append(val)
	if start:
	    ret.extend([(x[len(start):], self.get_str(x)) for x in self.keys(start=start)])
	return ret

    def keys(self, start='', end='', has=''):
        return filter(lambda x: x.startswith(start) and x.endswith(end) and has in x, self.form.keys())

    def find(self, field):
        keys = list()
        for key in self.form.keys():
            if key == field or key.startswith(field + '.'):
                keys.append(key)
        return keys

    def reformat(self, fields):
        return '&'.join(['%s=%s' % (x, self.get_str(x)) for x in fields])

    def where(self, cols=None, prefix=""):
        if not cols:
            cols = self.form.keys()
        wheres = list()
        for col in cols:
            if prefix + col in self.form:
                wheres.append(col + "='" + str(self.get_str(prefix + col)) + "'")
        return ' and '.join(wheres)

    def search(self, key):
        obj = self.form.get(key, "").split()
        nob = []
        col = ''
        for w in obj:
            if col:
                col = col + ' ' + w
                if col[-1] == '"':
                    nob.append(col[1:-1])
                    col = ''
            elif w[0] == '"' and w[-1] != '"':
                col = w
            else:
                nob.append(w)
        if col:
            nob.append(col[1:])
        return nob

    def checks(self, *args):
	return [self.get_bool(x) for x in args]

class PageInfoFile:
    def __init__(self, page_id, form_key='', defval='', args='', dbedit=None):
        self.render = self.dbh = None
        self.secure = secure.Security()
        self.htdocs = self.secure.docroot
        config.IS_BETA = self.secure.is_beta
        self.rawcookies = self.secure.get_cookies()
        user_id = self.rawcookies.get('id', '0')
	if isinstance(user_id, str):
	    user_id = eval(user_id)
	if isinstance(user_id, (int, long)):
	    self.id = user_id
	elif isinsance(user_id, tuple):
	    user_id = user_id[0]
	config.USER_ID = self.user_id = user_id
        self.args = args  # this is for unittest only!
	if not args:
	    self.argv = sys.argv[1:]  # this is for command line only!
        self.unittest = bool(args)
        self.form = BaseForm(cgi.FieldStorage(), args.split() if args else self.argv)
        self.page_id = self.get_page_id(page_id, form_key, defval)
        self.page_name = self.page_id[self.page_id.rfind('.') + 1:]
        self.time_start = datetime.datetime.now().strftime('%Y%m%d.%H%M%S')
        self.request_uri = os.environ.get('REQUEST_URI', 'unknown')
        self.remote_host = os.environ.get('REMOTE_HOST', 'host_unset')
        self.remote_addr = os.environ.get('REMOTE_ADDR', '127.0.0.1')
        self.set_server_env()
	self.log = logger.Logger()
        self.format_type = 'python'
        self.render = render.Presentation(self.page_id, self.form.get_int('verbose'))
        self.render.secure = self.secure
        self.render.comment('form', self.form.get_form())
        self.privs = self.rawcookies.get('pr', '')
        self.secure.cookies = self.rawcookies.get('co')
        if self.is_allowed(dbedit):
            self.secure.set_config('edit')

        os.chdir(self.secure.docroot)
        self.cwd = os.getcwd()
        self.render.is_beta = self.secure.is_beta
        self.cgibin = '../cgi-bin'
        self.render.simple = int(self.form.get_int("simple"))

        self.dbh = dbhand.DBHandler(self.secure.config, self.user_id, self.log.dbq, self.render.verbose)
        self.dbh.dbi.nowrites = self.unittest
        self.render.set_page_info(self.dbh.fetch_page(self.page_id))
        self.render.not_released = (self.render.flags & self.dbh.FLAG_PAGE_INFO_NOT_RELEASED) != 0
        self.render.hide_title = (self.render.flags & self.dbh.FLAG_PAGE_INFO_HIDE_TITLE) != 0
	self.render.is_admin = self.is_allowed('a')
	self.render.is_moderator = self.is_allowed('m')
	self.render.is_user = self.is_allowed('u')
	self.render.is_viewer = self.is_allowed('v')
        self.log_start()
        if 'REQUEST_METHOD' not in os.environ:  # not apache!
	    useful.header_done(is_web=False)

    def set_server_env(self):
        self.server_name = os.environ.get('SERVER_NAME', 'unset.server.name')
        parts = self.server_name.split('.')
        if len(parts) > 2:
            config.ENV = parts[-3]
        elif len(parts) == 2:
            config.ENV = 'www'

    def log_start(self):
        if not self.is_allowed('m') and not self.args:
	    if os.getenv('HTTP_USER_AGENT', '') in crawlers:
		self.log.bot.info('%s %s' % (self.remote_addr, self.request_uri))
	    else:
		self.dbh.increment_counter(self.page_id)
		self.log.count.info(self.page_id)
		self.log.url.info('%s %s' % (self.remote_addr, self.request_uri))
		self.log.debug.info(os.getenv('HTTP_USER_AGENT'))
		refer = os.environ.get('HTTP_REFERER', '')
		if refer and not refer.startswith('http://www.bamca.org') and \
			     not refer.startswith('http://bamca.org') and \
			     not refer.startswith('http://beta.bamca.org'):
		    self.log.refer.info(refer)

    def get_page_id(self, page_id, form_key, defval):
        if form_key:
            if self.form.get_str(form_key):
                if self.form.get_str(form_key).startswith(page_id + '.'):
                    return self.form.get_str(form_key)
                else:
                    return page_id + '.' + self.form.get_str(form_key)
            elif defval:
                return page_id + '.' + defval
        elif not form_key and self.form.has('page'):
            return self.form.get_str('page')
        return page_id

    # -- access control -------------------------------------------------

    def is_allowed(self, priv):
        if priv is None:  # None = never allowed
            return False
        if priv == '':  # '' = always allowed
            return True
        if set(priv) & set(self.privs):
            self.render.comment('is_allowed', priv, self.privs, 'YES')
            return True
        return False

    def restrict(self, priv):  # pragma: no cover
        if not self.is_allowed(priv):
	    raise useful.Redirect('/')

    # -- debugging and error handling -----------------------------------

    def dump(self, verbose=False):
        if self.render.verbose or verbose:
            useful.dump_dict_comment('pifile', self.__dict__)
            useful.dump_dict_comment('pifile.render', self.render.__dict__)
            useful.dump_dict_comment('pifile.dbh', self.dbh.__dict__)

    def error_report(self):
        ostr = 'pifile = ' + str(self.__dict__) + '\n'
        ostr += 'render = ' + self.render.error_report() + '\n'
        ostr += 'dbh = ' + self.dbh.error_report() + '\n'
        return ostr

    def show_error(self):
	useful.show_error()

#---- -------------------------------------------------------------------

if __name__ == '__main__':  # pragma: no cover
    print '''Content-Type: text/html\n\n'''
<commit_msg>More bots<commit_after>#!/usr/local/bin/python

import cgi, copy, datetime, getopt, os, stat, sys, time
if os.getenv('REQUEST_METHOD'):  # is this apache?  # pragma: no cover
    import cgitb; cgitb.enable()

import config
import dbhand
import logger
import render
import secure
import useful

# The file environ.py modifies the environment upon first import.
# It sets PYTHON_EGG_CACHE; it adds /usr/local/bin to PATH;
# and it sets DOCUMENT_ROOT and SERVER_NAME if this is being run
# from the command line.  Since those change based on environment,
# environ.py has not be checked into github.
import environ

crawlers = [  # precluded from normal url tracking
    'Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)',
    'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
    'Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)',
    'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    'Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)',
    'Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)',
    'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
    'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
    'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0;  http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    'SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)',
    'bhcBot',
    'ia_archiver',
    'tbot-nutch/Nutch-1.10',
]

class BaseForm:
    def __init__(self, cgi_form, args):
        '''Reads the cgi form and puts it into this object.'''
        form = dict()
        if 'REQUEST_METHOD' in os.environ:  # is this apache?
            for key in cgi_form.keys():
		field = cgi_form[key]
                if isinstance(field, list):
                    form.setdefault(key, list())
		    form[key].extend([elem.value for elem in field])
                elif key.endswith('.x'):
                    key_root = key[:-2]
                    if key_root + '.y' in cgi_form:
                        form[key_root] = (field.value, cgi_form[key_root + '.y'].value)
                    else:
                        form[key] = field.value
                elif key.endswith('.y'):
                    if key[:-2] + '.x' not in cgi_form:
                        form[key] = field.value
                elif 'filename' in field.__dict__:
                    form[key + '.name'] = field.filename
                    form[key] = field.value
                else:
                    form[key] = field.value

	else:
            for fl in args:
                if '=' in fl:
                    spl = fl.split('=')
                    form[spl[0]] = spl[1]
	self.form = form

    def __repr__(self):
	return self.form.__repr__()

    def __str__(self):
	return self.form.__str__()

    def set_val(self, key, val):
        self.form[key] = val

    def default(self, key, val):
        self.form.setdefault(key, val)

    def delete(self, key):
        if key in self.form:
            del self.form[key]

    def has(self, key):
        return key in self.form

    def get_form(self):
	return self.form

    def get(self, key, defval=None):
	return self.form.get(key, defval)

    def get_int(self, key, defval=0):
        try:
            return int(self.form[key])
        except:
            return int(defval)

    def get_bool(self, key, defval=False):
        try:
            return bool(self.form[key])
        except:
            return bool(defval)

    def get_str(self, key, defval=''):
        try:
            return str(self.form[key])
        except:
            return str(defval)

    def get_list(self, key=None, start=None, defval=None):
	ret = list()
	if key:
	    val = self.form.get(key, defval)
	    if isinstance(val, list):
		ret.extend(val)
	    elif val is not None:
		ret.append(val)
	if start:
	    ret.extend([(x[len(start):], self.get_str(x)) for x in self.keys(start=start)])
	return ret

    def keys(self, start='', end='', has=''):
        return filter(lambda x: x.startswith(start) and x.endswith(end) and has in x, self.form.keys())

    def find(self, field):
        keys = list()
        for key in self.form.keys():
            if key == field or key.startswith(field + '.'):
                keys.append(key)
        return keys

    def reformat(self, fields):
        return '&'.join(['%s=%s' % (x, self.get_str(x)) for x in fields])

    def where(self, cols=None, prefix=""):
        if not cols:
            cols = self.form.keys()
        wheres = list()
        for col in cols:
            if prefix + col in self.form:
                wheres.append(col + "='" + str(self.get_str(prefix + col)) + "'")
        return ' and '.join(wheres)

    def search(self, key):
        obj = self.form.get(key, "").split()
        nob = []
        col = ''
        for w in obj:
            if col:
                col = col + ' ' + w
                if col[-1] == '"':
                    nob.append(col[1:-1])
                    col = ''
            elif w[0] == '"' and w[-1] != '"':
                col = w
            else:
                nob.append(w)
        if col:
            nob.append(col[1:])
        return nob

    def checks(self, *args):
	return [self.get_bool(x) for x in args]

class PageInfoFile:
    def __init__(self, page_id, form_key='', defval='', args='', dbedit=None):
        self.render = self.dbh = None
        self.secure = secure.Security()
        self.htdocs = self.secure.docroot
        config.IS_BETA = self.secure.is_beta
        self.rawcookies = self.secure.get_cookies()
        user_id = self.rawcookies.get('id', '0')
	if isinstance(user_id, str):
	    user_id = eval(user_id)
	if isinstance(user_id, (int, long)):
	    self.id = user_id
	elif isinsance(user_id, tuple):
	    user_id = user_id[0]
	config.USER_ID = self.user_id = user_id
        self.args = args  # this is for unittest only!
	if not args:
	    self.argv = sys.argv[1:]  # this is for command line only!
        self.unittest = bool(args)
        self.form = BaseForm(cgi.FieldStorage(), args.split() if args else self.argv)
        self.page_id = self.get_page_id(page_id, form_key, defval)
        self.page_name = self.page_id[self.page_id.rfind('.') + 1:]
        self.time_start = datetime.datetime.now().strftime('%Y%m%d.%H%M%S')
        self.request_uri = os.environ.get('REQUEST_URI', 'unknown')
        self.remote_host = os.environ.get('REMOTE_HOST', 'host_unset')
        self.remote_addr = os.environ.get('REMOTE_ADDR', '127.0.0.1')
        self.set_server_env()
	self.log = logger.Logger()
        self.format_type = 'python'
        self.render = render.Presentation(self.page_id, self.form.get_int('verbose'))
        self.render.secure = self.secure
        self.render.comment('form', self.form.get_form())
        self.privs = self.rawcookies.get('pr', '')
        self.secure.cookies = self.rawcookies.get('co')
        if self.is_allowed(dbedit):
            self.secure.set_config('edit')

        os.chdir(self.secure.docroot)
        self.cwd = os.getcwd()
        self.render.is_beta = self.secure.is_beta
        self.cgibin = '../cgi-bin'
        self.render.simple = int(self.form.get_int("simple"))

        self.dbh = dbhand.DBHandler(self.secure.config, self.user_id, self.log.dbq, self.render.verbose)
        self.dbh.dbi.nowrites = self.unittest
        self.render.set_page_info(self.dbh.fetch_page(self.page_id))
        self.render.not_released = (self.render.flags & self.dbh.FLAG_PAGE_INFO_NOT_RELEASED) != 0
        self.render.hide_title = (self.render.flags & self.dbh.FLAG_PAGE_INFO_HIDE_TITLE) != 0
	self.render.is_admin = self.is_allowed('a')
	self.render.is_moderator = self.is_allowed('m')
	self.render.is_user = self.is_allowed('u')
	self.render.is_viewer = self.is_allowed('v')
        self.log_start()
        if 'REQUEST_METHOD' not in os.environ:  # not apache!
	    useful.header_done(is_web=False)

    def set_server_env(self):
        self.server_name = os.environ.get('SERVER_NAME', 'unset.server.name')
        parts = self.server_name.split('.')
        if len(parts) > 2:
            config.ENV = parts[-3]
        elif len(parts) == 2:
            config.ENV = 'www'

    def log_start(self):
        if not self.is_allowed('m') and not self.args:
	    if os.getenv('HTTP_USER_AGENT', '') in crawlers:
		self.log.bot.info('%s %s' % (self.remote_addr, self.request_uri))
	    else:
		self.dbh.increment_counter(self.page_id)
		self.log.count.info(self.page_id)
		self.log.url.info('%s %s' % (self.remote_addr, self.request_uri))
		self.log.debug.info(os.getenv('HTTP_USER_AGENT'))
		refer = os.environ.get('HTTP_REFERER', '')
		if refer and not refer.startswith('http://www.bamca.org') and \
			     not refer.startswith('http://bamca.org') and \
			     not refer.startswith('http://beta.bamca.org'):
		    self.log.refer.info(refer)

    def get_page_id(self, page_id, form_key, defval):
        if form_key:
            if self.form.get_str(form_key):
                if self.form.get_str(form_key).startswith(page_id + '.'):
                    return self.form.get_str(form_key)
                else:
                    return page_id + '.' + self.form.get_str(form_key)
            elif defval:
                return page_id + '.' + defval
        elif not form_key and self.form.has('page'):
            return self.form.get_str('page')
        return page_id

    # -- access control -------------------------------------------------

    def is_allowed(self, priv):
        if priv is None:  # None = never allowed
            return False
        if priv == '':  # '' = always allowed
            return True
        if set(priv) & set(self.privs):
            self.render.comment('is_allowed', priv, self.privs, 'YES')
            return True
        return False

    def restrict(self, priv):  # pragma: no cover
        if not self.is_allowed(priv):
	    raise useful.Redirect('/')

    # -- debugging and error handling -----------------------------------

    def dump(self, verbose=False):
        if self.render.verbose or verbose:
            useful.dump_dict_comment('pifile', self.__dict__)
            useful.dump_dict_comment('pifile.render', self.render.__dict__)
            useful.dump_dict_comment('pifile.dbh', self.dbh.__dict__)

    def error_report(self):
        ostr = 'pifile = ' + str(self.__dict__) + '\n'
        ostr += 'render = ' + self.render.error_report() + '\n'
        ostr += 'dbh = ' + self.dbh.error_report() + '\n'
        return ostr

    def show_error(self):
	useful.show_error()

#---- -------------------------------------------------------------------

if __name__ == '__main__':  # pragma: no cover
    print '''Content-Type: text/html\n\n'''
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Poio Corpus
#
# Copyright (C) 2009-2013 Poio Project
# Author: Peter Bouda <pbouda@cidles.eu>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import sys
import os
import glob
import pickle

from config import Config
import rdflib

###################################### Main

def main(argv):
    config_file = file(os.path.join('..', 'config.ini'))
    config = Config(config_file)

    languages_data = {}
    for lang in config['supported_languages']:
        iso3 = lang['iso_639_3']
        iso1 = lang['iso_639_1']

        language_info = get_language_info(iso1, iso3)
        languages_data[iso3] = language_info

    langinfo_dir = os.path.join("..", "build", "langinfo")
    pickle_file = os.path.join(langinfo_dir, "languages_data.pickle")
    with open(pickle_file, "wb") as f:
        pickle.dump(languages_data, f)

###################################### Helpers

def get_language_info(iso1, iso3):
    corpus_dir = os.path.abspath(os.path.join("..", "build", "corpus"))
    corpus_files = []
    for f in glob.glob(os.path.join(corpus_dir, "{0}*.zip".format(iso1))):
        corpus_files.append(os.path.basename(f))
    language_info = get_info_for_iso(iso3)
    language_info['files'] = corpus_files
    return language_info

def get_info_for_iso(iso):
    rdf = RdfIso(iso)
    (latitude, longitude) = rdf.geo()
    label = rdf.label()
    rv = {
        'label' : label,
        'geo': {
            'lat': latitude,
            'lon': longitude
        }
    }
    return rv

class RdfIso:

    def __init__(self, iso):
        self.g = rdflib.Graph()
        result = self.g.parse(
            "http://glottolog.org/resource/languoid/iso/{0}.rdf".format(iso))
        self.subject = self.g.subjects().next()

    def geo(self):
        latitude = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2003/01/geo/wgs84_pos#lat"))

        longitude = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2003/01/geo/wgs84_pos#long"))

        return (latitude.toPython(), longitude.toPython())

    def label(self):
        name = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2004/02/skos/core#prefLabel"))
        return name.toPython()

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>Updated langinfo update script for new config file.<commit_after># -*- coding: utf-8 -*-
#
# Poio Corpus
#
# Copyright (C) 2009-2013 Poio Project
# Author: Peter Bouda <pbouda@cidles.eu>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import sys
import os
import glob
import pickle
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

import rdflib

###################################### Main

def main(argv):
    config_file = os.path.join('..', 'config.ini')
    config = configparser.ConfigParser()
    config.read(config_file)

    languages_data = {}

    for iso_639_3 in config.options("LanguagesISOMap"):

        language_info = get_language_info(iso_639_3)
        languages_data[iso_639_3] = language_info

    langinfo_dir = os.path.join("..", "build", "langinfo")
    pickle_file = os.path.join(langinfo_dir, "languages_data.pickle")
    with open(pickle_file, "wb") as f:
        pickle.dump(languages_data, f)

###################################### Helpers

def get_language_info(iso):
    corpus_dir = os.path.abspath(os.path.join("..", "build", "corpus"))
    corpus_files = []
    for f in glob.glob(os.path.join(corpus_dir, "{0}*.zip".format(iso))):
        corpus_files.append(os.path.basename(f))
    language_info = get_info_for_iso(iso)
    language_info['files'] = corpus_files
    return language_info

def get_info_for_iso(iso):
    rdf = RdfIso(iso)
    (latitude, longitude) = rdf.geo()
    label = rdf.label()
    rv = {
        'label' : label,
        'geo': {
            'lat': latitude,
            'lon': longitude
        }
    }
    return rv

class RdfIso:

    def __init__(self, iso):
        self.g = rdflib.Graph()
        result = self.g.parse(
            "http://glottolog.org/resource/languoid/iso/{0}.rdf".format(iso))
        self.subject = self.g.subjects().next()

    def geo(self):
        latitude = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2003/01/geo/wgs84_pos#lat"))

        longitude = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2003/01/geo/wgs84_pos#long"))

        return (latitude.toPython(), longitude.toPython())

    def label(self):
        name = self.g.value(
            self.subject,
            rdflib.term.URIRef(
                u"http://www.w3.org/2004/02/skos/core#prefLabel"))
        return name.toPython()

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>'''
Modular Input Script for Qumulo
'''

from datetime import datetime
import json
import math
import os
import sys
import logging
import time
import xml.dom.minidom

#  qumulo_client wraps all of the Qumulo REST API interactions
from qumulo_client import QumuloClient
from qumulo.lib.request import RequestError

SPLUNK_HOME = os.environ.get("SPLUNK_HOME")
STANZA = None
EGG_DIR = SPLUNK_HOME + "/etc/apps/qumulo_splunk_app/bin/"

# Import any Eggs
for filename in os.listdir(EGG_DIR):
    if filename.endswith(".egg"):
        sys.path.append(EGG_DIR + filename)

from croniter import croniter

#set up logging
logging.root
logging.root.setLevel(logging.ERROR)
formatter = logging.Formatter('%(levelname)s %(message)s')
#with zero args , should go to STD ERR
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.root.addHandler(handler)

SCHEME = """<scheme>
    <title>Qumulo</title>
    <description>Poll data from the Qumulo REST API</description>
    <use_external_validation>true</use_external_validation>
    <streaming_mode>xml</streaming_mode>
    <use_single_instance>false</use_single_instance>

    <endpoint>
        <args>
            <arg name="name">
                <title>Qumulo input name</title>
                <description>Name of this Qumulo input</description>
            </arg>

            <arg name="username">
                <title>Username</title>
                <description>username for authentication , defaults to admin</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="password">
                <title>Password</title>
                <description>password for authentication , defaults to admin</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="nodehost">
                <title>Host</title>
                <description>An ip address or hostname of a node in the cluster; use 'localhost' when running directly on the node</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="port">
                <title>Port</title>
                <description>port number for the API server port, defaults to 8000 </description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

            <arg name="endpoint_to_poll">
                <title>Endpoint</title>
                <description>Qumulo REST endpoint to poll</description>
                <required_on_edit>true</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

            <arg name="request_body">
                <title>Custom Request Body</title>
                <description>Request body payload for POST and PUT</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="request_headers">
                <title>Custom Request Headers</title>
                <description>in format prop=value,prop2=value2</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>
            <arg name="request_url_args">
                <title>Custom Request URL Args</title>
                <description>in format arg=value,arg2=value2</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="polling_interval">
                <title>Polling Interval</title>
                <description>Interval time in seconds or a CRON pattern to poll the endpoint</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="delimiter">
                <title>Delimiter</title>
                <description>Delimiter to use for any multi "key=value" field inputs</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

        </args>
    </endpoint>
</scheme>
"""

def get_current_datetime_for_cron():
    current_dt = datetime.now()
    #dont need seconds/micros for cron
    current_dt = current_dt.replace(second=0, microsecond=0)
    return current_dt

def do_validate():
    config = get_validation_config()

def process_throughput():
    try:
        throughput = client.get_throughput()
    except RequestError, excpt:
        logging.error("Exception performing request for Throughput: %s" % str(excpt))
        return

    for entry in throughput:
        for i in range(len(entry['values'])):
            log_entry = {}
            log_entry['metric'] = entry['id']
            log_entry['time'] = entry['times'][i]
            log_entry['value'] = entry['values'][i]
            print_xml_stream(json.dumps(log_entry))

def process_iops():
    try:
        iops = client.get_iops()
    except RequestError, excpt:
        logging.error("Exception performing request for IOPS: %s" % str(excpt))
        return
    print_xml_stream(json.dumps(iops))

def process_capacity():
    try:
        capacity = client.get_capacity()
    except RequestError, excpt:
        logging.error("Exception performing request for Capacity: %s" % str(excpt))
        return

    cap = {}
    cap["free_gigabytes"] = int(long(float(capacity['free_size_bytes']))/math.pow(1024,3))
    cap["raw_gigabytes"] = int(long(float(capacity['raw_size_bytes']))/math.pow(1024,3))
    cap["total_gigabytes"] = int(long(float(capacity['total_size_bytes']))/math.pow(1024,3))
    print_xml_stream(json.dumps(cap))

def do_run(client):

    config = client.config

    #setup some globals
    global STANZA
    STANZA = config.get("name")

    #logical name of endpoint to poll
    endpoint_to_poll=config.get("endpoint_to_poll","")
    if endpoint_to_poll == "":
        logging.error("No polling endpoint was speciifed , exiting.")
        sys.exit(2)

    #polling can be a time interval or CRON pattern
    polling_interval_string = config.get("polling_interval","60")
    #defaults
    polling_interval = 60
    polling_type = 'interval'

    if polling_interval_string.isdigit():
        polling_type = 'interval'
        polling_interval=int(polling_interval_string)
    else:
        polling_type = 'cron'
        cron_start_date = datetime.now()
        cron_iter = croniter(polling_interval_string, cron_start_date)

    try:

        while True:

            if polling_type == 'cron':
                next_cron_firing = cron_iter.get_next(datetime)
                while get_current_datetime_for_cron() != next_cron_firing:
                    time.sleep(float(10))

            if(endpoint_to_poll == "iops"):
                process_iops()
                continue

            if(endpoint_to_poll == "capacity"):
                process_capacity()
                continue

            if(endpoint_to_poll == "throughput"):
                process_throughput()
                continue

            if polling_type == 'interval':
                time.sleep(float(polling_interval))

    except RuntimeError,e:
        logging.error("Looks like an error: %s" % str(e))
        sys.exit(2)

# prints validation error data to be consumed by Splunk
def print_validation_error(s):
    print "<error><message>%s</message></error>" % encodeXMLText(s)

# prints XML stream
def print_xml_stream(s):
    print "<stream><event unbroken=\"1\"><data>%s</data><done/></event></stream>" % encodeXMLText(s)

def encodeXMLText(text):
    text = text.replace("&", "&")
    text = text.replace("\"", """)
    text = text.replace("'", "'")
    text = text.replace("<", "<")
    text = text.replace(">", ">")
    text = text.replace("\n", "")
    return text

def usage():
    print "usage: %s [--scheme|--validate-arguments]"
    logging.error("Incorrect Program Usage")
    sys.exit(2)

def do_scheme():
    print SCHEME

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_input_config():
    config = {}

    try:
        # read everything from stdin
        config_str = sys.stdin.read()

        # parse the config XML
        doc = xml.dom.minidom.parseString(config_str)
        root = doc.documentElement

        session_key_node = root.getElementsByTagName("session_key")[0]
        if session_key_node and session_key_node.firstChild and session_key_node.firstChild.nodeType == session_key_node.firstChild.TEXT_NODE:
            data = session_key_node.firstChild.data
            config["session_key"] = data

        server_uri_node = root.getElementsByTagName("server_uri")[0]
        if server_uri_node and server_uri_node.firstChild and server_uri_node.firstChild.nodeType == server_uri_node.firstChild.TEXT_NODE:
            data = server_uri_node.firstChild.data
            config["server_uri"] = data

        conf_node = root.getElementsByTagName("configuration")[0]
        if conf_node:
            logging.debug("XML: found configuration")
            stanza = conf_node.getElementsByTagName("stanza")[0]
            if stanza:
                stanza_name = stanza.getAttribute("name")
                if stanza_name:
                    logging.debug("XML: found stanza " + stanza_name)
                    config["name"] = stanza_name

                    params = stanza.getElementsByTagName("param")
                    for param in params:
                        param_name = param.getAttribute("name")
                        logging.debug("XML: found param '%s'" % param_name)
                        if param_name and param.firstChild and \
                           param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                            data = param.firstChild.data
                            config[param_name] = data
                            logging.debug("XML: '%s' -> '%s'" % (param_name, data))

        checkpnt_node = root.getElementsByTagName("checkpoint_dir")[0]
        if checkpnt_node and checkpnt_node.firstChild and \
           checkpnt_node.firstChild.nodeType == checkpnt_node.firstChild.TEXT_NODE:
            config["checkpoint_dir"] = checkpnt_node.firstChild.data

        if not config:
            raise Exception, "Invalid configuration received from Splunk."

    except Exception, e:
        raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)

    return config

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_validation_config():
    val_data = {}

    # read everything from stdin
    val_str = sys.stdin.read()

    # parse the validation XML
    doc = xml.dom.minidom.parseString(val_str)
    root = doc.documentElement

    logging.debug("XML: found items")
    item_node = root.getElementsByTagName("item")[0]
    if item_node:
        logging.debug("XML: found item")

        name = item_node.getAttribute("name")
        val_data["stanza"] = name

        params_node = item_node.getElementsByTagName("param")
        for param in params_node:
            name = param.getAttribute("name")
            logging.debug("Found param %s" % name)
            if name and param.firstChild and \
               param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                val_data[name] = param.firstChild.data

    return val_data

if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "--scheme":
            do_scheme()
        elif sys.argv[1] == "--validate-arguments":
            do_validate()
        else:
            usage()
    else:
        config = get_input_config()
        client = QumuloClient(config)
        do_run(client)

    sys.exit(0)
<commit_msg>iops entries first version<commit_after>'''
Modular Input Script for Qumulo
'''

from datetime import datetime
import json
import math
import os
import sys
import logging
import time
import xml.dom.minidom

#  qumulo_client wraps all of the Qumulo REST API interactions
from qumulo_client import QumuloClient
from qumulo.lib.request import RequestError

SPLUNK_HOME = os.environ.get("SPLUNK_HOME")
STANZA = None
EGG_DIR = SPLUNK_HOME + "/etc/apps/qumulo_splunk_app/bin/"

# Import any Eggs
for filename in os.listdir(EGG_DIR):
    if filename.endswith(".egg"):
        sys.path.append(EGG_DIR + filename)

from croniter import croniter

#set up logging
logging.root
logging.root.setLevel(logging.ERROR)
formatter = logging.Formatter('%(levelname)s %(message)s')
#with zero args , should go to STD ERR
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.root.addHandler(handler)

SCHEME = """<scheme>
    <title>Qumulo</title>
    <description>Poll data from the Qumulo REST API</description>
    <use_external_validation>true</use_external_validation>
    <streaming_mode>xml</streaming_mode>
    <use_single_instance>false</use_single_instance>

    <endpoint>
        <args>
            <arg name="name">
                <title>Qumulo input name</title>
                <description>Name of this Qumulo input</description>
            </arg>

            <arg name="username">
                <title>Username</title>
                <description>username for authentication , defaults to admin</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="password">
                <title>Password</title>
                <description>password for authentication , defaults to admin</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="nodehost">
                <title>Host</title>
                <description>An ip address or hostname of a node in the cluster; use 'localhost' when running directly on the node</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>
            <arg name="port">
                <title>Port</title>
                <description>port number for the API server port, defaults to 8000 </description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

            <arg name="endpoint_to_poll">
                <title>Endpoint</title>
                <description>Qumulo REST endpoint to poll</description>
                <required_on_edit>true</required_on_edit>
                <required_on_create>true</required_on_create>
            </arg>

            <arg name="request_body">
                <title>Custom Request Body</title>
                <description>Request body payload for POST and PUT</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="request_headers">
                <title>Custom Request Headers</title>
                <description>in format prop=value,prop2=value2</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>
            <arg name="request_url_args">
                <title>Custom Request URL Args</title>
                <description>in format arg=value,arg2=value2</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="polling_interval">
                <title>Polling Interval</title>
                <description>Interval time in seconds or a CRON pattern to poll the endpoint</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

            <arg name="delimiter">
                <title>Delimiter</title>
                <description>Delimiter to use for any multi "key=value" field inputs</description>
                <required_on_edit>false</required_on_edit>
                <required_on_create>false</required_on_create>
            </arg>

        </args>
    </endpoint>
</scheme>
"""

def get_current_datetime_for_cron():
    current_dt = datetime.now()
    #dont need seconds/micros for cron
    current_dt = current_dt.replace(second=0, microsecond=0)
    return current_dt

def do_validate():
    config = get_validation_config()

def process_throughput():
    try:
        throughput = client.get_throughput()
    except RequestError, excpt:
        logging.error("Exception performing request for Throughput: %s" % str(excpt))
        return

    for entry in throughput:
        for i in range(len(entry['values'])):
            log_entry = {}
            log_entry['metric'] = entry['id']
            log_entry['time'] = entry['times'][i]
            log_entry['value'] = entry['values'][i]
            print_xml_stream(json.dumps(log_entry))

def process_iops():
    try:
        iops = client.get_iops()
    except RequestError, excpt:
        logging.error("Exception performing request for IOPS: %s" % str(excpt))
        return

    for entry in iops["entries"]:
        print_xml_stream(json.dumps(entry))

def process_capacity():
    try:
        capacity = client.get_capacity()
    except RequestError, excpt:
        logging.error("Exception performing request for Capacity: %s" % str(excpt))
        return

    cap = {}
    cap["free_gigabytes"] = int(long(float(capacity['free_size_bytes']))/math.pow(1024,3))
    cap["raw_gigabytes"] = int(long(float(capacity['raw_size_bytes']))/math.pow(1024,3))
    cap["total_gigabytes"] = int(long(float(capacity['total_size_bytes']))/math.pow(1024,3))
    print_xml_stream(json.dumps(cap))

def do_run(client):

    config = client.config

    #setup some globals
    global STANZA
    STANZA = config.get("name")

    #logical name of endpoint to poll
    endpoint_to_poll=config.get("endpoint_to_poll","")
    if endpoint_to_poll == "":
        logging.error("No polling endpoint was speciifed , exiting.")
        sys.exit(2)

    #polling can be a time interval or CRON pattern
    polling_interval_string = config.get("polling_interval","60")
    #defaults
    polling_interval = 60
    polling_type = 'interval'

    if polling_interval_string.isdigit():
        polling_type = 'interval'
        polling_interval=int(polling_interval_string)
    else:
        polling_type = 'cron'
        cron_start_date = datetime.now()
        cron_iter = croniter(polling_interval_string, cron_start_date)

    try:

        while True:

            if polling_type == 'cron':
                next_cron_firing = cron_iter.get_next(datetime)
                while get_current_datetime_for_cron() != next_cron_firing:
                    time.sleep(float(10))

            if(endpoint_to_poll == "iops"):
                process_iops()
                continue

            if(endpoint_to_poll == "capacity"):
                process_capacity()
                continue

            if(endpoint_to_poll == "throughput"):
                process_throughput()
                continue

            if polling_type == 'interval':
                time.sleep(float(polling_interval))

    except RuntimeError,e:
        logging.error("Looks like an error: %s" % str(e))
        sys.exit(2)

# prints validation error data to be consumed by Splunk
def print_validation_error(s):
    print "<error><message>%s</message></error>" % encodeXMLText(s)

# prints XML stream
def print_xml_stream(s):
    print "<stream><event unbroken=\"1\"><data>%s</data><done/></event></stream>" % encodeXMLText(s)

def encodeXMLText(text):
    text = text.replace("&", "&")
    text = text.replace("\"", """)
    text = text.replace("'", "'")
    text = text.replace("<", "<")
    text = text.replace(">", ">")
    text = text.replace("\n", "")
    return text

def usage():
    print "usage: %s [--scheme|--validate-arguments]"
    logging.error("Incorrect Program Usage")
    sys.exit(2)

def do_scheme():
    print SCHEME

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_input_config():
    config = {}

    try:
        # read everything from stdin
        config_str = sys.stdin.read()

        # parse the config XML
        doc = xml.dom.minidom.parseString(config_str)
        root = doc.documentElement

        session_key_node = root.getElementsByTagName("session_key")[0]
        if session_key_node and session_key_node.firstChild and session_key_node.firstChild.nodeType == session_key_node.firstChild.TEXT_NODE:
            data = session_key_node.firstChild.data
            config["session_key"] = data

        server_uri_node = root.getElementsByTagName("server_uri")[0]
        if server_uri_node and server_uri_node.firstChild and server_uri_node.firstChild.nodeType == server_uri_node.firstChild.TEXT_NODE:
            data = server_uri_node.firstChild.data
            config["server_uri"] = data

        conf_node = root.getElementsByTagName("configuration")[0]
        if conf_node:
            logging.debug("XML: found configuration")
            stanza = conf_node.getElementsByTagName("stanza")[0]
            if stanza:
                stanza_name = stanza.getAttribute("name")
                if stanza_name:
                    logging.debug("XML: found stanza " + stanza_name)
                    config["name"] = stanza_name

                    params = stanza.getElementsByTagName("param")
                    for param in params:
                        param_name = param.getAttribute("name")
                        logging.debug("XML: found param '%s'" % param_name)
                        if param_name and param.firstChild and \
                           param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                            data = param.firstChild.data
                            config[param_name] = data
                            logging.debug("XML: '%s' -> '%s'" % (param_name, data))

        checkpnt_node = root.getElementsByTagName("checkpoint_dir")[0]
        if checkpnt_node and checkpnt_node.firstChild and \
           checkpnt_node.firstChild.nodeType == checkpnt_node.firstChild.TEXT_NODE:
            config["checkpoint_dir"] = checkpnt_node.firstChild.data

        if not config:
            raise Exception, "Invalid configuration received from Splunk."

    except Exception, e:
        raise Exception, "Error getting Splunk configuration via STDIN: %s" % str(e)

    return config

#read XML configuration passed from splunkd, need to refactor to support single instance mode
def get_validation_config():
    val_data = {}

    # read everything from stdin
    val_str = sys.stdin.read()

    # parse the validation XML
    doc = xml.dom.minidom.parseString(val_str)
    root = doc.documentElement

    logging.debug("XML: found items")
    item_node = root.getElementsByTagName("item")[0]
    if item_node:
        logging.debug("XML: found item")

        name = item_node.getAttribute("name")
        val_data["stanza"] = name

        params_node = item_node.getElementsByTagName("param")
        for param in params_node:
            name = param.getAttribute("name")
            logging.debug("Found param %s" % name)
            if name and param.firstChild and \
               param.firstChild.nodeType == param.firstChild.TEXT_NODE:
                val_data[name] = param.firstChild.data

    return val_data

if __name__ == '__main__':

    if len(sys.argv) > 1:
        if sys.argv[1] == "--scheme":
            do_scheme()
        elif sys.argv[1] == "--validate-arguments":
            do_validate()
        else:
            usage()
    else:
        config = get_input_config()
        client = QumuloClient(config)
        do_run(client)

    sys.exit(0)
<|endoftext|>