<commit_before>from tsgettoolbox.odo import odo, resource
import pandas as pd
import mando

try:
    from mando.rst_text_formatter import RSTHelpFormatter as HelpFormatter
except ImportError:
    from argparse import RawTextHelpFormatter as HelpFormatter

from tstoolbox import tsutils

@mando.command("ldas", formatter_class=HelpFormatter, doctype="numpy")
def ldas_cli(
    lat=None,
    lon=None,
    xindex=None,
    yindex=None,
    variable=None,
    startDate=None,
    endDate=None,
):
    r"""Download data from NLDAS or GLDAS.

    The time zone is always UTC.

    Parameters
    ----------
    lat :  float
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.

        Latitude (required): Enter single geographic point by
        latitude.::

            Example: --lat=43.1

    lon : float
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  Longitude (required): Enter single
        geographic point by longitude::

            Example: --lon=-85.3

    xindex : int
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  xindex (required if using xindex/yindex):
        Enter the x index of the NLDAS or GLDAS grid.::

            Example: --xindex=301

    yindex : int
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  yindex (required if using xindex/yindex):
        Enter the y index of the NLDAS or GLDAS grid.::

            Example: --yindex=80

    variable : str
        Use the variable codes from the following table:

        +--------------------------------------------+-----------+
        | LDAS "variable" string Description         | Units     |
        +============================================+===========+
        | NLDAS:NLDAS_FORA0125_H.002:APCPsfc         | kg/m^2    |
        | Precipitation hourly total                 |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:DLWRFsfc        | W/m^2     |
        | Surface DW longwave radiation flux         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:DSWRFsfc        | W/m^2     |
        | Surface DW shortwave radiation flux        |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:PEVAPsfc        | kg/m^2    |
        | Potential evaporation                      |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:SPFH2m          | kg/kg     |
        | 2-m above ground specific humidity         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:TMP2m           | degK      |
        | 2-m above ground temperature               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:UGRD10m         | m/s       |
        | 10-m above ground zonal wind               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:VGRD10m         | m/s       |
        | 10-m above ground meridional wind          |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:EVPsfc          | kg/m^2    |
        | Total evapotranspiration                   |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:GFLUXsfc        | w/m^2     |
        | Ground heat flux                           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:LHTFLsfc        | w/m^2     |
        | Latent heat flux                           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SHTFLsfc        | w/m^2     |
        | Sensible heat flux                         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SSRUNsfc        | kg/m^2    |
        | Surface runoff (non-infiltrating)          |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:BGRIUNdfc       | kg/m^2    |
        | Subsurface runoff (baseflow)               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-10cm     | kg/m^2    |
        | 0-10 cm soil moisture content              |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-100cm    | kg/m^2    |
        | 0-100 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-200cm    | kg/m^2    |
        | 0-200 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM10-40cm    | kg/m^2    |
        | 10-40 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM40-100cm   | kg/m^2    |
        | 40-100 cm soil moisture content            |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM100-200cm  | kg/m^2    |
        | 100-200 cm soil moisture content           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:TSOIL0-10cm     | degK      |
        | 0-10 cm soil temperature                   |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Evap            | kg/m^2/s  |
        | Evapotranspiration                         |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:precip          | kg/m^s/hr |
        | Precipitation rate                         |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Rainf           | kg/m^2/s  |
        | Rain rate                                  |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Snowf           | kg/m^2/s  |
        | Snow rate                                  |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Qs              | kg/m^2/s  |
        | Surface Runoff                             |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Qsb             | kg/m^2/s  |
        | Subsurface Runoff                          |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM0-100cm    | kg/m^2    |
        | 0-100 cm top 1 meter soil moisture content |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM0-10cm     | kg/m^2    |
        | 0-10 cm layer 1 soil moisture content      |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM10-40cm    | kg/m^2    |
        | 10-40 cm layer 2 soil moisture content     |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM40-100cm   | kg/m^2    |
        | 40-100 cm layer 3 soil moisture content    |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Tair            | degK      |
        | Near surface air temperature               |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:TSOIL0-10cm     | degK      |
        | Average layer 1 soil temperature           |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Wind            | m/s       |
        | Near surface wind magnitude                |           |
        +--------------------------------------------+-----------+

    startDate : str
        The start date of the time series.::

            Example: --startDate=2001-01-01T05

        If startDate and endDate are None, returns the entire series.

    endDate : str
        The end date of the time series.::

            Example: --endDate=2002-01-05T05

        If startDate and endDate are None, returns the entire series.
    """
    tsutils._printiso(
        ldas(
            lat=lat,
            lon=lon,
            xindex=xindex,
            yindex=yindex,
            variable=variable,
            startDate=startDate,
            endDate=endDate,
        )
    )

@tsutils.validator(
    lat=[float, ["range", [-90.0, 90.0]], 1],
    lon=[float, ["range", [-180.0, 180.0]], 1],
    xindex=[int, ["range", [0,]], 1],
    yindex=[int, ["range", [0,]], 1],
    variable=[str, ["pass", []], None],
    startDate=[tsutils.parsedate, ["pass", []], 1],
    endDate=[tsutils.parsedate, ["pass", []], 1],
)
def ldas(
    lat=None,
    lon=None,
    xindex=None,
    yindex=None,
    variable=None,
    startDate=None,
    endDate=None,
):
    r"""Download data from NLDAS or GLDAS."""
    from tsgettoolbox.services import ldas as placeholder

    project = variable.split(":")[0]
    if lat is not None:
        location = "GEOM:POINT({0}, {1})".format(lon, lat)
    else:
        if project == "NLDAS":
            location = "{0}:X{1:03d}-Y{2:03d}".format(project, xindex, yindex)
        else:
            location = "{0}:X{1:04d}-Y{2:03d}".format(project, xindex, yindex)

    ndf = pd.DataFrame()
    for cnt, var in enumerate(tsutils.make_list(variable)):
        r = resource(
            r"https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/access/timeseries.cgi",
            variable=var,
            location=location,
            startDate=startDate,
            endDate=endDate,
        )
        ndf = ndf.join(odo(r, pd.DataFrame), how="outer", rsuffix="_{0}".format(cnt))

    return ndf

ldas.__doc__ = ldas_cli.__doc__
<commit_msg>BUG: None is the correct value to represent unbounded.<commit_after>from tsgettoolbox.odo import odo, resource
import pandas as pd
import mando

try:
    from mando.rst_text_formatter import RSTHelpFormatter as HelpFormatter
except ImportError:
    from argparse import RawTextHelpFormatter as HelpFormatter

from tstoolbox import tsutils

@mando.command("ldas", formatter_class=HelpFormatter, doctype="numpy")
def ldas_cli(
    lat=None,
    lon=None,
    xindex=None,
    yindex=None,
    variable=None,
    startDate=None,
    endDate=None,
):
    r"""Download data from NLDAS or GLDAS.

    The time zone is always UTC.

    Parameters
    ----------
    lat :  float
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.

        Latitude (required): Enter single geographic point by
        latitude.::

            Example: --lat=43.1

    lon : float
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  Longitude (required): Enter single
        geographic point by longitude::

            Example: --lon=-85.3

    xindex : int
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  xindex (required if using xindex/yindex):
        Enter the x index of the NLDAS or GLDAS grid.::

            Example: --xindex=301

    yindex : int
        Either 'lat' and 'lon', or 'xindex' and
        'yindex' is required.  yindex (required if using xindex/yindex):
        Enter the y index of the NLDAS or GLDAS grid.::

            Example: --yindex=80

    variable : str
        Use the variable codes from the following table:

        +--------------------------------------------+-----------+
        | LDAS "variable" string Description         | Units     |
        +============================================+===========+
        | NLDAS:NLDAS_FORA0125_H.002:APCPsfc         | kg/m^2    |
        | Precipitation hourly total                 |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:DLWRFsfc        | W/m^2     |
        | Surface DW longwave radiation flux         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:DSWRFsfc        | W/m^2     |
        | Surface DW shortwave radiation flux        |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:PEVAPsfc        | kg/m^2    |
        | Potential evaporation                      |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:SPFH2m          | kg/kg     |
        | 2-m above ground specific humidity         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:TMP2m           | degK      |
        | 2-m above ground temperature               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:UGRD10m         | m/s       |
        | 10-m above ground zonal wind               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_FORA0125_H.002:VGRD10m         | m/s       |
        | 10-m above ground meridional wind          |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:EVPsfc          | kg/m^2    |
        | Total evapotranspiration                   |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:GFLUXsfc        | w/m^2     |
        | Ground heat flux                           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:LHTFLsfc        | w/m^2     |
        | Latent heat flux                           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SHTFLsfc        | w/m^2     |
        | Sensible heat flux                         |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SSRUNsfc        | kg/m^2    |
        | Surface runoff (non-infiltrating)          |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:BGRIUNdfc       | kg/m^2    |
        | Subsurface runoff (baseflow)               |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-10cm     | kg/m^2    |
        | 0-10 cm soil moisture content              |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-100cm    | kg/m^2    |
        | 0-100 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM0-200cm    | kg/m^2    |
        | 0-200 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM10-40cm    | kg/m^2    |
        | 10-40 cm soil moisture content             |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM40-100cm   | kg/m^2    |
        | 40-100 cm soil moisture content            |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:SOILM100-200cm  | kg/m^2    |
        | 100-200 cm soil moisture content           |           |
        +--------------------------------------------+-----------+
        | NLDAS:NLDAS_NOAH0125_H.002:TSOIL0-10cm     | degK      |
        | 0-10 cm soil temperature                   |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Evap            | kg/m^2/s  |
        | Evapotranspiration                         |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:precip          | kg/m^s/hr |
        | Precipitation rate                         |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Rainf           | kg/m^2/s  |
        | Rain rate                                  |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Snowf           | kg/m^2/s  |
        | Snow rate                                  |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Qs              | kg/m^2/s  |
        | Surface Runoff                             |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Qsb             | kg/m^2/s  |
        | Subsurface Runoff                          |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM0-100cm    | kg/m^2    |
        | 0-100 cm top 1 meter soil moisture content |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM0-10cm     | kg/m^2    |
        | 0-10 cm layer 1 soil moisture content      |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM10-40cm    | kg/m^2    |
        | 10-40 cm layer 2 soil moisture content     |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:SOILM40-100cm   | kg/m^2    |
        | 40-100 cm layer 3 soil moisture content    |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Tair            | degK      |
        | Near surface air temperature               |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:TSOIL0-10cm     | degK      |
        | Average layer 1 soil temperature           |           |
        +--------------------------------------------+-----------+
        | GLDAS:GLDAS_NOAH025_3H.001:Wind            | m/s       |
        | Near surface wind magnitude                |           |
        +--------------------------------------------+-----------+

    startDate : str
        The start date of the time series.::

            Example: --startDate=2001-01-01T05

        If startDate and endDate are None, returns the entire series.

    endDate : str
        The end date of the time series.::

            Example: --endDate=2002-01-05T05

        If startDate and endDate are None, returns the entire series.
    """
    tsutils._printiso(
        ldas(
            lat=lat,
            lon=lon,
            xindex=xindex,
            yindex=yindex,
            variable=variable,
            startDate=startDate,
            endDate=endDate,
        )
    )

@tsutils.validator(
    lat=[float, ["range", [-90.0, 90.0]], 1],
    lon=[float, ["range", [-180.0, 180.0]], 1],
    xindex=[int, ["range", [0, None]], 1],
    yindex=[int, ["range", [0, None]], 1],
    variable=[str, ["pass", []], None],
    startDate=[tsutils.parsedate, ["pass", []], 1],
    endDate=[tsutils.parsedate, ["pass", []], 1],
)
def ldas(
    lat=None,
    lon=None,
    xindex=None,
    yindex=None,
    variable=None,
    startDate=None,
    endDate=None,
):
    r"""Download data from NLDAS or GLDAS."""
    from tsgettoolbox.services import ldas as placeholder

    project = variable.split(":")[0]
    if lat is not None:
        location = "GEOM:POINT({0}, {1})".format(lon, lat)
    else:
        if project == "NLDAS":
            location = "{0}:X{1:03d}-Y{2:03d}".format(project, xindex, yindex)
        else:
            location = "{0}:X{1:04d}-Y{2:03d}".format(project, xindex, yindex)

    ndf = pd.DataFrame()
    for cnt, var in enumerate(tsutils.make_list(variable)):
        r = resource(
            r"https://hydro1.gesdisc.eosdis.nasa.gov/daac-bin/access/timeseries.cgi",
            variable=var,
            location=location,
            startDate=startDate,
            endDate=endDate,
        )
        ndf = ndf.join(odo(r, pd.DataFrame), how="outer", rsuffix="_{0}".format(cnt))

    return ndf

ldas.__doc__ = ldas_cli.__doc__
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
makes modifications to the controlled vocabulary (implemented as
ckanext-scheming "choices")
HvW - 2016-06-07
"""
from pprint import pprint
import argparse
import sys
import json
import os

LOCAL_SCHEMA=("/usr/lib/ckan/default/src/ckanext-eaw_schema/ckanext/" +
               "eaw_schema/eaw_schema_default.json")

def mkparser():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description=
"""
Make modifications to the controlled vocabulary FIELD
(implemented as ckanext-scheming "choices").
""",                                  epilog=
"""
Examples:
ck_choices.py variables "new_var_1,New Variable One" newvar2,"Another One"
adds two new terms to the dataset_field "variables".

ck_choices.py variables --del new_var_1 newvar2
deletes them.
"""
    )
    parser.add_argument('field', nargs='?', help='the schema field to be modified',
                        metavar='FIELD')
    parser.add_argument('--listfields', action='store_true', help='list fields'+\
                        ' (the rest of the command line is ignored)')
    parser.add_argument('--del', action='store_true', help='delete terms '+
                        '(default is adding terms)')
    parser.add_argument('--resource', action='store_true', help='action ' +
                        'refers to resource field (default is dataset field)')
    parser.add_argument('terms', nargs='*', help='the terms to be added '
                        +'(removed). Have the format "value,label" for adding,' +
                        ' and "value" for removing', metavar='TERM')
    return(parser)

def listfields(schema):
    fields = schema['dataset_fields']
    for f in ('{}, {}\n{}\n'.format(x['field_name'], x['label'],
                                    [c['value'] for c in x['choices']])
              for x in fields if 'choices' in x):
        print f

def postparse(params, parser):
    if params['listfields']:
        return
    terms = params['terms']
    if not terms or not params['field']:
        parser.print_help()
        sys.exit(1)
    terms = [tuple(x.split(',')) for x in terms]
    if params['del'] and not all([len(x) == 1 for x in terms]):
        parser.print_help()
        sys.exit(1)
    elif not params['del'] and not all([len(x) == 2 for x in terms]):
        parser.print_help()
        sys.exit(1)
    terms = [x[0] if len(x) == 1 else x for x in terms]
    return(terms)

def load_schema(schemafile):
    try:
        with open(schemafile) as sf:
            schema = json.load(sf)
    except ValueError:
        raise(SystemExit("Schema file {} doesn't parse into JSON"
                         .format(schemafile), 1))
    except IOError:
        raise(SystemExit("Could not open schema file 'testschema'", 1))
    return(schema)

def check_unique(field, choices, terms):
    for t in [x[0] for x in terms]:
        if t in [x['value'] for x in choices]:
            raise SystemExit('{} already in {}'.format(t, field))

def update_field(schema, typ, field, remove, terms):
    " typ: 'dataset_field' or 'resource_field'"
    def _build_choices(terms):
        ch = [{'value': t[0], 'label': t[1]} for t in terms]
        return ch

    def _get_val_index(val, choices):
        idx = [x.get('value') for x in choices].index(val)
        return(idx)
    try:
        f = [x for x in schema[typ] if x["field_name"] == field]
    except KeyError:
        raise SystemExit('Could not find field_type "{}"'.format(typ))
    if not f:
        raise SystemExit("Could not find field \"{}\" in \"{}\""
                         .format(field, typ))
    assert(len(f) == 1)
    c = f[0]['choices']
    if not remove:
        check_unique(field, c, terms)
        c.extend(_build_choices(terms))
    else:
        try:
            rmidx = [_get_val_index(val, c) for val in terms]
        except ValueError:
            raise SystemExit('Not all terms found in ' +
                             'field "{}" in "{}"'.format(field, typ))
        if len(rmidx) < len(terms):
            raise SystemExit('Not all terms found in ' +
                 'field "{}" in "{}"'.format(field, typ))
        cnew = [x[1] for x in enumerate(c) if x[0] not in rmidx]
        f[0]['choices'] = cnew
    return(schema)

def write_schema(newschema, path):
    with open(path, 'w') as f:
        json.dump(newschema, f, indent=2)

def main():
    schema = load_schema(LOCAL_SCHEMA)
    parser = mkparser()
    params = vars(parser.parse_args())
    print params
    terms = postparse(params, parser)
    if params['listfields']:
        listfields(schema)
        sys.exit()
    field = params['field']
    remove = params['del']
    typ = 'resource_fields' if params['resource'] else 'dataset_fields'
    newschema = update_field(schema, typ, field, remove, terms)
    write_schema(newschema, LOCAL_SCHEMA)

if __name__ == '__main__':
    main()

<commit_msg>remove debug print statement<commit_after>#!/usr/bin/env python

"""
makes modifications to the controlled vocabulary (implemented as
ckanext-scheming "choices")
HvW - 2016-06-07
"""
from pprint import pprint
import argparse
import sys
import json
import os

LOCAL_SCHEMA=("/usr/lib/ckan/default/src/ckanext-eaw_schema/ckanext/" +
               "eaw_schema/eaw_schema_default.json")

def mkparser():
    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description=
"""
Make modifications to the controlled vocabulary FIELD
(implemented as ckanext-scheming "choices").
""",                                  epilog=
"""
Examples:
ck_choices.py variables "new_var_1,New Variable One" newvar2,"Another One"
adds two new terms to the dataset_field "variables".

ck_choices.py variables --del new_var_1 newvar2
deletes them.
"""
    )
    parser.add_argument('field', nargs='?', help='the schema field to be modified',
                        metavar='FIELD')
    parser.add_argument('--listfields', action='store_true', help='list fields'+\
                        ' (the rest of the command line is ignored)')
    parser.add_argument('--del', action='store_true', help='delete terms '+
                        '(default is adding terms)')
    parser.add_argument('--resource', action='store_true', help='action ' +
                        'refers to resource field (default is dataset field)')
    parser.add_argument('terms', nargs='*', help='the terms to be added '
                        +'(removed). Have the format "value,label" for adding,' +
                        ' and "value" for removing', metavar='TERM')
    return(parser)

def listfields(schema):
    fields = schema['dataset_fields']
    for f in ('{}, {}\n{}\n'.format(x['field_name'], x['label'],
                                    [c['value'] for c in x['choices']])
              for x in fields if 'choices' in x):
        print f

def postparse(params, parser):
    if params['listfields']:
        return
    terms = params['terms']
    if not terms or not params['field']:
        parser.print_help()
        sys.exit(1)
    terms = [tuple(x.split(',')) for x in terms]
    if params['del'] and not all([len(x) == 1 for x in terms]):
        parser.print_help()
        sys.exit(1)
    elif not params['del'] and not all([len(x) == 2 for x in terms]):
        parser.print_help()
        sys.exit(1)
    terms = [x[0] if len(x) == 1 else x for x in terms]
    return(terms)

def load_schema(schemafile):
    try:
        with open(schemafile) as sf:
            schema = json.load(sf)
    except ValueError:
        raise(SystemExit("Schema file {} doesn't parse into JSON"
                         .format(schemafile), 1))
    except IOError:
        raise(SystemExit("Could not open schema file 'testschema'", 1))
    return(schema)

def check_unique(field, choices, terms):
    for t in [x[0] for x in terms]:
        if t in [x['value'] for x in choices]:
            raise SystemExit('{} already in {}'.format(t, field))

def update_field(schema, typ, field, remove, terms):
    " typ: 'dataset_field' or 'resource_field'"
    def _build_choices(terms):
        ch = [{'value': t[0], 'label': t[1]} for t in terms]
        return ch

    def _get_val_index(val, choices):
        idx = [x.get('value') for x in choices].index(val)
        return(idx)
    try:
        f = [x for x in schema[typ] if x["field_name"] == field]
    except KeyError:
        raise SystemExit('Could not find field_type "{}"'.format(typ))
    if not f:
        raise SystemExit("Could not find field \"{}\" in \"{}\""
                         .format(field, typ))
    assert(len(f) == 1)
    c = f[0]['choices']
    if not remove:
        check_unique(field, c, terms)
        c.extend(_build_choices(terms))
    else:
        try:
            rmidx = [_get_val_index(val, c) for val in terms]
        except ValueError:
            raise SystemExit('Not all terms found in ' +
                             'field "{}" in "{}"'.format(field, typ))
        if len(rmidx) < len(terms):
            raise SystemExit('Not all terms found in ' +
                 'field "{}" in "{}"'.format(field, typ))
        cnew = [x[1] for x in enumerate(c) if x[0] not in rmidx]
        f[0]['choices'] = cnew
    return(schema)

def write_schema(newschema, path):
    with open(path, 'w') as f:
        json.dump(newschema, f, indent=2)

def main():
    schema = load_schema(LOCAL_SCHEMA)
    parser = mkparser()
    params = vars(parser.parse_args())
    terms = postparse(params, parser)
    if params['listfields']:
        listfields(schema)
        sys.exit()
    field = params['field']
    remove = params['del']
    typ = 'resource_fields' if params['resource'] else 'dataset_fields'
    newschema = update_field(schema, typ, field, remove, terms)
    write_schema(newschema, LOCAL_SCHEMA)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ML-ENSEMBLE

author: Sebastian Flennerhag
date: 10/01/2017
licence: MIT
Collection of preprocessing functions
"""

from __future__ import division, print_function

from pandas import DataFrame, Series
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import StandardScaler

class StandardScalerDf(StandardScaler):

    """Standard Scale a DataFrame

    Wrapper around StandardScaler to preserve DataFrame representation.
    See original StandardScaler for documentation
    """

    def __init__(self, copy=True, with_mean=True, with_std=True):
        super().__init__(copy=copy, with_mean=with_mean, with_std=with_std)

    def transform(self, X, y=None, copy=None):
        """StandardScales data and assigns it to a copy of passed DataFrame"""
        X.loc[:, :] = super().transform(X, y, copy)
        return X

class Subset(BaseEstimator, TransformerMixin):

    """
    Class for manually selecting a subset

    Parameters
    ----------
    subset : list
        list of columns indexes which can be either strings or integers
    """

    def __init__(self, subset=None):
        self.subset = subset

    def fit(self, X, y=None):
        """Fit method for learning datatype"""
        self.is_df_ = isinstance(X, (DataFrame, Series))

        if self.subset is not None:
            self.use_loc_ = any([isinstance(x, str) for x in self.subset])

        return self

    def transform(self, X):
        """Return selected subset"""
        if self.subset is None:
            return X
        else:
            Xt = X.copy()
            if self.is_df_ and self.use_loc_:
                Xt = Xt.loc[:, self.subset]
            elif self.is_df_:
                Xt = Xt.iloc[:, self.subset]
            else:
                Xt = Xt[:, self.subset]
            return Xt
<commit_msg>super compatability with 2.7<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ML-ENSEMBLE

author: Sebastian Flennerhag
date: 10/01/2017
licence: MIT
Collection of preprocessing functions
"""

from __future__ import division, print_function

from pandas import DataFrame, Series
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.preprocessing import StandardScaler

class StandardScalerDf(StandardScaler):

    """Standard Scale a DataFrame

    Wrapper around StandardScaler to preserve DataFrame representation.
    See original StandardScaler for documentation
    """

    def __init__(self, copy=True, with_mean=True, with_std=True):
        super(StandardScalerDf, self).__init__(copy=copy, with_mean=with_mean,
                                             with_std=with_std)

    def transform(self, X, y=None, copy=None):
        """StandardScales data and assigns it to a copy of passed DataFrame"""
        X.loc[:, :] = super(StandardScalerDf, self).transform(X, y, copy)
        return X

class Subset(BaseEstimator, TransformerMixin):

    """
    Class for manually selecting a subset

    Parameters
    ----------
    subset : list
        list of columns indexes which can be either strings or integers
    """

    def __init__(self, subset=None):
        self.subset = subset

    def fit(self, X, y=None):
        """Fit method for learning datatype"""
        self.is_df_ = isinstance(X, (DataFrame, Series))

        if self.subset is not None:
            self.use_loc_ = any([isinstance(x, str) for x in self.subset])

        return self

    def transform(self, X):
        """Return selected subset"""
        if self.subset is None:
            return X
        else:
            Xt = X.copy()
            if self.is_df_ and self.use_loc_:
                Xt = Xt.loc[:, self.subset]
            elif self.is_df_:
                Xt = Xt.iloc[:, self.subset]
            else:
                Xt = Xt[:, self.subset]
            return Xt
<|endoftext|>


<commit_before>from lxml import etree
import pytz
import xmltodict

from .ts_writer import TimeSeriesWriter

DNS = 'http://www.wldelft.nl/fews/PI'  # default namespace
XSI = 'http://www.w3.org/2001/XMLSchema-instance'
XSD = 'http://fews.wldelft.nl/schemas/version1.0/pi-schemas/pi_timeseries.xsd'

nsmap = {None: DNS, 'xsi': XSI}

DATE_FMT = '%Y-%m-%d'
TIME_FMT = '%H:%M:%S'

class PiXmlWriter(TimeSeriesWriter):
    """docstring"""

    def __init__(self, offset_in_hours=0.0):
        """docstring

        Keyword arguments:
        offset_in_hours -- fixed offset in hours from UTC (default 0.0)

        Most time zones are offset from UTC by a whole number of hours,
        but a few are offset by 30 or 45 minutes. Pass `None` to omit
        the optional timeZone element in the resulting xml.

        """
        self.root = etree.Element('TimeSeries', nsmap=nsmap)
        self.root.attrib['{%s}schemaLocation' % XSI] = "%s %s" % (DNS, XSD)
        self.root.attrib['version'] = '1.2'
        if offset_in_hours is not None:
            etree.SubElement(self.root, 'timeZone').text = str(offset_in_hours)
            self.tz = pytz.FixedOffset(offset_in_hours * 60)

    def set_series(self, metadata, dataframe):
        """docstring"""
        series = etree.SubElement(self.root, 'series')

        header = xmltodict.unparse(metadata)
        header = bytes(bytearray(header, encoding='utf-8'))
        header = etree.XML(header)
        series.append(header)

        if dataframe.empty:
            return

        if hasattr(self, 'tz') and dataframe.index.tz is not None:
            dataframe.tz_convert(self.tz, copy=False)

        for idx, row in dataframe.iterrows():
            event = etree.SubElement(series, 'event')
            event.attrib['date'] = idx.strftime(DATE_FMT)
            event.attrib['time'] = idx.strftime(TIME_FMT)
            for col in dataframe.columns.tolist():
                event.attrib[col] = str(row[col])

    def write(self, out, pretty_print=True):
        """docstring"""
        out.write(etree.tostring(self.root, pretty_print=pretty_print))
<commit_msg>Synchronize datetimes in header with events in dataframe<commit_after>from lxml import etree
import pytz
import xmltodict

from .ts_writer import TimeSeriesWriter

DNS = 'http://www.wldelft.nl/fews/PI'  # default namespace
XSI = 'http://www.w3.org/2001/XMLSchema-instance'
XSD = 'http://fews.wldelft.nl/schemas/version1.0/pi-schemas/pi_timeseries.xsd'

nsmap = {None: DNS, 'xsi': XSI}

DATE_FMT = '%Y-%m-%d'
TIME_FMT = '%H:%M:%S'

def set_datetime(md, df):
    md['header']['startDate']['@date'] = df.index[0].strftime(DATE_FMT)
    md['header']['startDate']['@time'] = df.index[0].strftime(TIME_FMT)
    md['header']['endDate']['@date'] = df.index[-1].strftime(DATE_FMT)
    md['header']['endDate']['@time'] = df.index[-1].strftime(TIME_FMT)

class PiXmlWriter(TimeSeriesWriter):
    """docstring"""

    def __init__(self, offset_in_hours=0.0):
        """docstring

        Keyword arguments:
        offset_in_hours -- fixed offset in hours from UTC (default 0.0)

        Most time zones are offset from UTC by a whole number of hours,
        but a few are offset by 30 or 45 minutes. Pass `None` to omit
        the optional timeZone element in the resulting xml.

        """
        self.root = etree.Element('TimeSeries', nsmap=nsmap)
        self.root.attrib['{%s}schemaLocation' % XSI] = "%s %s" % (DNS, XSD)
        self.root.attrib['version'] = '1.2'
        if offset_in_hours is not None:
            etree.SubElement(self.root, 'timeZone').text = str(offset_in_hours)
            self.tz = pytz.FixedOffset(offset_in_hours * 60)

    def set_series(self, metadata, dataframe):
        """docstring"""
        series = etree.SubElement(self.root, 'series')

        if not dataframe.empty:
            dataframe.tz_convert(self.tz, copy=False)
            set_datetime(metadata, dataframe)

        header = xmltodict.unparse(metadata)
        header = bytes(bytearray(header, encoding='utf-8'))
        header = etree.XML(header)
        series.append(header)

        if dataframe.empty:
            return

        for idx, row in dataframe.iterrows():
            event = etree.SubElement(series, 'event')
            event.attrib['date'] = idx.strftime(DATE_FMT)
            event.attrib['time'] = idx.strftime(TIME_FMT)
            for col in dataframe.columns.tolist():
                event.attrib[col] = str(row[col])

    def write(self, out, pretty_print=True):
        """docstring"""
        out.write(etree.tostring(self.root, pretty_print=pretty_print))
<|endoftext|>


<commit_before>import warnings

import torch.nn as nn
from mmcv.cnn import constant_init, kaiming_init

from mmdet.ops import DeformConvPack, ModulatedDeformConvPack
from .conv_ws import ConvWS2d
from .norm import build_norm_layer

conv_cfg = {
    'Conv': nn.Conv2d,
    'ConvWS': ConvWS2d,
    'DCN': DeformConvPack,
    'DCNv2': ModulatedDeformConvPack,
    # TODO: octave conv
}

def build_conv_layer(cfg, *args, **kwargs):
    """ Build convolution layer

    Args:
        cfg (None or dict): cfg should contain:
            type (str): identify conv layer type.
            layer args: args needed to instantiate a conv layer.

    Returns:
        layer (nn.Module): created conv layer
    """
    if cfg is None:
        cfg_ = dict(type='Conv')
    else:
        assert isinstance(cfg, dict) and 'type' in cfg
        cfg_ = cfg.copy()

    layer_type = cfg_.pop('type')
    if layer_type not in conv_cfg:
        raise KeyError('Unrecognized norm type {}'.format(layer_type))
    else:
        conv_layer = conv_cfg[layer_type]

    layer = conv_layer(*args, **kwargs, **cfg_)

    return layer

class ConvModule(nn.Module):
    """A conv block that contains conv/norm/activation layers.

    Args:
        in_channels (int): Same as nn.Conv2d.
        out_channels (int): Same as nn.Conv2d.
        kernel_size (int or tuple[int]): Same as nn.Conv2d.
        stride (int or tuple[int]): Same as nn.Conv2d.
        padding (int or tuple[int]): Same as nn.Conv2d.
        dilation (int or tuple[int]): Same as nn.Conv2d.
        groups (int): Same as nn.Conv2d.
        bias (bool or str): If specified as `auto`, it will be decided by the
            norm_cfg. Bias will be set as True if norm_cfg is None, otherwise
            False.
        conv_cfg (dict): Config dict for convolution layer.
        norm_cfg (dict): Config dict for normalization layer.
        activation (str or None): Activation type, "ReLU" by default.
        inplace (bool): Whether to use inplace mode for activation.
        order (tuple[str]): The order of conv/norm/activation layers. It is a
            sequence of "conv", "norm" and "act". Examples are
            ("conv", "norm", "act") and ("act", "conv", "norm").
    """

    def __init__(self,
                 in_channels,
                 out_channels,
                 kernel_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=1,
                 bias='auto',
                 conv_cfg=None,
                 norm_cfg=None,
                 activation='relu',
                 inplace=True,
                 order=('conv', 'norm', 'act')):
        super(ConvModule, self).__init__()
        assert conv_cfg is None or isinstance(conv_cfg, dict)
        assert norm_cfg is None or isinstance(norm_cfg, dict)
        self.conv_cfg = conv_cfg
        self.norm_cfg = norm_cfg
        self.activation = activation
        self.inplace = inplace
        self.order = order
        assert isinstance(self.order, tuple) and len(self.order) == 3
        assert set(order) == set(['conv', 'norm', 'act'])

        self.with_norm = norm_cfg is not None
        self.with_activatation = activation is not None
        # if the conv layer is before a norm layer, bias is unnecessary.
        if bias == 'auto':
            bias = False if self.with_norm else True
        self.with_bias = bias

        if self.with_norm and self.with_bias:
            warnings.warn('ConvModule has norm and bias at the same time')

        # build convolution layer
        self.conv = build_conv_layer(
            conv_cfg,
            in_channels,
            out_channels,
            kernel_size,
            stride=stride,
            padding=padding,
            dilation=dilation,
            groups=groups,
            bias=bias)
        # export the attributes of self.conv to a higher level for convenience
        self.in_channels = self.conv.in_channels
        self.out_channels = self.conv.out_channels
        self.kernel_size = self.conv.kernel_size
        self.stride = self.conv.stride
        self.padding = self.conv.padding
        self.dilation = self.conv.dilation
        self.transposed = self.conv.transposed
        self.output_padding = self.conv.output_padding
        self.groups = self.conv.groups

        # build normalization layers
        if self.with_norm:
            # norm layer is after conv layer
            if order.index('norm') > order.index('conv'):
                norm_channels = out_channels
            else:
                norm_channels = in_channels
            self.norm_name, norm = build_norm_layer(norm_cfg, norm_channels)
            self.add_module(self.norm_name, norm)

        # build activation layer
        if self.with_activatation:
            # TODO: introduce `act_cfg` and supports more activation layers
            if self.activation not in ['relu']:
                raise ValueError('{} is currently not supported.'.format(
                    self.activation))
            if self.activation == 'relu':
                self.activate = nn.ReLU(inplace=inplace)

        # Use msra init by default
        self.init_weights()

    @property
    def norm(self):
        return getattr(self, self.norm_name)

    def init_weights(self):
        nonlinearity = 'relu' if self.activation is None else self.activation
        kaiming_init(self.conv, nonlinearity=nonlinearity)
        if self.with_norm:
            constant_init(self.norm, 1, bias=0)

    def forward(self, x, activate=True, norm=True):
        for layer in self.order:
            if layer == 'conv':
                x = self.conv(x)
            elif layer == 'norm' and norm and self.with_norm:
                x = self.norm(x)
            elif layer == 'act' and activate and self.with_activatation:
                x = self.activate(x)
        return x
<commit_msg>Fix typo activatation -> activation (#2007)<commit_after>import warnings

import torch.nn as nn
from mmcv.cnn import constant_init, kaiming_init

from mmdet.ops import DeformConvPack, ModulatedDeformConvPack
from .conv_ws import ConvWS2d
from .norm import build_norm_layer

conv_cfg = {
    'Conv': nn.Conv2d,
    'ConvWS': ConvWS2d,
    'DCN': DeformConvPack,
    'DCNv2': ModulatedDeformConvPack,
    # TODO: octave conv
}

def build_conv_layer(cfg, *args, **kwargs):
    """ Build convolution layer

    Args:
        cfg (None or dict): cfg should contain:
            type (str): identify conv layer type.
            layer args: args needed to instantiate a conv layer.

    Returns:
        layer (nn.Module): created conv layer
    """
    if cfg is None:
        cfg_ = dict(type='Conv')
    else:
        assert isinstance(cfg, dict) and 'type' in cfg
        cfg_ = cfg.copy()

    layer_type = cfg_.pop('type')
    if layer_type not in conv_cfg:
        raise KeyError('Unrecognized norm type {}'.format(layer_type))
    else:
        conv_layer = conv_cfg[layer_type]

    layer = conv_layer(*args, **kwargs, **cfg_)

    return layer

class ConvModule(nn.Module):
    """A conv block that contains conv/norm/activation layers.

    Args:
        in_channels (int): Same as nn.Conv2d.
        out_channels (int): Same as nn.Conv2d.
        kernel_size (int or tuple[int]): Same as nn.Conv2d.
        stride (int or tuple[int]): Same as nn.Conv2d.
        padding (int or tuple[int]): Same as nn.Conv2d.
        dilation (int or tuple[int]): Same as nn.Conv2d.
        groups (int): Same as nn.Conv2d.
        bias (bool or str): If specified as `auto`, it will be decided by the
            norm_cfg. Bias will be set as True if norm_cfg is None, otherwise
            False.
        conv_cfg (dict): Config dict for convolution layer.
        norm_cfg (dict): Config dict for normalization layer.
        activation (str or None): Activation type, "ReLU" by default.
        inplace (bool): Whether to use inplace mode for activation.
        order (tuple[str]): The order of conv/norm/activation layers. It is a
            sequence of "conv", "norm" and "act". Examples are
            ("conv", "norm", "act") and ("act", "conv", "norm").
    """

    def __init__(self,
                 in_channels,
                 out_channels,
                 kernel_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 groups=1,
                 bias='auto',
                 conv_cfg=None,
                 norm_cfg=None,
                 activation='relu',
                 inplace=True,
                 order=('conv', 'norm', 'act')):
        super(ConvModule, self).__init__()
        assert conv_cfg is None or isinstance(conv_cfg, dict)
        assert norm_cfg is None or isinstance(norm_cfg, dict)
        self.conv_cfg = conv_cfg
        self.norm_cfg = norm_cfg
        self.activation = activation
        self.inplace = inplace
        self.order = order
        assert isinstance(self.order, tuple) and len(self.order) == 3
        assert set(order) == set(['conv', 'norm', 'act'])

        self.with_norm = norm_cfg is not None
        self.with_activation = activation is not None
        # if the conv layer is before a norm layer, bias is unnecessary.
        if bias == 'auto':
            bias = False if self.with_norm else True
        self.with_bias = bias

        if self.with_norm and self.with_bias:
            warnings.warn('ConvModule has norm and bias at the same time')

        # build convolution layer
        self.conv = build_conv_layer(
            conv_cfg,
            in_channels,
            out_channels,
            kernel_size,
            stride=stride,
            padding=padding,
            dilation=dilation,
            groups=groups,
            bias=bias)
        # export the attributes of self.conv to a higher level for convenience
        self.in_channels = self.conv.in_channels
        self.out_channels = self.conv.out_channels
        self.kernel_size = self.conv.kernel_size
        self.stride = self.conv.stride
        self.padding = self.conv.padding
        self.dilation = self.conv.dilation
        self.transposed = self.conv.transposed
        self.output_padding = self.conv.output_padding
        self.groups = self.conv.groups

        # build normalization layers
        if self.with_norm:
            # norm layer is after conv layer
            if order.index('norm') > order.index('conv'):
                norm_channels = out_channels
            else:
                norm_channels = in_channels
            self.norm_name, norm = build_norm_layer(norm_cfg, norm_channels)
            self.add_module(self.norm_name, norm)

        # build activation layer
        if self.with_activation:
            # TODO: introduce `act_cfg` and supports more activation layers
            if self.activation not in ['relu']:
                raise ValueError('{} is currently not supported.'.format(
                    self.activation))
            if self.activation == 'relu':
                self.activate = nn.ReLU(inplace=inplace)

        # Use msra init by default
        self.init_weights()

    @property
    def norm(self):
        return getattr(self, self.norm_name)

    def init_weights(self):
        nonlinearity = 'relu' if self.activation is None else self.activation
        kaiming_init(self.conv, nonlinearity=nonlinearity)
        if self.with_norm:
            constant_init(self.norm, 1, bias=0)

    def forward(self, x, activate=True, norm=True):
        for layer in self.order:
            if layer == 'conv':
                x = self.conv(x)
            elif layer == 'norm' and norm and self.with_norm:
                x = self.norm(x)
            elif layer == 'act' and activate and self.with_activation:
                x = self.activate(x)
        return x
<|endoftext|>


<commit_before>
import bisect
import struct
from typing import Tuple, Union, List

__all__ = ('Clemory',)

class Clemory:

    __slots__ = ('_arch', '_backers', '_updates', '_pointer', '_root', '_cbackers', '_needs_flattening_personal',
                 'consecutive', 'min_addr', 'max_addr', 'concrete_target' )

    """
    An object representing a memory space.

    Accesses can be made with [index] notation.
    """
    def __init__(self, arch, root=False):
        self._arch = arch
        self._backers = []  # type: Tuple[int, Union[bytearray, Clemory, List[int]]]
        self._pointer = 0
        self._root = root

        self.consecutive = True
        self.min_addr = 0
        self.max_addr = 0

        self.concrete_target = None

    def is_concrete_target_set(self):
        return self.concrete_target is not None

    def set_concrete_target(self, concrete_target):
        self.concrete_target = concrete_target

    def add_backer(self, start, data):
        """
        Adds a backer to the memory.

        :param start:   The address where the backer should be loaded.
        :param data:    The backer itself. Can be either a bytestring or another :class:`Clemory`.
        """
        if not data:
            raise ValueError("Backer is empty!")

        if not isinstance(data, (bytes, list, Clemory)):
            raise TypeError("Data must be a bytes, list, or Clemory object.")
        if start in self:
            raise ValueError("Address %#x is already backed!" % start)
        if isinstance(data, Clemory) and data._root:
            raise ValueError("Cannot add a root clemory as a backer!")
        if type(data) is bytes:
            data = bytearray(data)
        bisect.insort(self._backers, (start, data))
        self._update_min_max()

    def update_backer(self, start, data):
        if not isinstance(data, (bytes, list, Clemory)):
            raise TypeError("Data must be a bytes, list, or Clemory object.")
        if type(data) is bytes:
            data = bytearray(data)
        for i, (oldstart, _) in enumerate(self._backers):
            if oldstart == start:
                self._backers[i] = (start, data)
                break
        else:
            raise ValueError("Can't find backer to update")

        self._update_min_max()

    def remove_backer(self, start):
        for i, (oldstart, _) in enumerate(self._backers):
            if oldstart == start:
                self._backers.pop(i)
                break
        else:
            raise ValueError("Can't find backer to remove")

        self._update_min_max()

    def __iter__(self):
        for start, string in self._backers:
            if isinstance(string, (bytes, list)):
                for x in range(len(string)):
                    yield start + x
            else:
                for x in string:
                    yield start + x

    def __getitem__(self, k):

        # concrete memory read
        if self.is_concrete_target_set():
            # l.debug("invoked get_byte %x" % (k))
            return self.concrete_target.read_memory(k, 1)

        for start, data in self._backers:
            if type(data) in (bytearray, list):
                if 0 <= k - start < len(data):
                    return data[k - start]
            elif isinstance(data, Clemory):
                if data.min_addr <= k - start < data.max_addr:
                    try:
                        return data[k - start]
                    except KeyError:
                        pass
        raise KeyError(k)

    def __setitem__(self, k, v):
        for start, data in self._backers:
            if type(data) in (bytearray, list):
                if 0 <= k - start < len(data):
                    data[k - start] = v
                    return
            elif isinstance(data, Clemory):
                if data.min_addr <= k - start < data.max_addr:
                    try:
                        data[k - start] = v
                        return
                    except KeyError:
                        pass
        raise KeyError(k)

    def __contains__(self, k):
        # Fast path
        if self.consecutive:
            return self.min_addr <= k < self.max_addr
        else:
            # Check if this is an empty Clemory instance
            if not self._backers:
                return False
            # Check if it is out of the memory range
            if k < self.min_addr or k >= self.max_addr:
                return False

        try:
            self.__getitem__(k)
        except KeyError:
            return False
        else:
            return True

    def __getstate__(self):
        s = {
            '_arch': self._arch,
            '_backers': self._backers,
            '_pointer': self._pointer,
            '_root': self._root,
            'consecutive': self.consecutive,
            'min_addr': self.min_addr,
            'max_addr': self.max_addr,
            'concrete_target': self.concrete_target
        }

        return s

    def __setstate__(self, s):
        self._arch = s['_arch']
        self._backers = s['_backers']
        self._pointer = s['_pointer']
        self._root = s['_root']
        self.consecutive = s['consecutive']
        self.min_addr = s['min_addr']
        self.max_addr = s['max_addr']
        self.concrete_target = s['concrete_target']

    def backers(self, addr=0):
        """
        Iterate through each backer for this clemory and all its children, yielding tuples of
        ``(start_addr, backer)`` where each backer is a bytearray.

        :param addr:    An optional starting address - all backers before and not including this
                        address will be skipped.
        """
        started = addr <= 0
        for start, backer in self._backers:
            if not started:
                end = start + backer.max_addr if type(backer) is Clemory else start + len(backer)
                if addr >= end:
                    continue
                started = True
            if type(backer) is Clemory:
                for s, b in backer.backers(addr - start):
                    yield s + start, b
            else:
                yield start, backer

    def load(self, addr, n):
        """
        Read up to `n` bytes at address `addr` in memory and return a bytes object.

        Reading will stop at the beginning of the first unallocated region found, or when
        `n` bytes have been read.
        """

        # concrete memory read
        if self.is_concrete_target_set():
            # l.debug("invoked read_bytes %x %x" % (addr, n))
            return self.concrete_target.read_memory(addr, n)

        views = []

        for start, backer in self.backers(addr):
            if start > addr:
                break
            offset = addr - start
            size = len(backer) - offset
            views.append(memoryview(backer)[offset:offset + n])

            addr += size
            n -= size

            if n <= 0:
                break

        if not views:
            raise KeyError(n)
        return b''.join(views)

    def store(self, addr, data):
        """
        Write bytes from `data` at address `addr`.

        Note: If the store runs off the end of a backer and into unbacked space, this function
        will update the backer but also raise ``KeyError``.
        """
        for start, backer in self.backers(addr):
            if start > addr:
                raise KeyError(addr)
            offset = addr - start
            size = len(backer) - offset
            backer[offset:offset + len(data)] = data if len(data) <= size else data[:size]

            addr += size
            data = data[size:]

            if not data:
                break

        if data:
            raise KeyError(addr)

    def unpack(self, addr, fmt):
        """
        Use the ``struct`` module to unpack the data at address `addr` with the format `fmt`.
        """

        try:
            start, backer = next(self.backers(addr))
        except StopIteration:
            raise KeyError(addr)

        if start > addr:
            raise KeyError(addr)

        try:
            return struct.unpack_from(fmt, backer, addr - start)
        except struct.error as e:
            if len(backer) - (addr - start) >= struct.calcsize(fmt):
                raise e
            raise KeyError(addr)

    def unpack_word(self, addr, size=None, signed=False, endness=None):
        """
        Use the ``struct`` module to unpack a single integer from the address `addr`.

        You may override any of the attributes of the word being extracted:

        :param int size:    The size in bytes to pack/unpack. Defaults to wordsize (e.g. 4 bytes on
                            a 32 bit architecture)
        :param bool signed: Whether the data should be extracted signed/unsigned. Default unsigned
        :param str archinfo.Endness: The endian to use in packing/unpacking. Defaults to memory endness
        """
        return self.unpack(addr, self._arch.struct_fmt(size=size, signed=signed, endness=endness))[0]

    def pack(self, addr, fmt, *data):
        """
        Use the ``struct`` module to pack `data` into memory at address `addr` with the format `fmt`.
        """

        try:
            start, backer = next(self.backers(addr))
        except StopIteration:
            raise KeyError(addr)

        if start > addr:
            raise KeyError(addr)

        try:
            return struct.pack_into(fmt, backer, addr - start, *data)
        except struct.error as e:
            if len(backer) - (addr - start) >= struct.calcsize(fmt):
                raise e
            raise KeyError(addr)

    def pack_word(self, addr, data, size=None, signed=False, endness=None):
        """
        Use the ``struct`` module to pack a single integer `data` into memory at the address `addr`.

        You may override any of the attributes of the word being packed:

        :param int size:    The size in bytes to pack/unpack. Defaults to wordsize (e.g. 4 bytes on
                            a 32 bit architecture)
        :param bool signed: Whether the data should be extracted signed/unsigned. Default unsigned
        :param str archinfo.Endness: The endian to use in packing/unpacking. Defaults to memory endness
        """
        if not signed:
            data &= (1 << (size*8 if size is not None else self._arch.bits)) - 1
        return self.pack(addr, self._arch.struct_fmt(size=size, signed=signed, endness=endness), data)

    def read(self, nbytes):
        """
        The stream-like function that reads up to a number of bytes starting from the current
        position and updates the current position. Use with :func:`seek`.

        Up to `nbytes` bytes will be read, halting at the beginning of the first unmapped region
        encountered.
        """

        if self.is_concrete_target_set():
            # l.debug("invoked read %x" % (nbytes))
            return self.concrete_target.read_memory(self._pointer, nbytes)

        try:
            out = self.load(self._pointer, nbytes)
        except KeyError:
            return b''
        else:
            self._pointer += len(out)
            return out

    def seek(self, value):
        """
        The stream-like function that sets the "file's" current position. Use with :func:`read()`.

        :param value:        The position to seek to.
        """
        self._pointer = value

    def tell(self):
        return self._pointer

    def find(self, data, search_min=None, search_max=None):
        """
        Find all occurances of a bytestring in memory.

        :param bytes data:          The bytestring to search for
        :param int search_min:      Optional: The first address to include as valid
        :param int search_max:      Optional: The last address to include as valid
        :return Iterator[int]:      Iterates over addresses at which the bytestring occurs
        """
        if search_min is None:
            search_min = self.min_addr
        if search_max is None:
            search_max = self.max_addr

        for start, backer in self._backers:
            if type(backer) is Clemory:
                if search_max < backer.min_addr + start or search_min > backer.max_addr + start:
                    continue
                yield from (addr + start for addr in backer.find(data, search_min-start, search_max-start))
            elif type(backer) is list:
                raise TypeError("find is not supported for list-backed clemories")
            else:
                if search_max < start or search_min > start + len(data):
                    continue
                ptr = search_min - start - 1
                while True:
                    ptr += 1
                    ptr = backer.find(data, ptr)
                    if ptr == -1 or ptr + len(data) > search_max - start - 1:
                        break
                    yield ptr + start

    def _update_min_max(self):
        """
        Update the three properties of Clemory: consecutive, min_addr, and max_addr.
        """

        is_consecutive = True
        next_start = None
        min_addr, max_addr = None, None

        for start, backer in self._backers:
            if min_addr is None:
                min_addr = start

            if next_start is not None:
                # Check the predicted start equals to the real one
                if next_start != start:
                    is_consecutive = False

            if isinstance(backer, (bytearray, list)):
                backer_length = len(backer)
                # Update max_addr
                if max_addr is None or start + backer_length > max_addr:
                    max_addr = start + backer_length
                # Update the predicted starting address
                next_start = start + backer_length

            elif isinstance(backer, Clemory):
                if backer.max_addr is not None and backer.min_addr is not None:
                    # Update max_addr
                    if max_addr is None or start + backer.max_addr > max_addr:
                        max_addr = start + backer.max_addr
                    if backer.min_addr > 0:
                        is_consecutive = False
                    # Update the predicted starting address
                    next_start = start + backer.max_addr
            else:
                raise TypeError("Unsupported backer type %s." % type(backer))

        self.consecutive = is_consecutive
        self.min_addr = min_addr
        self.max_addr = max_addr
<commit_msg>Remove unused slots params<commit_after>
import bisect
import struct
from typing import Tuple, Union, List

__all__ = ('Clemory',)

class Clemory:

    __slots__ = ('_arch', '_backers', '_pointer', '_root',
                 'consecutive', 'min_addr', 'max_addr', 'concrete_target' )

    """
    An object representing a memory space.

    Accesses can be made with [index] notation.
    """
    def __init__(self, arch, root=False):
        self._arch = arch
        self._backers = []  # type: Tuple[int, Union[bytearray, Clemory, List[int]]]
        self._pointer = 0
        self._root = root

        self.consecutive = True
        self.min_addr = 0
        self.max_addr = 0

        self.concrete_target = None

    def is_concrete_target_set(self):
        return self.concrete_target is not None

    def set_concrete_target(self, concrete_target):
        self.concrete_target = concrete_target

    def add_backer(self, start, data):
        """
        Adds a backer to the memory.

        :param start:   The address where the backer should be loaded.
        :param data:    The backer itself. Can be either a bytestring or another :class:`Clemory`.
        """
        if not data:
            raise ValueError("Backer is empty!")

        if not isinstance(data, (bytes, list, Clemory)):
            raise TypeError("Data must be a bytes, list, or Clemory object.")
        if start in self:
            raise ValueError("Address %#x is already backed!" % start)
        if isinstance(data, Clemory) and data._root:
            raise ValueError("Cannot add a root clemory as a backer!")
        if type(data) is bytes:
            data = bytearray(data)
        bisect.insort(self._backers, (start, data))
        self._update_min_max()

    def update_backer(self, start, data):
        if not isinstance(data, (bytes, list, Clemory)):
            raise TypeError("Data must be a bytes, list, or Clemory object.")
        if type(data) is bytes:
            data = bytearray(data)
        for i, (oldstart, _) in enumerate(self._backers):
            if oldstart == start:
                self._backers[i] = (start, data)
                break
        else:
            raise ValueError("Can't find backer to update")

        self._update_min_max()

    def remove_backer(self, start):
        for i, (oldstart, _) in enumerate(self._backers):
            if oldstart == start:
                self._backers.pop(i)
                break
        else:
            raise ValueError("Can't find backer to remove")

        self._update_min_max()

    def __iter__(self):
        for start, string in self._backers:
            if isinstance(string, (bytes, list)):
                for x in range(len(string)):
                    yield start + x
            else:
                for x in string:
                    yield start + x

    def __getitem__(self, k):

        # concrete memory read
        if self.is_concrete_target_set():
            # l.debug("invoked get_byte %x" % (k))
            return self.concrete_target.read_memory(k, 1)

        for start, data in self._backers:
            if type(data) in (bytearray, list):
                if 0 <= k - start < len(data):
                    return data[k - start]
            elif isinstance(data, Clemory):
                if data.min_addr <= k - start < data.max_addr:
                    try:
                        return data[k - start]
                    except KeyError:
                        pass
        raise KeyError(k)

    def __setitem__(self, k, v):
        for start, data in self._backers:
            if type(data) in (bytearray, list):
                if 0 <= k - start < len(data):
                    data[k - start] = v
                    return
            elif isinstance(data, Clemory):
                if data.min_addr <= k - start < data.max_addr:
                    try:
                        data[k - start] = v
                        return
                    except KeyError:
                        pass
        raise KeyError(k)

    def __contains__(self, k):
        # Fast path
        if self.consecutive:
            return self.min_addr <= k < self.max_addr
        else:
            # Check if this is an empty Clemory instance
            if not self._backers:
                return False
            # Check if it is out of the memory range
            if k < self.min_addr or k >= self.max_addr:
                return False

        try:
            self.__getitem__(k)
        except KeyError:
            return False
        else:
            return True

    def __getstate__(self):
        s = {
            '_arch': self._arch,
            '_backers': self._backers,
            '_pointer': self._pointer,
            '_root': self._root,
            'consecutive': self.consecutive,
            'min_addr': self.min_addr,
            'max_addr': self.max_addr,
            'concrete_target': self.concrete_target
        }

        return s

    def __setstate__(self, s):
        self._arch = s['_arch']
        self._backers = s['_backers']
        self._pointer = s['_pointer']
        self._root = s['_root']
        self.consecutive = s['consecutive']
        self.min_addr = s['min_addr']
        self.max_addr = s['max_addr']
        self.concrete_target = s['concrete_target']

    def backers(self, addr=0):
        """
        Iterate through each backer for this clemory and all its children, yielding tuples of
        ``(start_addr, backer)`` where each backer is a bytearray.

        :param addr:    An optional starting address - all backers before and not including this
                        address will be skipped.
        """
        started = addr <= 0
        for start, backer in self._backers:
            if not started:
                end = start + backer.max_addr if type(backer) is Clemory else start + len(backer)
                if addr >= end:
                    continue
                started = True
            if type(backer) is Clemory:
                for s, b in backer.backers(addr - start):
                    yield s + start, b
            else:
                yield start, backer

    def load(self, addr, n):
        """
        Read up to `n` bytes at address `addr` in memory and return a bytes object.

        Reading will stop at the beginning of the first unallocated region found, or when
        `n` bytes have been read.
        """

        # concrete memory read
        if self.is_concrete_target_set():
            # l.debug("invoked read_bytes %x %x" % (addr, n))
            return self.concrete_target.read_memory(addr, n)

        views = []

        for start, backer in self.backers(addr):
            if start > addr:
                break
            offset = addr - start
            size = len(backer) - offset
            views.append(memoryview(backer)[offset:offset + n])

            addr += size
            n -= size

            if n <= 0:
                break

        if not views:
            raise KeyError(n)
        return b''.join(views)

    def store(self, addr, data):
        """
        Write bytes from `data` at address `addr`.

        Note: If the store runs off the end of a backer and into unbacked space, this function
        will update the backer but also raise ``KeyError``.
        """
        for start, backer in self.backers(addr):
            if start > addr:
                raise KeyError(addr)
            offset = addr - start
            size = len(backer) - offset
            backer[offset:offset + len(data)] = data if len(data) <= size else data[:size]

            addr += size
            data = data[size:]

            if not data:
                break

        if data:
            raise KeyError(addr)

    def unpack(self, addr, fmt):
        """
        Use the ``struct`` module to unpack the data at address `addr` with the format `fmt`.
        """

        try:
            start, backer = next(self.backers(addr))
        except StopIteration:
            raise KeyError(addr)

        if start > addr:
            raise KeyError(addr)

        try:
            return struct.unpack_from(fmt, backer, addr - start)
        except struct.error as e:
            if len(backer) - (addr - start) >= struct.calcsize(fmt):
                raise e
            raise KeyError(addr)

    def unpack_word(self, addr, size=None, signed=False, endness=None):
        """
        Use the ``struct`` module to unpack a single integer from the address `addr`.

        You may override any of the attributes of the word being extracted:

        :param int size:    The size in bytes to pack/unpack. Defaults to wordsize (e.g. 4 bytes on
                            a 32 bit architecture)
        :param bool signed: Whether the data should be extracted signed/unsigned. Default unsigned
        :param str archinfo.Endness: The endian to use in packing/unpacking. Defaults to memory endness
        """
        return self.unpack(addr, self._arch.struct_fmt(size=size, signed=signed, endness=endness))[0]

    def pack(self, addr, fmt, *data):
        """
        Use the ``struct`` module to pack `data` into memory at address `addr` with the format `fmt`.
        """

        try:
            start, backer = next(self.backers(addr))
        except StopIteration:
            raise KeyError(addr)

        if start > addr:
            raise KeyError(addr)

        try:
            return struct.pack_into(fmt, backer, addr - start, *data)
        except struct.error as e:
            if len(backer) - (addr - start) >= struct.calcsize(fmt):
                raise e
            raise KeyError(addr)

    def pack_word(self, addr, data, size=None, signed=False, endness=None):
        """
        Use the ``struct`` module to pack a single integer `data` into memory at the address `addr`.

        You may override any of the attributes of the word being packed:

        :param int size:    The size in bytes to pack/unpack. Defaults to wordsize (e.g. 4 bytes on
                            a 32 bit architecture)
        :param bool signed: Whether the data should be extracted signed/unsigned. Default unsigned
        :param str archinfo.Endness: The endian to use in packing/unpacking. Defaults to memory endness
        """
        if not signed:
            data &= (1 << (size*8 if size is not None else self._arch.bits)) - 1
        return self.pack(addr, self._arch.struct_fmt(size=size, signed=signed, endness=endness), data)

    def read(self, nbytes):
        """
        The stream-like function that reads up to a number of bytes starting from the current
        position and updates the current position. Use with :func:`seek`.

        Up to `nbytes` bytes will be read, halting at the beginning of the first unmapped region
        encountered.
        """

        if self.is_concrete_target_set():
            # l.debug("invoked read %x" % (nbytes))
            return self.concrete_target.read_memory(self._pointer, nbytes)

        try:
            out = self.load(self._pointer, nbytes)
        except KeyError:
            return b''
        else:
            self._pointer += len(out)
            return out

    def seek(self, value):
        """
        The stream-like function that sets the "file's" current position. Use with :func:`read()`.

        :param value:        The position to seek to.
        """
        self._pointer = value

    def tell(self):
        return self._pointer

    def find(self, data, search_min=None, search_max=None):
        """
        Find all occurances of a bytestring in memory.

        :param bytes data:          The bytestring to search for
        :param int search_min:      Optional: The first address to include as valid
        :param int search_max:      Optional: The last address to include as valid
        :return Iterator[int]:      Iterates over addresses at which the bytestring occurs
        """
        if search_min is None:
            search_min = self.min_addr
        if search_max is None:
            search_max = self.max_addr

        for start, backer in self._backers:
            if type(backer) is Clemory:
                if search_max < backer.min_addr + start or search_min > backer.max_addr + start:
                    continue
                yield from (addr + start for addr in backer.find(data, search_min-start, search_max-start))
            elif type(backer) is list:
                raise TypeError("find is not supported for list-backed clemories")
            else:
                if search_max < start or search_min > start + len(data):
                    continue
                ptr = search_min - start - 1
                while True:
                    ptr += 1
                    ptr = backer.find(data, ptr)
                    if ptr == -1 or ptr + len(data) > search_max - start - 1:
                        break
                    yield ptr + start

    def _update_min_max(self):
        """
        Update the three properties of Clemory: consecutive, min_addr, and max_addr.
        """

        is_consecutive = True
        next_start = None
        min_addr, max_addr = None, None

        for start, backer in self._backers:
            if min_addr is None:
                min_addr = start

            if next_start is not None:
                # Check the predicted start equals to the real one
                if next_start != start:
                    is_consecutive = False

            if isinstance(backer, (bytearray, list)):
                backer_length = len(backer)
                # Update max_addr
                if max_addr is None or start + backer_length > max_addr:
                    max_addr = start + backer_length
                # Update the predicted starting address
                next_start = start + backer_length

            elif isinstance(backer, Clemory):
                if backer.max_addr is not None and backer.min_addr is not None:
                    # Update max_addr
                    if max_addr is None or start + backer.max_addr > max_addr:
                        max_addr = start + backer.max_addr
                    if backer.min_addr > 0:
                        is_consecutive = False
                    # Update the predicted starting address
                    next_start = start + backer.max_addr
            else:
                raise TypeError("Unsupported backer type %s." % type(backer))

        self.consecutive = is_consecutive
        self.min_addr = min_addr
        self.max_addr = max_addr
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""Evaluates one homework.

Usage:
    ./commander.py directory - where directory contains (see submit.py)
        `archive.zip' `tests.zip' `config' `storer' `callback'

The script parses config and storer and invokes vm_executor with
the requiered arguments.

VMExecutor excepts files in `executor_jobs' so it's not safe
to run two instances of commander simultaneously.

When done `callback' is invoked with arguments
    ./callback config file1 file2 file3 ...
Missing files should be ignored (except config).

NOTE: This commander is a major HACK (ie lots of wtf)
TODO: Split VMExecutor, one for each machine.
"""

from __future__ import with_statement

__author__ = 'Alexandru Moșoi <brtzsnr@gmail.com>'

import ConfigParser
import logging
import shutil
import sys
import os
from subprocess import check_call
from os.path import join, isdir

import misc
import vmcheckerpaths

_logger = logging.getLogger("vmchecker.commander")

_FILES_TO_SEND = (
    'job_build',
    'job_run',
    'job_errors',
    'job_results',
    'job_km', )

_logger = logging.getLogger('vmchecker.commander')

def _run_callback(dir, ejobs):
    """Runs callback script to upload results"""

    args = (join(dir, 'callback'), join(dir, 'config'))
    args += tuple(join(ejobs, f) for f in _FILES_TO_SEND)

    _logger.info('Homework evaluated; sending results')
    _logger.debug('calling %s', args)

    try:
        check_call(args)
    except:
        _logger.error('Sending results failed')
        raise

def _run_executor(machine, assignment):
    # starts job
    # XXX lots of wtf per minute
    # parsing config should be executors' job
    tester = misc.tester_config()
    args = [
            # '/bin/echo',
            vmcheckerpaths.abspath('VMExecutor/vm_executor'),
            machine,
            '1',                                      # enables kernel_messages
            tester.get(machine, 'VMPath'),
            tester.get('Global', 'LocalAddress'),     # did I review commander.cpp?
            tester.get(machine, 'GuestUser'),
            tester.get(machine, 'GuestPassword'),     # XXX keys?
            tester.get(machine, 'GuestBasePath'),
            tester.get(machine, 'GuestShellPath'),
            tester.get(machine, 'GuestHomeInBash'),   # why is this needed?
            vmcheckerpaths.root(),
            assignment,
            ]
    _logger.info('Begin homework evaluation')
    _logger.debug('calling %s', args)

    try:
        check_call(args)
    except:
        _logger.error('failed to run VMExecutor')
        raise

def main(dir):
    # reads assignment config
    with open(join(dir, 'config')) as handle:
        config = ConfigParser.RawConfigParser()
        config.readfp(handle)

    # reads vmchecker_storer.ini
    with open(join(dir, 'storer')) as handle:
        storer = ConfigParser.RawConfigParser()
        storer.readfp(handle)

    # copies files to where vmchecker expects them (wtf
    # XXX 'executor_jobs' path is hardcoded in executor

    ejobs = vmcheckerpaths.abspath('executor_jobs')
    # cleans up executor_jobs, if not already clean
    if isdir(ejobs):
        shutil.rmtree(ejobs)
    os.mkdir(ejobs)

    shutil.copy(        # copies assignment
        join(dir, 'archive.zip'),
        vmcheckerpaths.abspath('executor_jobs', 'file.zip'))
    shutil.copy(        # copies tests
        join(dir, 'tests.zip'),
        vmcheckerpaths.abspath('executor_jobs', 'tests.zip'))

    try:
        assignment = config.get('Assignment', 'Assignment')
        machine = storer.get(assignment, 'Machine')

        _run_executor(machine, assignment)
        _run_callback(dir, ejobs)

        _logger.info('all done')
    except:
        _logger.exception('failed miserable')
        raise
    finally:
        # clears files
        shutil.rmtree(ejobs)

def _print_help():
    print >>sys.stderr, """Usage:
    ./commander.py directory - where directory contains (see submit.py)
        `archive.zip' `tests.zip' `config' `storer' `callback'"""

def _check_required_files(path):
    found_all = True
    needed_files = ['archive.zip', 'tests.zip', 'config', 'storer', 'callback']
    found_files = os.listdir(path)
    for need in needed_files:
        if not need in found_files:
            _logger.error('Could not find necessary file [%s] in [%s]' % (
                    need, path))
            found_all = False
    if not found_all:
        exit(-1)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    if len(sys.argv) != 2:
        print >>sys.stderr, 'Invalid number of arguments.'
        _print_help()
        exit(1)

    start_dir = sys.argv[1]
    if not os.path.isdir(start_dir):
        print >>sys.stderr, 'Not a directory', start_dir
        _print_help()
        exit(1)
    _check_required_files(start_dir)
    main(start_dir)
<commit_msg>change env to include VMCHECKER_ROOT/bin in python module search path<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""Evaluates one homework.

Usage:
    ./commander.py directory - where directory contains (see submit.py)
        `archive.zip' `tests.zip' `config' `storer' `callback'

The script parses config and storer and invokes vm_executor with
the requiered arguments.

VMExecutor excepts files in `executor_jobs' so it's not safe
to run two instances of commander simultaneously.

When done `callback' is invoked with arguments
    ./callback config file1 file2 file3 ...
Missing files should be ignored (except config).

NOTE: This commander is a major HACK (ie lots of wtf)
TODO: Split VMExecutor, one for each machine.
"""

from __future__ import with_statement

__author__ = 'Alexandru Moșoi <brtzsnr@gmail.com>'

import ConfigParser
import logging
import shutil
import sys
import os
from subprocess import check_call
from os.path import join, isdir

import misc
import vmcheckerpaths

_logger = logging.getLogger("vmchecker.commander")

_FILES_TO_SEND = (
    'job_build',
    'job_run',
    'job_errors',
    'job_results',
    'job_km', )

_logger = logging.getLogger('vmchecker.commander')

def _env_with_python_module_search_path():
    """ Setup python module search path to include '$VMCHECKER_ROOT/bin'
    so that the callback python script can access misc, vmcheckerpaths, etc.
    """
    e = os.environ
    module_search_path = os.path.join(vmcheckerpaths.root(), 'bin')
    if 'PYTHONPATH' in e.keys():
        newval = os.pathsep.join(e['PYTHONPATH'], module_search_path)
    e['PYTHONPATH'] = module_search_path
    return e

def _run_callback(dir, ejobs):
    """Runs callback script to upload results"""

    args = (join(dir, 'callback'), join(dir, 'config'))
    args += tuple(join(ejobs, f) for f in _FILES_TO_SEND)

    _logger.info('Homework evaluated; sending results')
    _logger.debug('calling %s', args)

    try:
        env = _env_with_python_module_search_path()
        check_call(args=args, env=env)
    except:
        _logger.error('Sending results failed')
        raise

def _run_executor(machine, assignment):
    # starts job
    # XXX lots of wtf per minute
    # parsing config should be executors' job
    tester = misc.tester_config()
    args = [
            # '/bin/echo',
            vmcheckerpaths.abspath('VMExecutor/vm_executor'),
            machine,
            '1',                                      # enables kernel_messages
            tester.get(machine, 'VMPath'),
            tester.get('Global', 'LocalAddress'),     # did I review commander.cpp?
            tester.get(machine, 'GuestUser'),
            tester.get(machine, 'GuestPassword'),     # XXX keys?
            tester.get(machine, 'GuestBasePath'),
            tester.get(machine, 'GuestShellPath'),
            tester.get(machine, 'GuestHomeInBash'),   # why is this needed?
            vmcheckerpaths.root(),
            assignment,
            ]
    _logger.info('Begin homework evaluation')
    _logger.debug('calling %s', args)

    try:
        check_call(args)
    except:
        _logger.error('failed to run VMExecutor')
        raise

def main(dir):
    # reads assignment config
    with open(join(dir, 'config')) as handle:
        config = ConfigParser.RawConfigParser()
        config.readfp(handle)

    # reads vmchecker_storer.ini
    with open(join(dir, 'storer')) as handle:
        storer = ConfigParser.RawConfigParser()
        storer.readfp(handle)

    # copies files to where vmchecker expects them (wtf
    # XXX 'executor_jobs' path is hardcoded in executor

    ejobs = vmcheckerpaths.abspath('executor_jobs')
    # cleans up executor_jobs, if not already clean
    if isdir(ejobs):
        shutil.rmtree(ejobs)
    os.mkdir(ejobs)

    shutil.copy(        # copies assignment
        join(dir, 'archive.zip'),
        vmcheckerpaths.abspath('executor_jobs', 'file.zip'))
    shutil.copy(        # copies tests
        join(dir, 'tests.zip'),
        vmcheckerpaths.abspath('executor_jobs', 'tests.zip'))

    try:
        assignment = config.get('Assignment', 'Assignment')
        machine = storer.get(assignment, 'Machine')

        _run_executor(machine, assignment)
        _run_callback(dir, ejobs)

        _logger.info('all done')
    except:
        _logger.exception('failed miserable')
        raise
    finally:
        # clears files
        shutil.rmtree(ejobs)

def _print_help():
    print >>sys.stderr, """Usage:
    ./commander.py directory - where directory contains (see submit.py)
        `archive.zip' `tests.zip' `config' `storer' `callback'"""

def _check_required_files(path):
    found_all = True
    needed_files = ['archive.zip', 'tests.zip', 'config', 'storer', 'callback']
    found_files = os.listdir(path)
    for need in needed_files:
        if not need in found_files:
            _logger.error('Could not find necessary file [%s] in [%s]' % (
                    need, path))
            found_all = False
    if not found_all:
        exit(-1)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    if len(sys.argv) != 2:
        print >>sys.stderr, 'Invalid number of arguments.'
        _print_help()
        exit(1)

    start_dir = sys.argv[1]
    if not os.path.isdir(start_dir):
        print >>sys.stderr, 'Not a directory', start_dir
        _print_help()
        exit(1)
    _check_required_files(start_dir)
    main(start_dir)
<|endoftext|>