<commit_before>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2013, The SAGA Project"
__license__   = "MIT"

__author__    = ["Ole Weidner", "Andre Merzky"]
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

import os
import sys
import saga
import saga.utils.test_config as sutc

from copy import deepcopy

js = None

def test_job_service_get_url () :

    try:
        global js
        tc = sutc.TestConfig()
        js = saga.job.Service (tc.js_url, tc.session)
        assert js, "job service creation failed?"
        assert (tc.js_url == str(js.url)), "%s == %s" % (tc.js_url, str(js.url))

    except saga.SagaException as ex:
      assert False, "unexpected exception %s" % ex

# # ------------------------------------------------------------------------------
# #
# def test_job_invalid_session():
#     """ Test if an invalid session results in a proper exception
#     """
#     try:
#         tc = sutc.TestConfig()
#         # generate an invalid session
#         valid_session   = tc.session
#         invalid_session = saga.Session(default=False)
#
#         for valid_context in valid_session.contexts :
#             invalid_context = saga.Context (valid_context.type)
#
#             for a in valid_context.list_attributes () :
#                 invalid_context.set_attribute (a, valid_context.get_attribute (a) + "-hahaha")
#
#             invalid_session.add_context (invalid_context)
#
#         js = saga.job.Service(tc.js_url, invalid_session)
#         assert False, "Expected XYZ exception but got none."
#
#     except saga.BadParameter:
#         assert True
#     except saga.SagaException as ex:
#         assert False, "Expected BadParameter exception, but got %s" % ex

# ------------------------------------------------------------------------------
#
def test_job_service_invalid_url():
    """ Test if a non-resolvable hostname results in a proper exception
    """
    try:
        tc = sutc.TestConfig()
        invalid_url       = deepcopy(saga.Url(tc.js_url))
        invalid_url.host  = "does.not.exist"
        tmp_js = saga.job.Service(invalid_url, tc.session)
        assert False, "Expected XYZ exception but got none."

        del tmp_js

    except saga.BadParameter:
        assert True
    except saga.SagaException as ex:
        assert False, "Expected BadParameter exception, but got %s" % ex

# ------------------------------------------------------------------------------
#
def test_job_service_create():
    """ Test service.create_job() - expecting state 'NEW'
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)
        assert j1.state == j1.get_state()
        assert j1.state == saga.job.NEW

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_run():
    """ Test job.run() - expecting state: RUNNING/PENDING
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()
        assert (j1.state in [saga.job.RUNNING, saga.job.PENDING])
        assert j1.state == j1.get_state()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_wait():
    """ Test job.wait() - expecting state: DONE (this test might take a while)
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()
        j1.wait()
        assert j1.state == saga.job.DONE

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
# ------------------------------------------------------------------------------
#
def test_job_multiline_run():
    """ Test job.run() with multiline command
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sh'
        jd.arguments = ["""-c "python -c '
import time
if True :
  if True :
    time.sleep (3)
'
"
"""]

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)
        j1 = js.create_job(jd)

        j1.run()
        assert (j1.state in [saga.job.RUNNING, saga.job.PENDING])
        assert j1.state == j1.get_state()
        j1.wait()
        assert (j1.state == saga.job.DONE), "%s == %s" % (j1.state, saga.job.DONE)

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_suspend_resume():
    """ Test job.suspend()/resume() - expecting state: SUSPENDED/RUNNIG
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)
        j1.run()

        j1.suspend()
        assert j1.state == saga.job.SUSPENDED
        assert j1.state == j1.get_state()

        j1.resume()
        assert j1.state == saga.job.RUNNING
        assert j1.state == j1.get_state()

        j1.cancel()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_cancel():
    """ Test job.cancel() - expecting state: CANCELED
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()
        j1.cancel()
        assert j1.state == saga.job.CANCELED

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_states_OLD():
    """ Test job states (disabled) """

    return 0

    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['3']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j3 = js.run_job ("/bin/sleep 3 ; /bin/true")
        assert j3.state == saga.job.RUNNING

        j3.wait ()
        assert j3.state == saga.job.DONE

        j4 = js.run_job ("/bin/sleep 3 ; /bin/false")
        assert j4.state == saga.job.RUNNING

        j4.wait ()
        assert j4.state == saga.job.FAILED

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_exit_code():
    """ Test job.exit_code
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = "/bin/sh"
        jd.arguments = ["-c \"exit 3\""]

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)
        j.run()
        j.wait()

        ec = j.exit_code
        assert ec == 3, "%s != 3" % ec

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_service_url():
    """ Test if job.service_url == Service.url
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)

        assert j.service_url == js.url

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_id():
    """ Test job.get_id() / job.id
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)
        j.run()

        assert j.id != None
        assert j.id == j.get_id()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

<commit_msg>fixed race condition<commit_after>
__author__    = "Andre Merzky, Ole Weidner"
__copyright__ = "Copyright 2013, The SAGA Project"
__license__   = "MIT"

__author__    = ["Ole Weidner", "Andre Merzky"]
__copyright__ = "Copyright 2012-2013, The SAGA Project"
__license__   = "MIT"

import os
import sys
import saga
import saga.utils.test_config as sutc

from copy import deepcopy

js = None

def test_job_service_get_url () :

    try:
        global js
        tc = sutc.TestConfig()
        js = saga.job.Service (tc.js_url, tc.session)
        assert js, "job service creation failed?"
        assert (tc.js_url == str(js.url)), "%s == %s" % (tc.js_url, str(js.url))

    except saga.SagaException as ex:
      assert False, "unexpected exception %s" % ex

# # ------------------------------------------------------------------------------
# #
# def test_job_invalid_session():
#     """ Test if an invalid session results in a proper exception
#     """
#     try:
#         tc = sutc.TestConfig()
#         # generate an invalid session
#         valid_session   = tc.session
#         invalid_session = saga.Session(default=False)
#
#         for valid_context in valid_session.contexts :
#             invalid_context = saga.Context (valid_context.type)
#
#             for a in valid_context.list_attributes () :
#                 invalid_context.set_attribute (a, valid_context.get_attribute (a) + "-hahaha")
#
#             invalid_session.add_context (invalid_context)
#
#         js = saga.job.Service(tc.js_url, invalid_session)
#         assert False, "Expected XYZ exception but got none."
#
#     except saga.BadParameter:
#         assert True
#     except saga.SagaException as ex:
#         assert False, "Expected BadParameter exception, but got %s" % ex

# ------------------------------------------------------------------------------
#
def test_job_service_invalid_url():
    """ Test if a non-resolvable hostname results in a proper exception
    """
    try:
        tc = sutc.TestConfig()
        invalid_url       = deepcopy(saga.Url(tc.js_url))
        invalid_url.host  = "does.not.exist"
        tmp_js = saga.job.Service(invalid_url, tc.session)
        assert False, "Expected XYZ exception but got none."

        del tmp_js

    except saga.BadParameter:
        assert True
    except saga.SagaException as ex:
        assert False, "Expected BadParameter exception, but got %s" % ex

# ------------------------------------------------------------------------------
#
def test_job_service_create():
    """ Test service.create_job() - expecting state 'NEW'
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)
        assert j1.state == j1.get_state()
        assert j1.state == saga.job.NEW

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_run():
    """ Test job.run() - expecting state: RUNNING/PENDING
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()

        assert (j1.state in [saga.job.RUNNING, saga.job.PENDING])

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_wait():
    """ Test job.wait() - expecting state: DONE (this test might take a while)
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()
        j1.wait()
        assert j1.state == saga.job.DONE, "%s != %s" % (j1.state, saga.job.DONE)

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se
# ------------------------------------------------------------------------------
#
def test_job_multiline_run():
    """ Test job.run() with multiline command
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sh'
        jd.arguments = ["""-c "python -c '
import time
if True :
  if True :
    time.sleep (3)
'
"
"""]

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)
        j1 = js.create_job(jd)

        j1.run()
        assert (j1.state in [saga.job.RUNNING, saga.job.PENDING])
        j1.wait()
        assert (j1.state == saga.job.DONE), "%s == %s" % (j1.state, saga.job.DONE)

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_suspend_resume():
    """ Test job.suspend()/resume() - expecting state: SUSPENDED/RUNNIG
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)
        j1.run()

        j1.suspend()
        assert j1.state == saga.job.SUSPENDED
        assert j1.state == j1.get_state()

        j1.resume()
        assert j1.state == saga.job.RUNNING
        assert j1.state == j1.get_state()

        j1.cancel()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_cancel():
    """ Test job.cancel() - expecting state: CANCELED
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j1 = js.create_job(jd)

        j1.run()
        j1.cancel()
        assert j1.state == saga.job.CANCELED

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_job_states_OLD():
    """ Test job states (disabled) """

    return 0

    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()
        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['3']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j3 = js.run_job ("/bin/sleep 3 ; /bin/true")
        assert j3.state == saga.job.RUNNING

        j3.wait ()
        assert j3.state == saga.job.DONE

        j4 = js.run_job ("/bin/sleep 3 ; /bin/false")
        assert j4.state == saga.job.RUNNING

        j4.wait ()
        assert j4.state == saga.job.FAILED

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_exit_code():
    """ Test job.exit_code
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = "/bin/sh"
        jd.arguments = ["-c \"exit 3\""]

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)
        j.run()
        j.wait()

        ec = j.exit_code
        assert ec == 3, "%s != 3" % ec

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_service_url():
    """ Test if job.service_url == Service.url
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)

        assert j.service_url == js.url

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

# ------------------------------------------------------------------------------
#
def test_get_id():
    """ Test job.get_id() / job.id
    """
    try:
        global js
        assert js, "no job service"
        tc = sutc.TestConfig()

        jd = saga.job.Description()
        jd.executable = '/bin/sleep'
        jd.arguments = ['10']

        # add options from the test .cfg file if set
        jd = sutc.add_tc_params_to_jd(tc=tc, jd=jd)

        j = js.create_job(jd)
        j.run()

        assert j.id != None
        assert j.id == j.get_id()

    except saga.NotImplemented as ni:
            assert tc.notimpl_warn_only, "%s " % ni
            if tc.notimpl_warn_only:
                print "%s " % ni
    except saga.SagaException as se:
        assert False, "Unexpected exception: %s" % se

<|endoftext|>


<commit_before>from jinja2 import Template

import yaml

def yaml_to_md(layer_name):
    out.write('## ' + layer_name + '\n\n')
    data = yaml.load(inp)

    layer = data['layers'][layer_name]
    attributes = layer['attributes']

    # values of layer attribute "type" (not to be confused with an attribute's type)
    type_values = attributes["type"]['values']

    with open('templates/layer_attributes.md.jinja2') as f:
        layer_attributes_template = Template(f.read())
    out.write(layer_attributes_template.render(attributes=attributes))

    if any('correlated_attributes' in definition for definition in type_values.values()):
        with open('templates/attribute_values_with_aggtype.md.jinja2') as f:
            attribute_values_template = Template(f.read())
    else:
        with open('templates/attribute_values.md.jinja2') as f:
            attribute_values_template = Template(f.read())

    out.write(attribute_values_template.render(type_values=type_values))
    out.write('\n\n')

with open("osmaxx_schema.yaml", 'r') as in_file:
    inp = in_file.read()
with open("documentation.md", 'w') as out_file:
    out = out_file
    yaml_to_md('adminarea_a')
    yaml_to_md('building_a')
    yaml_to_md('geoname_p')
    yaml_to_md('landuse_a')
    yaml_to_md('military_p')
    yaml_to_md('misc_l')
    yaml_to_md('natural_a')
    yaml_to_md('nonop_l')
    yaml_to_md('poi_p')
    yaml_to_md('pow_p')
    yaml_to_md('railway_bridge_l')
    yaml_to_md('road_ground_l')
    yaml_to_md('route_l')
    yaml_to_md('traffic_a')
    yaml_to_md('traffic_p')
    yaml_to_md('transport_a')
    yaml_to_md('utility_a')
    yaml_to_md('utility_p')
    yaml_to_md('utility_l')
    yaml_to_md('water_a')
    yaml_to_md('water_p')
    yaml_to_md('water_l')
<commit_msg>only parse the YAML document once<commit_after>from jinja2 import Template

import yaml

def yaml_to_md(layer_name):
    out.write('## ' + layer_name + '\n\n')

    layer = data['layers'][layer_name]
    attributes = layer['attributes']

    # values of layer attribute "type" (not to be confused with an attribute's type)
    type_values = attributes["type"]['values']

    with open('templates/layer_attributes.md.jinja2') as f:
        layer_attributes_template = Template(f.read())
    out.write(layer_attributes_template.render(attributes=attributes))

    if any('correlated_attributes' in definition for definition in type_values.values()):
        with open('templates/attribute_values_with_aggtype.md.jinja2') as f:
            attribute_values_template = Template(f.read())
    else:
        with open('templates/attribute_values.md.jinja2') as f:
            attribute_values_template = Template(f.read())

    out.write(attribute_values_template.render(type_values=type_values))
    out.write('\n\n')

with open("osmaxx_schema.yaml", 'r') as in_file:
    inp = in_file.read()
    data = yaml.load(inp)
with open("documentation.md", 'w') as out_file:
    out = out_file
    yaml_to_md('adminarea_a')
    yaml_to_md('building_a')
    yaml_to_md('geoname_p')
    yaml_to_md('landuse_a')
    yaml_to_md('military_p')
    yaml_to_md('misc_l')
    yaml_to_md('natural_a')
    yaml_to_md('nonop_l')
    yaml_to_md('poi_p')
    yaml_to_md('pow_p')
    yaml_to_md('railway_bridge_l')
    yaml_to_md('road_ground_l')
    yaml_to_md('route_l')
    yaml_to_md('traffic_a')
    yaml_to_md('traffic_p')
    yaml_to_md('transport_a')
    yaml_to_md('utility_a')
    yaml_to_md('utility_p')
    yaml_to_md('utility_l')
    yaml_to_md('water_a')
    yaml_to_md('water_p')
    yaml_to_md('water_l')
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import netaddr
from textwrap import dedent
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.system.network import IP_NONNETWORK
from pyfarm.models.mixins import StateValidationMixin
from pyfarm.models.core import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE, IDColumn)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTaggingMixin(object):
    """
    Mixin used which provides some common structures to
    :class:`.AgentTagsModel` and :class:`.AgentSoftwareModel`
    """
    @validates("tag", "software")
    def validate_string_column(self, key, value):
        """
        Ensures `value` is a string or something that can be converted
        to a string.
        """
        if isinstance(value, (int, long)):
            value = str(value)
        elif not isinstance(value, basestring):
            raise ValueError("expected a string for `%s`" % key)

        return value

class AgentTagsModel(db.Model, AgentTaggingMixin):
    """
    Table model used to store tags for an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`tag`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_TAGS
    __table_args__ = (UniqueConstraint("_agentid", "tag"), )
    id = IDColumn()
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    tag = db.Column(db.String,
                    doc=dedent("""
                    A string value to tag an :class:`.AgentModel`. Generally
                    this value is used for grouping like resources together
                    on the network but could also be used by jobs as a sort of
                    requirement."""))

class AgentSoftwareModel(db.Model, AgentTaggingMixin):
    """
    Stores information about an the software installed on
    an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`version`
            * :attr:`software`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_SOFTWARE
    __table_args__ = (UniqueConstraint("_agentid", "version", "software"), )
    id = IDColumn()
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    software = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The name of the software installed.  No normalization
                         is performed prior to being stored in the database"""))
    version = db.Column(db.String, default="any", nullable=False,
                        doc=dedent("""
                        The version of the software installed on a host.  This
                        value does not follow any special formatting rules
                        because the format depends on the 3rd party."""))

class AgentModel(db.Model, StateValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`hostname`
            * :attr:`ip`
            * :attr:`subnet`
            * :attr:`port`

    """
    __tablename__ = TABLE_AGENT
    __table_args__ = (UniqueConstraint("hostname", "ip", "subnet", "port"), )
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE
    id = IDColumn()

    # host state information
    enabled = db.Column(db.Boolean, default=True, nullable=False,
                        doc=dedent("""
                        Tells the queue and various other parts of PyFarm if
                        this host is considered part of the pool.  Setting
                        this to True will prevent:

                        * new work from being sent to an agent
                        * failed job types from rerunning
                        * batch commands being picked up over the pub/sub
                          channel"""))
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False,
                      doc=dedent("""
                      Stores the current state of the host.  This value can be
                      changed either by a master telling the host to do
                      something with a task or from the host via REST api.

                      .. csv-table:: **Values (from enum.yml:AgentState)**
                          :header: Integer, Description
                          :widths: 10, 50

                          16,Offline - host is unreachable
                          17,Online - ready to receive work
                          18,Disabled - same as online but cannot receive work
                          19,Running - currently processing work"""))
    hostname = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The hostname we should use to talk to this host.
                         Preferably this value will be the fully qualified
                         name instead of the base hostname alone."""))
    ip = db.Column(db.String(15), nullable=False,
                   doc=dedent("""
                   The IPv4 network address this host resides on.  This is
                   'best guess' address using factors described in
                   :meth:`pyfarm.system.network.NetworkInfo.ip`"""))
    subnet = db.Column(db.String(15), nullable=False,
                       doc=dedent("""
                       The subnet associated with the agent at the time we
                       discovered :attr:`ip`"""))
    ram = db.Column(db.Integer, nullable=False,
                    doc=dedent("""
                    The amount of ram (in megabytes) installed on the agent.
                    This value is provided by
                    :attr:`pyfarm.system.memory.MemoryInfo.TOTAL_RAM`"""))
    cpus = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The number of cpus installed on the agent.  This value
                     is provided by
                     :attr:`pyfarm.system.processor.ProcessorInfo.NUM_CPUS`
                     """))
    port = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The port the agent is currently running on"""))

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.ram_allocation", .8),
                               doc=dedent("""
                               The amount of ram the agent is allowed to
                               allocate towards work.  A value of `1.0` would
                               mean to let the agent use all of the memory
                               installed on the system when assigning work.

                               **configured by**: `agent.ram_allocation`"""))
    cpu_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.cpu_allocation", 1.0),
                               doc=dedent("""
                               The total amount of cpu space an agent is
                               allowed to process work in.  A value of `1.0`
                               would mean an agent can handle as much work
                               as the system could handle given the
                               requirements of a task.  For example if an agent
                               has 8 cpus, cpu_allocation is .5, and a task
                               requires 4 cpus then only that task will run
                               on the system.

                               **configured by**: `agent.cpu_allocation`"""))

    # relationships
    tasks = db.relationship("TaskModel", backref="agent", lazy="dynamic",
                            doc=dedent("""
                            Relationship between an :class:`AgentModel`
                            and any :class:`pyfarm.models.TaskModel`
                            objects"""))
    tags = db.relationship("AgentTagsModel", backref="agent", lazy="dynamic",
                           doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentTagsModel`
                           objects"""))
    software = db.relation("AgentSoftwareModel", backref="agent",
                           lazy="dynamic", doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentSoftwareModel`
                           objects"""))

    @validates("hostname")
    def validate_hostname(self, key, value):
        """
        Ensures that the hostname provided by `value` matches a regular
        expression that expresses what a valid hostname is.
        """
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        """
        Ensures the :attr:`ip` and :attr:`subnet` are valid.  For ip addresses
        this will make sure `value` is:

            * not a hostmask
            * not link local (:rfc:`3927`)
            * not used for multicast (:rfc:`1112`)
            * not a netmask (:rfc:`4632`)
            * not reserved (:rfc:`6052`)
            * a private address (:rfc:`1918`)

        This method will also value subnet masks to ensure their format is
        valid.
        """
        try:
            ip = netaddr.IPAddress(value)

        except ValueError, e:
            raise ValueError(
                "%s is not a valid address format: %s" % (value, e))

        if key == "ip":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                not ip.is_netmask(), ip.is_private(),
                not ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s it not a private ip address" % value)

        elif key == "subnet":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                ip.is_netmask(), not ip.is_private(),
                ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s is not valid subnet" % value)

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        """
        Ensure the `value` provided for `key` is within an expected range as
        specified in `agent.yml`
        """
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

class AgentTag(AgentTagsModel):
    """
    Provides :meth:`__init__` for :class:`.AgentTagsModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, tag):
        if isinstance(agent, Agent):
            agentid = agent.id
        else:
            agentid = agent

        self._agentid = agentid
        self.tag = tag

class AgentSoftware(AgentSoftwareModel):
    """
    Provides :meth:`__init__` for :class:`AgentSoftwareModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, software, version="any"):
        if isinstance(agent, Agent):
            agentid = agent.id
        else:
            agentid = agent

        self._agentid = agentid
        self.software = software
        self.version = version

class Agent(AgentModel):
    """
    Provides :meth:`__init__` for :class:`AgentModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, hostname, ip, subnet, state=None, enabled=None,
                 ram=None, cpus=None, port=None, ram_allocation=None,
                 cpu_allocation=None):
        self.hostname = hostname
        self.ip = ip
        self.subnet = subnet

        if state is not None:
            self.state = state

        if enabled is not None:
            self.enabled = enabled

        if ram is not None:
            self.ram = ram

        if cpus is not None:
            self.cpus = cpus

        if port is not None:
            self.port = port

        if ram is not None:
            self.ram_allocation = ram_allocation

        if cpu_allocation is not None:
            self.cpu_allocation = cpu_allocation<commit_msg>should be checking against AgentModel<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import netaddr
from textwrap import dedent
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.system.network import IP_NONNETWORK
from pyfarm.models.mixins import StateValidationMixin
from pyfarm.models.core import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE, IDColumn)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTaggingMixin(object):
    """
    Mixin used which provides some common structures to
    :class:`.AgentTagsModel` and :class:`.AgentSoftwareModel`
    """
    @validates("tag", "software")
    def validate_string_column(self, key, value):
        """
        Ensures `value` is a string or something that can be converted
        to a string.
        """
        if isinstance(value, (int, long)):
            value = str(value)
        elif not isinstance(value, basestring):
            raise ValueError("expected a string for `%s`" % key)

        return value

class AgentTagsModel(db.Model, AgentTaggingMixin):
    """
    Table model used to store tags for an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`tag`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_TAGS
    __table_args__ = (UniqueConstraint("_agentid", "tag"), )
    id = IDColumn()
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    tag = db.Column(db.String,
                    doc=dedent("""
                    A string value to tag an :class:`.AgentModel`. Generally
                    this value is used for grouping like resources together
                    on the network but could also be used by jobs as a sort of
                    requirement."""))

class AgentSoftwareModel(db.Model, AgentTaggingMixin):
    """
    Stores information about an the software installed on
    an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`version`
            * :attr:`software`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_SOFTWARE
    __table_args__ = (UniqueConstraint("_agentid", "version", "software"), )
    id = IDColumn()
    _agentid = db.Column(db.Integer, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    software = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The name of the software installed.  No normalization
                         is performed prior to being stored in the database"""))
    version = db.Column(db.String, default="any", nullable=False,
                        doc=dedent("""
                        The version of the software installed on a host.  This
                        value does not follow any special formatting rules
                        because the format depends on the 3rd party."""))

class AgentModel(db.Model, StateValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`hostname`
            * :attr:`ip`
            * :attr:`subnet`
            * :attr:`port`

    """
    __tablename__ = TABLE_AGENT
    __table_args__ = (UniqueConstraint("hostname", "ip", "subnet", "port"), )
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE
    id = IDColumn()

    # host state information
    enabled = db.Column(db.Boolean, default=True, nullable=False,
                        doc=dedent("""
                        Tells the queue and various other parts of PyFarm if
                        this host is considered part of the pool.  Setting
                        this to True will prevent:

                        * new work from being sent to an agent
                        * failed job types from rerunning
                        * batch commands being picked up over the pub/sub
                          channel"""))
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False,
                      doc=dedent("""
                      Stores the current state of the host.  This value can be
                      changed either by a master telling the host to do
                      something with a task or from the host via REST api.

                      .. csv-table:: **Values (from enum.yml:AgentState)**
                          :header: Integer, Description
                          :widths: 10, 50

                          16,Offline - host is unreachable
                          17,Online - ready to receive work
                          18,Disabled - same as online but cannot receive work
                          19,Running - currently processing work"""))
    hostname = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The hostname we should use to talk to this host.
                         Preferably this value will be the fully qualified
                         name instead of the base hostname alone."""))
    ip = db.Column(db.String(15), nullable=False,
                   doc=dedent("""
                   The IPv4 network address this host resides on.  This is
                   'best guess' address using factors described in
                   :meth:`pyfarm.system.network.NetworkInfo.ip`"""))
    subnet = db.Column(db.String(15), nullable=False,
                       doc=dedent("""
                       The subnet associated with the agent at the time we
                       discovered :attr:`ip`"""))
    ram = db.Column(db.Integer, nullable=False,
                    doc=dedent("""
                    The amount of ram (in megabytes) installed on the agent.
                    This value is provided by
                    :attr:`pyfarm.system.memory.MemoryInfo.TOTAL_RAM`"""))
    cpus = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The number of cpus installed on the agent.  This value
                     is provided by
                     :attr:`pyfarm.system.processor.ProcessorInfo.NUM_CPUS`
                     """))
    port = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The port the agent is currently running on"""))

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.ram_allocation", .8),
                               doc=dedent("""
                               The amount of ram the agent is allowed to
                               allocate towards work.  A value of `1.0` would
                               mean to let the agent use all of the memory
                               installed on the system when assigning work.

                               **configured by**: `agent.ram_allocation`"""))
    cpu_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.cpu_allocation", 1.0),
                               doc=dedent("""
                               The total amount of cpu space an agent is
                               allowed to process work in.  A value of `1.0`
                               would mean an agent can handle as much work
                               as the system could handle given the
                               requirements of a task.  For example if an agent
                               has 8 cpus, cpu_allocation is .5, and a task
                               requires 4 cpus then only that task will run
                               on the system.

                               **configured by**: `agent.cpu_allocation`"""))

    # relationships
    tasks = db.relationship("TaskModel", backref="agent", lazy="dynamic",
                            doc=dedent("""
                            Relationship between an :class:`AgentModel`
                            and any :class:`pyfarm.models.TaskModel`
                            objects"""))
    tags = db.relationship("AgentTagsModel", backref="agent", lazy="dynamic",
                           doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentTagsModel`
                           objects"""))
    software = db.relation("AgentSoftwareModel", backref="agent",
                           lazy="dynamic", doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentSoftwareModel`
                           objects"""))

    @validates("hostname")
    def validate_hostname(self, key, value):
        """
        Ensures that the hostname provided by `value` matches a regular
        expression that expresses what a valid hostname is.
        """
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        """
        Ensures the :attr:`ip` and :attr:`subnet` are valid.  For ip addresses
        this will make sure `value` is:

            * not a hostmask
            * not link local (:rfc:`3927`)
            * not used for multicast (:rfc:`1112`)
            * not a netmask (:rfc:`4632`)
            * not reserved (:rfc:`6052`)
            * a private address (:rfc:`1918`)

        This method will also value subnet masks to ensure their format is
        valid.
        """
        try:
            ip = netaddr.IPAddress(value)

        except ValueError, e:
            raise ValueError(
                "%s is not a valid address format: %s" % (value, e))

        if key == "ip":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                not ip.is_netmask(), ip.is_private(),
                not ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s it not a private ip address" % value)

        elif key == "subnet":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                ip.is_netmask(), not ip.is_private(),
                ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s is not valid subnet" % value)

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        """
        Ensure the `value` provided for `key` is within an expected range as
        specified in `agent.yml`
        """
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

class AgentTag(AgentTagsModel):
    """
    Provides :meth:`__init__` for :class:`.AgentTagsModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, tag):
        if isinstance(agent, AgentModel):
            agentid = agent.id
        else:
            agentid = agent

        self._agentid = agentid
        self.tag = tag

class AgentSoftware(AgentSoftwareModel):
    """
    Provides :meth:`__init__` for :class:`AgentSoftwareModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, software, version="any"):
        if isinstance(agent, AgentModel):
            agentid = agent.id
        else:
            agentid = agent

        self._agentid = agentid
        self.software = software
        self.version = version

class Agent(AgentModel):
    """
    Provides :meth:`__init__` for :class:`AgentModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, hostname, ip, subnet, state=None, enabled=None,
                 ram=None, cpus=None, port=None, ram_allocation=None,
                 cpu_allocation=None):
        self.hostname = hostname
        self.ip = ip
        self.subnet = subnet

        if state is not None:
            self.state = state

        if enabled is not None:
            self.enabled = enabled

        if ram is not None:
            self.ram = ram

        if cpus is not None:
            self.cpus = cpus

        if port is not None:
            self.port = port

        if ram is not None:
            self.ram_allocation = ram_allocation

        if cpu_allocation is not None:
            self.cpu_allocation = cpu_allocation<|endoftext|>


<commit_before>import random

print "Correct!" if raw_input("Guess (1 - 100): ") == str(random.randint(1, 100)) else "Incorrect!"
<commit_msg>Allows player to guess the number mutiple times.<commit_after>import random

number = str(random.randint(1, 100))
while True:
    if raw_input("Guess (1 - 100): ") == number:
        print "Correct!"
        break
    else:
        print "Incorrect, Guess Again!"
<|endoftext|>


<commit_before>from .helpers import public_api
from .twitter import TwitterBasePlugin
from ircbot.plugin import BotPlugin

class UrlShow(TwitterBasePlugin):
    github_repo = r'.*https?://github\.com/([\w\d_\.-]+)/([\w\d_\.-]+)'
    urlab_url = r'.*https?://urlab\.be'

    @BotPlugin.command(r'.*https?://twitter.com/[^/]+/status/(\d+)')
    def twitter_status(self, msg):
        url = 'statuses/show/{}.json'.format(msg.args[0])
        tweet = yield from self.twitter_request('GET', url)
        msg.reply(self.format_tweet(tweet))

    @BotPlugin.command(github_repo + r'/?(\s|$)')
    def github(self, msg):
        url = "https://api.github.com/repos/{}/{}".format(*msg.args)
        repo = yield from public_api(url)
        repo['name'] = self.bot.text.bold(repo['name'])
        repo['language'] = self.bot.text.purple('[', repo['language'], ']')
        repo['stars'] = self.bot.text.yellow('(', repo['stargazers_count'], '*)')
        fmt = "{name} {language} {stars}: «{description}»"
        msg.reply(fmt.format(**repo))

    @BotPlugin.command(github_repo + r'/(issues|pull)/(\d+)')
    def github_issue(self, msg):
        user, repo, kind, id = msg.args
        args = user, repo, id
        url = "https://api.github.com/repos/{}/{}/issues/{}".format(*args)
        issue = yield from public_api(url)
        issue['author'] = self.bot.text.bold('@' + issue['user']['login'])
        issue['labels'] = ' '.join(self.bot.text.grey('(%s)') % x['name'] for x in issue['labels'])
        if issue['state'] == 'open':
            issue['number'] = self.bot.text.green('[#', issue['number'], ' (open)]')
        elif issue['state'] == 'closed':
            issue['number'] = self.bot.text.red('[#', issue['number'], ' (closed)]')
        fmt = "{author} {number}: «{title}» {labels}"
        msg.reply(fmt.format(**issue))

    @BotPlugin.command(github_repo + r'/commit/([0-9a-fA-F]{,40})')
    def github_commit(self, msg):
        url = "https://api.github.com/repos/{}/{}/commits/{}".format(*msg.args)
        commit = yield from public_api(url)
        commit['author'] = self.bot.text.bold('@' + commit['author']['login'])
        commit['title'] = commit['commit']['message']
        additions = self.bot.text.green("%d+" % commit['stats']['additions'])
        deletions = self.bot.text.red("%d-" % commit['stats']['deletions'])
        files_changed = self.bot.text.yellow("%d files" % len(commit['files']))
        commit['stats'] = " ".join([additions, deletions, files_changed])
        msg.reply("{author} «{title}» ({stats})".format(**commit))

    @BotPlugin.command(r'.*https?://www\.reddit\.com/r/([\w\d_\.-]+)/comments/([\w\d_\.-]+)')
    def reddit(self, msg):
        url = "https://api.bot.text.reddit.com/r/{}/comments/{}".format(*msg.args[:2])
        data = yield from public_api(url)
        post = data[0]['data']['children'][0]['data']
        post['author'] = self.bot.text.bold('@' + post['author'])
        post['upvote_ratio'] = self.bot.text.yellow('(', post['upvote_ratio'], '+)')
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{author} {upvote_ratio}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(r'.*https?://news\.ycombinator\.com/item\?id=(\d+)')
    def hackernews(self, msg):
        url = "https://hacker-news.firebaseio.com/v0/item/"
        post = yield from public_api(url + "{}.json".format(msg.args[0]))
        post['by'] = self.bot.text.bold('@', post['by'])
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{by}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(urlab_url + r'/(projects/\d+)')
    def urlab_project(self, msg):
        project_status = {
            'p': self.bot.text.yellow,  # Proposition
            'i': self.bot.text.blue,    # In progress
            'f': self.bot.text.green,   # Finished
        }

        proj = yield from public_api(msg.args[0])
        color = project_status.get(proj['status'], self.bot.text.grey)
        proj['desc'] = color(proj['short_description'])
        proj['title'] = self.bot.text.bold(proj['title'])
        fmt = "{title}: {desc}"
        msg.reply(fmt.format(**proj))

    @BotPlugin.command(urlab_url + r'/(events/\d+)')
    def urlab_event(self, msg):
        event_status = {
            'r': self.bot.text.bold,    # Ready
            'i': self.bot.text.yellow,  # Incubation
        }

        evt = yield from public_api(msg.args[0])
        if evt.get('start', None):
            evt['when'] = self.bot.text.yellow(self.bot.naturaltime(evt['start']))
        else:
            evt['when'] = "pas de date"
        color = event_status.get(evt['status'], self.bot.text.grey)
        evt['title'] = color(evt['title'])
        evt['place'] = self.bot.text.purple(evt['place'])
        fmt = "{title} ({when} :: {place})"
        msg.reply(fmt.format(**evt))
<commit_msg>[add] Gist and time of creation<commit_after>from .helpers import public_api
from .twitter import TwitterBasePlugin
from ircbot.plugin import BotPlugin

class UrlShow(TwitterBasePlugin):
    """
    Post a preview for some well-known and frequent URLs
    """

    github_repo = r'.*https?://github\.com/([\w\d_\.-]+)/([\w\d_\.-]+)'
    urlab_url = r'.*https?://urlab\.be'

    @BotPlugin.command(r'.*https?://twitter.com/[^/]+/status/(\d+)')
    def twitter_status(self, msg):
        url = 'statuses/show/{}.json'.format(msg.args[0])
        tweet = yield from self.twitter_request('GET', url)
        msg.reply(self.format_tweet(tweet))

    @BotPlugin.command(github_repo + r'/?(\s|$)')
    def github(self, msg):
        url = "https://api.github.com/repos/{}/{}".format(*msg.args)
        repo = yield from public_api(url)
        repo['name'] = self.bot.text.bold(repo['name'])
        repo['language'] = self.bot.text.purple('[', repo['language'], ']')
        repo['stars'] = self.bot.text.yellow('(', repo['stargazers_count'], '*)')
        fmt = "{name} {language} {stars}: «{description}»"
        msg.reply(fmt.format(**repo))

    @BotPlugin.command(github_repo + r'/(issues|pull)/(\d+)')
    def github_issue(self, msg):
        user, repo, kind, id = msg.args
        args = user, repo, id
        url = "https://api.github.com/repos/{}/{}/issues/{}".format(*args)
        issue = yield from public_api(url)
        issue['when'] = self.bot.text.grey(self.bot.naturaltime(issue['created_at']))
        issue['author'] = self.bot.text.bold('@' + issue['user']['login'])
        issue['labels'] = ' '.join(self.bot.text.grey('(%s)') % x['name'] for x in issue['labels'])
        if issue['state'] == 'open':
            issue['number'] = self.bot.text.green('[#', issue['number'], ' (open)]')
        elif issue['state'] == 'closed':
            issue['number'] = self.bot.text.red('[#', issue['number'], ' (closed)]')
        fmt = "{author} {when} {number}: «{title}» {labels}"
        msg.reply(fmt.format(**issue))

    @BotPlugin.command(github_repo + r'/commit/([0-9a-fA-F]{,40})')
    def github_commit(self, msg):
        url = "https://api.github.com/repos/{}/{}/commits/{}".format(*msg.args)
        commit = yield from public_api(url)
        additions = self.bot.text.green("%d+" % commit['stats']['additions'])
        deletions = self.bot.text.red("%d-" % commit['stats']['deletions'])
        files_changed = self.bot.text.yellow("%d files" % len(commit['files']))
        f = {
            'author': self.bot.text.bold('@' + commit['author']['login']),
            'title': commit['commit']['message'],
            'stats': " ".join([additions, deletions, files_changed]),
            'when': self.bot.text.grey(self.bot.naturaltime(commit['commit']['author']['date'])),
        }
        msg.reply("{author} {when} «{title}» ({stats})".format(**f))

    @BotPlugin.command(r'.*https?://gist\.github\.com/[^/]+/([0-9a-z]+)')
    def gist(self, msg):
        url = "https://api.github.com/gists/{}".format(msg.args[0])
        gist = yield from public_api(url)
        filelist = ', '.join(i['filename'] for i in gist['files'].values())
        f = {
            'author': self.bot.text.bold('@' + gist['owner']['login']),
            'when': self.bot.text.grey(self.bot.naturaltime(gist['updated_at'])),
            'title': gist['description'],
            'files': self.bot.text.yellow(filelist),
        }
        msg.reply("{author} {when} «{title}» ({files})".format(**f))

    @BotPlugin.command(r'.*https?://www\.reddit\.com/r/([\w\d_\.-]+)/comments/([\w\d_\.-]+)')
    def reddit(self, msg):
        url = "https://api.bot.text.reddit.com/r/{}/comments/{}".format(*msg.args[:2])
        data = yield from public_api(url)
        post = data[0]['data']['children'][0]['data']
        post['author'] = self.bot.text.bold('@' + post['author'])
        post['upvote_ratio'] = self.bot.text.yellow('(', post['upvote_ratio'], '+)')
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{author} {upvote_ratio}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(r'.*https?://news\.ycombinator\.com/item\?id=(\d+)')
    def hackernews(self, msg):
        url = "https://hacker-news.firebaseio.com/v0/item/"
        post = yield from public_api(url + "{}.json".format(msg.args[0]))
        post['by'] = self.bot.text.bold('@', post['by'])
        post['url'] = self.bot.text.blue(post['url'])
        fmt = "{by}: «{title}» {url}"
        msg.reply(fmt.format(**post))

    @BotPlugin.command(urlab_url + r'/(projects/\d+)')
    def urlab_project(self, msg):
        project_status = {
            'p': self.bot.text.yellow,  # Proposition
            'i': self.bot.text.blue,    # In progress
            'f': self.bot.text.green,   # Finished
        }

        proj = yield from public_api(msg.args[0])
        color = project_status.get(proj['status'], self.bot.text.grey)
        proj['desc'] = color(proj['short_description'])
        proj['title'] = self.bot.text.bold(proj['title'])
        fmt = "{title}: {desc}"
        msg.reply(fmt.format(**proj))

    @BotPlugin.command(urlab_url + r'/(events/\d+)')
    def urlab_event(self, msg):
        event_status = {
            'r': self.bot.text.bold,    # Ready
            'i': self.bot.text.yellow,  # Incubation
        }

        evt = yield from public_api(msg.args[0])
        if evt.get('start', None):
            evt['when'] = self.bot.text.yellow(self.bot.naturaltime(evt['start']))
        else:
            evt['when'] = "pas de date"
        color = event_status.get(evt['status'], self.bot.text.grey)
        evt['title'] = color(evt['title'])
        evt['place'] = self.bot.text.purple(evt['place'])
        fmt = "{title} ({when} :: {place})"
        msg.reply(fmt.format(**evt))
<|endoftext|>


<commit_before>import re
import StringIO
import email
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header

class Struct(object):
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return value.encode(default_charset, 'replace').decode(default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                headers[index] = text.decode(charset or default_charset,
                                             'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = text.decode(default_charset, 'replace')

        return u"".join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}

    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.search(r'=\?(\w+)\?(Q|B)\?(.+)\?=', value)
        if match:
            encoding, type_, code = match.groups()
            if type_ == 'Q':
                value = quopri.decodestring(code)
            elif type_ == 'B':
                value = base64.decodestring(code)
                value = unicode(value, encoding)
                value_results.append(value)
                if value_results:
                    v = ''.join(value_results)
                return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None:
        dispositions = content_disposition.strip().split(";")

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': StringIO.StringIO(file_data)
            }

            for param in dispositions[1:]:
                name, value = decode_param(param)

                if 'file' in name:
                    attachment['filename'] = value

                if 'create-date' in name:
                    attachment['create-date'] = value

            return attachment

    return None

def parse_email(raw_email):
    email_message = email.message_from_string(raw_email)
    maintype = email_message.get_content_maintype()
    parsed_email = {}

    parsed_email['raw_email'] = raw_email

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype == 'multipart':
        for part in email_message.walk():
            content = part.get_payload(decode=True)
            content_type = part.get_content_type()
            content_disposition = part.get('Content-Disposition', None)

            is_inline = content_disposition is None \
                or content_disposition == "inline"
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

                elif maintype == 'text':
                    payload = email_message.get_payload(decode=True)
                    body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.iteritems():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        timetuple = email.utils.parsedate(parsed_email['date'])
        parsed_date = datetime.fromtimestamp(time.mktime(timetuple)) \
            if timetuple else None
        parsed_email['parsed_date'] = parsed_date

    return Struct(**parsed_email)
<commit_msg>Bug fix. Set body when maintype is text<commit_after>import re
import StringIO
import email
import base64
import quopri
import time
from datetime import datetime
from email.header import decode_header

class Struct(object):
    def __init__(self, **entries):
        self.__dict__.update(entries)

    def keys(self):
        return self.__dict__.keys()

    def __repr__(self):
        return str(self.__dict__)

def decode_mail_header(value, default_charset='us-ascii'):
    """
    Decode a header value into a unicode string.
    """
    try:
        headers = decode_header(value)
    except email.errors.HeaderParseError:
        return value.encode(default_charset, 'replace').decode(default_charset)
    else:
        for index, (text, charset) in enumerate(headers):
            try:
                headers[index] = text.decode(charset or default_charset,
                                             'replace')
            except LookupError:
                # if the charset is unknown, force default
                headers[index] = text.decode(default_charset, 'replace')

        return u"".join(headers)

def get_mail_addresses(message, header_name):
    """
    Retrieve all email addresses from one message header.
    """
    headers = [h for h in message.get_all(header_name, [])]
    addresses = email.utils.getaddresses(headers)

    for index, (address_name, address_email) in enumerate(addresses):
        addresses[index] = {'name': decode_mail_header(address_name),
                            'email': address_email}

    return addresses

def decode_param(param):
    name, v = param.split('=', 1)
    values = v.split('\n')
    value_results = []
    for value in values:
        match = re.search(r'=\?(\w+)\?(Q|B)\?(.+)\?=', value)
        if match:
            encoding, type_, code = match.groups()
            if type_ == 'Q':
                value = quopri.decodestring(code)
            elif type_ == 'B':
                value = base64.decodestring(code)
                value = unicode(value, encoding)
                value_results.append(value)
                if value_results:
                    v = ''.join(value_results)
                return name, v

def parse_attachment(message_part):
    # Check again if this is a valid attachment
    content_disposition = message_part.get("Content-Disposition", None)
    if content_disposition is not None:
        dispositions = content_disposition.strip().split(";")

        if dispositions[0].lower() in ["attachment", "inline"]:
            file_data = message_part.get_payload(decode=True)

            attachment = {
                'content-type': message_part.get_content_type(),
                'size': len(file_data),
                'content': StringIO.StringIO(file_data)
            }

            for param in dispositions[1:]:
                name, value = decode_param(param)

                if 'file' in name:
                    attachment['filename'] = value

                if 'create-date' in name:
                    attachment['create-date'] = value

            return attachment

    return None

def parse_email(raw_email):
    email_message = email.message_from_string(raw_email)
    maintype = email_message.get_content_maintype()
    parsed_email = {}

    parsed_email['raw_email'] = raw_email

    body = {
        "plain": [],
        "html": []
    }
    attachments = []

    if maintype == 'multipart':
        for part in email_message.walk():
            content = part.get_payload(decode=True)
            content_type = part.get_content_type()
            content_disposition = part.get('Content-Disposition', None)

            is_inline = content_disposition is None \
                or content_disposition == "inline"
            if content_type == "text/plain" and is_inline:
                body['plain'].append(content)
            elif content_type == "text/html" and is_inline:
                body['html'].append(content)
            elif content_disposition:
                attachment = parse_attachment(part)
                if attachment:
                    attachments.append(attachment)

    elif maintype == 'text':
        payload = email_message.get_payload(decode=True)
        body['plain'].append(payload)

    parsed_email['attachments'] = attachments

    parsed_email['body'] = body
    email_dict = dict(email_message.items())

    parsed_email['sent_from'] = get_mail_addresses(email_message, 'from')
    parsed_email['sent_to'] = get_mail_addresses(email_message, 'to')

    value_headers_keys = ['subject', 'date', 'message-id']
    key_value_header_keys = ['received-spf',
                             'mime-version',
                             'x-spam-status',
                             'x-spam-score',
                             'content-type']

    parsed_email['headers'] = []
    for key, value in email_dict.iteritems():

        if key.lower() in value_headers_keys:
            valid_key_name = key.lower().replace('-', '_')
            parsed_email[valid_key_name] = decode_mail_header(value)

        if key.lower() in key_value_header_keys:
            parsed_email['headers'].append({'Name': key,
                                            'Value': value})

    if parsed_email.get('date'):
        timetuple = email.utils.parsedate(parsed_email['date'])
        parsed_date = datetime.fromtimestamp(time.mktime(timetuple)) \
            if timetuple else None
        parsed_email['parsed_date'] = parsed_date

    return Struct(**parsed_email)
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import urlparse

from buildbot.changes import base
from buildbot.schedulers.trysched import BadJobfile
from twisted.internet import defer
from twisted.python import log
from twisted.web import client

from master.try_job_base import TryJobBase

class _RietveldPoller(base.PollingChangeSource):
  """Polls Rietveld for any pending patch sets to build.

  Periodically polls Rietveld to see if any patch sets have been marked by
  users to be tried.  If so, send them to the trybots.
  """

  def __init__(self, get_pending_endpoint, interval):
    """
    Args:
      get_pending_endpoint: Rietveld URL string used to retrieve jobs to try.
      interval: Interval used to poll Rietveld, in seconds.
    """
    # Set interval time in base class.
    self.pollInterval = interval

    # A string URL for the Rietveld endpoint to query for pending try jobs.
    self._get_pending_endpoint = get_pending_endpoint

    # Cursor used to keep track of next patchset(s) to try.  If the cursor
    # is None, then try from the beginning.
    self._cursor = None

    # Try job parent of this poller.
    self._try_job_rietveld = None

  # base.PollingChangeSource overrides:
  def poll(self):
    """Polls Rietveld for any pending try jobs and submit them.

    Returns:
      A deferred objects to be called once the operation completes.
    """
    log.msg('RietveldPoller.poll')
    d = defer.succeed(None)
    d.addCallback(self._OpenUrl)
    d.addCallback(self._ParseJson)
    d.addErrback(log.err, 'error in RietveldPoller')  # eat errors
    return d

  def setServiceParent(self, parent):
    base.PollingChangeSource.setServiceParent(self, parent)
    self._try_job_rietveld = parent

  def _OpenUrl(self, _):
    """Downloads pending patch sets from Rietveld.

    Returns: A string containing the pending patchsets from Rietveld
        encoded as JSON.
    """
    endpoint = self._get_pending_endpoint
    if self._cursor:
      sep = '&' if '?' in endpoint else '?'
      endpoint = endpoint + '%scursor=%s' % (sep, self._cursor)

    log.msg('RietveldPoller._OpenUrl: %s' % endpoint)
    return client.getPage(endpoint, agent='buildbot')

  def _ParseJson(self, json_string):
    """Parses the JSON pending patch set information.

    Args:
      json_string: A string containing the serialized JSON jobs.

    Returns: A list of pending try jobs.  This is the list of all jobs returned
        by Rietveld, not simply the ones we tried this time.
    """
    data = json.loads(json_string)
    self._cursor = str(data['cursor'])
    self._try_job_rietveld.SubmitJobs(data['jobs'])

class TryJobRietveld(TryJobBase):
  """A try job source that gets jobs from pending Rietveld patch sets."""

  def __init__(self, name, pools, properties=None, last_good_urls=None,
               code_review_sites=None, project=None):
    """Creates a try job source for Rietveld patch sets.

    Args:
      name: Name of this scheduler.
      pools: No idea.
      properties: Extra build properties specific to this scheduler.
      last_good_urls: Dictionary of project to last known good build URL.
      code_review_sites: Dictionary of project to code review site.  This
          class care only about the 'chrome' project.
      project: The name of the project whose review site URL to extract.
          If the project is not found in the dictionary, an exception is raised.
    """
    TryJobBase.__init__(self, name, pools, properties,
                        last_good_urls, code_review_sites)
    endpoint = self._GetRietveldEndPointForProject(code_review_sites, project)
    self._poller = _RietveldPoller(endpoint, interval=10)
    self._project = project
    log.msg('TryJobRietveld created, get_pending_endpoint=%s '
            'project=%s' % (endpoint, project))

  @staticmethod
  def _GetRietveldEndPointForProject(code_review_sites, project):
    """Determines the correct endpoint for the chrome review site URL.

    Args:
      code_review_sites: Dictionary of project name to review site URL.
      project: The name of the project whose review site URL to extract.
          If the project is not found in the dictionary, an exception is raised.

    Returns: A string with the endpoint extracted from the chrome
        review site URL, which is the URL to poll for new patch
        sets to try.
    """
    if project not in code_review_sites:
      raise Exception('No review site for "%s"' % project)

    return urlparse.urljoin(code_review_sites[project],
                            'get_pending_try_patchsets?limit=100')

  def SubmitJobs(self, jobs):
    """Submit pending try jobs to slaves for building.

    Args:
      jobs: a list of jobs.  Each job is a dictionary of properties describing
          what to build.
    """
    log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))

    exceptions = []
    for job in jobs:
      # Add the 'project' property to each job based on the project of this
      # instance.
      job['project'] = self._project
      # Add a 'bot' property which is a map of builder name to ?.
      job['bot'] = {job['builder']: None}
      # Set some default values for properties not needed for this job.
      job['patch'] = ''
      job['patchlevel'] = ''
      job['branch'] = None
      job['repository'] = None
      job['try_job_key'] = job['key']

      try:
        self.SubmitJob(job, None)
      except BadJobfile, ex:
        exceptions.append(ex)

    for ex in exceptions:
      log.msg(ex)

  # TryJobBase overrides:
  def setServiceParent(self, parent):
    TryJobBase.setServiceParent(self, parent)
    self._poller.setServiceParent(self)
    self._poller.master = self.master
<commit_msg>Temporary white list of @chromium.org users.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import urlparse

from buildbot.changes import base
from buildbot.schedulers.trysched import BadJobfile
from twisted.internet import defer
from twisted.python import log
from twisted.web import client

from master.try_job_base import TryJobBase

class _RietveldPoller(base.PollingChangeSource):
  """Polls Rietveld for any pending patch sets to build.

  Periodically polls Rietveld to see if any patch sets have been marked by
  users to be tried.  If so, send them to the trybots.
  """

  def __init__(self, get_pending_endpoint, interval):
    """
    Args:
      get_pending_endpoint: Rietveld URL string used to retrieve jobs to try.
      interval: Interval used to poll Rietveld, in seconds.
    """
    # Set interval time in base class.
    self.pollInterval = interval

    # A string URL for the Rietveld endpoint to query for pending try jobs.
    self._get_pending_endpoint = get_pending_endpoint

    # Cursor used to keep track of next patchset(s) to try.  If the cursor
    # is None, then try from the beginning.
    self._cursor = None

    # Try job parent of this poller.
    self._try_job_rietveld = None

  # base.PollingChangeSource overrides:
  def poll(self):
    """Polls Rietveld for any pending try jobs and submit them.

    Returns:
      A deferred objects to be called once the operation completes.
    """
    log.msg('RietveldPoller.poll')
    d = defer.succeed(None)
    d.addCallback(self._OpenUrl)
    d.addCallback(self._ParseJson)
    d.addErrback(log.err, 'error in RietveldPoller')  # eat errors
    return d

  def setServiceParent(self, parent):
    base.PollingChangeSource.setServiceParent(self, parent)
    self._try_job_rietveld = parent

  def _OpenUrl(self, _):
    """Downloads pending patch sets from Rietveld.

    Returns: A string containing the pending patchsets from Rietveld
        encoded as JSON.
    """
    endpoint = self._get_pending_endpoint
    if self._cursor:
      sep = '&' if '?' in endpoint else '?'
      endpoint = endpoint + '%scursor=%s' % (sep, self._cursor)

    log.msg('RietveldPoller._OpenUrl: %s' % endpoint)
    return client.getPage(endpoint, agent='buildbot')

  def _ParseJson(self, json_string):
    """Parses the JSON pending patch set information.

    Args:
      json_string: A string containing the serialized JSON jobs.

    Returns: A list of pending try jobs.  This is the list of all jobs returned
        by Rietveld, not simply the ones we tried this time.
    """
    data = json.loads(json_string)
    self._cursor = str(data['cursor'])
    self._try_job_rietveld.SubmitJobs(data['jobs'])

class TryJobRietveld(TryJobBase):
  """A try job source that gets jobs from pending Rietveld patch sets."""

  def __init__(self, name, pools, properties=None, last_good_urls=None,
               code_review_sites=None, project=None):
    """Creates a try job source for Rietveld patch sets.

    Args:
      name: Name of this scheduler.
      pools: No idea.
      properties: Extra build properties specific to this scheduler.
      last_good_urls: Dictionary of project to last known good build URL.
      code_review_sites: Dictionary of project to code review site.  This
          class care only about the 'chrome' project.
      project: The name of the project whose review site URL to extract.
          If the project is not found in the dictionary, an exception is raised.
    """
    TryJobBase.__init__(self, name, pools, properties,
                        last_good_urls, code_review_sites)
    endpoint = self._GetRietveldEndPointForProject(code_review_sites, project)
    self._poller = _RietveldPoller(endpoint, interval=10)
    self._project = project
    log.msg('TryJobRietveld created, get_pending_endpoint=%s '
            'project=%s' % (endpoint, project))

  @staticmethod
  def _GetRietveldEndPointForProject(code_review_sites, project):
    """Determines the correct endpoint for the chrome review site URL.

    Args:
      code_review_sites: Dictionary of project name to review site URL.
      project: The name of the project whose review site URL to extract.
          If the project is not found in the dictionary, an exception is raised.

    Returns: A string with the endpoint extracted from the chrome
        review site URL, which is the URL to poll for new patch
        sets to try.
    """
    if project not in code_review_sites:
      raise Exception('No review site for "%s"' % project)

    return urlparse.urljoin(code_review_sites[project],
                            'get_pending_try_patchsets?limit=100')

  def SubmitJobs(self, jobs):
    """Submit pending try jobs to slaves for building.

    Args:
      jobs: a list of jobs.  Each job is a dictionary of properties describing
          what to build.
    """
    log.msg('TryJobRietveld.SubmitJobs: %s' % json.dumps(jobs, indent=2))

    exceptions = []
    for job in jobs:
      if not job['user'].endswith('@chromium.org'):
        continue
      # Add the 'project' property to each job based on the project of this
      # instance.
      job['project'] = self._project
      # Add a 'bot' property which is a map of builder name to ?.
      job['bot'] = {job['builder']: None}
      # Set some default values for properties not needed for this job.
      job['patch'] = ''
      job['patchlevel'] = ''
      job['branch'] = None
      job['repository'] = None
      job['try_job_key'] = job['key']

      try:
        self.SubmitJob(job, None)
      except BadJobfile, ex:
        exceptions.append(ex)

    for ex in exceptions:
      log.msg(ex)

  # TryJobBase overrides:
  def setServiceParent(self, parent):
    TryJobBase.setServiceParent(self, parent)
    self._poller.setServiceParent(self)
    self._poller.master = self.master
<|endoftext|>