<commit_before>#
# Run-Related Pages
#

import pscheduler
import time

from pschedulerapiserver import application

from flask import request

from .dbcursor import dbcursor_query
from .json import *
from .limitproc import *
from .log import log
from .response import *
from .tasks import task_exists

# Proposed times for a task
@application.route("/tasks/<task>/runtimes", methods=['GET'])
def task_uuid_runtimes(task):
    try:
        range_start = arg_datetime('start')
        range_end   = arg_datetime('end')
    except ValueError:
        return bad_request('Invalid start or end time')
    if not task_exists(task):
        return not_found()

    # TODO: At some point, we will likely have to consider making the
    # proposals fall within the limits, which is going to be complex.

    return json_query_simple("""
        SELECT row_to_json(apt.*)
        FROM  api_proposed_times(%s, %s, %s) apt
        """, [task, range_start, range_end], empty_ok=True)

# Evaluate the limits for a run.
def __evaluate_limits(
    task,       # Task UUID
    start_time  # When the task should start
    ):

    log.debug("Applying limits")
    # Let this throw what it may; callers have to catch it.
    cursor = dbcursor_query(
        "SELECT json, duration, hints FROM task where uuid = %s", [task])
    if cursor.rowcount == 0:
        # TODO: This or bad_request when the task isn't there?
        return not_found()
    task_spec, duration, hints = cursor.fetchone()
    log.debug("Task is %s, duration is %s" % (task_spec, duration))

    limit_input = {
        'type': task_spec['test']['type'],
        'spec': task_spec['test']['spec'],
        'schedule': {
            'start': pscheduler.datetime_as_iso8601(start_time),
            'duration': pscheduler.timedelta_as_iso8601(duration)
            }
        }

    log.debug("Checking limits against %s" % str(limit_input))

    processor, whynot = limitprocessor()
    if processor is None:
        log.debug("Limit processor is not initialized. %s", whynot)
        return no_can_do("Limit processor is not initialized: %s" % whynot)

    # Don't pass hints since that would have been covered when the
    # task was submitted and only the scheduler will be submitting
    # runs.
    passed, diags = processor.process(limit_input, hints)

    log.debug("Passed: %s.  Diags: %s" % (passed, diags))

    # This prevents the run from being put in a non-starter state
    if passed:
        diags = None

    return passed, diags

# Established runs for a task
@application.route("/tasks/<task>/runs", methods=['GET', 'POST'])
def tasks_uuid_runs(task):

    if request.method == 'GET':

        query = "SELECT '" + base_url() + """/' || run.uuid
             FROM
                 run
                 JOIN task ON task.id = run.task
             WHERE
                task.uuid = %s"""
        args = [task]

        try:

            start_time = arg_datetime('start')
            if start_time is not None:
                query += " AND lower(times) >= %s"
                args.append(start_time)

            end_time = arg_datetime('end')
            if end_time is not None:
                query += " AND upper(times) <= %s"
                args.append(end_time)

            if arg_boolean('upcoming'):
                query += " AND lower(times) > now()"
                query += " AND state IN (run_state_pending(), run_state_on_deck(), run_state_running(), run_state_nonstart())"

            query += " ORDER BY times"

            limit = arg_cardinal('limit')
            if limit is not None:
                query += " LIMIT " + str(limit)

            # TODO: This should be exapandable

        except ValueError as ex:

            return bad_request(str(ex))

        return json_query_simple(query, args, empty_ok=True)

    elif request.method == 'POST':

        log.debug("Run POST: %s --> %s", request.url, request.data)

        try:
            data = pscheduler.json_load(request.data)
            start_time = pscheduler.iso8601_as_datetime(data['start-time'])
        except KeyError:
            return bad_request("Missing start time")
        except ValueError:
            return bad_request("Invalid JSON:" + request.data)

        try:
            passed, diags = __evaluate_limits(task, start_time)
        except Exception as ex:
            log.exception()
            return error(str(ex))

        try:
            log.debug("Posting run for task %s starting %s"
                      % (task, start_time))
            cursor = dbcursor_query("SELECT api_run_post(%s, %s, NULL, %s)",
                               [task, start_time, diags], onerow=True)
            uuid = cursor.fetchone()[0]
        except Exception as ex:
            log.exception()
            return error(str(ex))

        url = base_url() + '/' + uuid
        log.debug("New run posted to %s", url)
        return ok_json(url)

    else:

        return not_allowed()

def __runs_first_run(
    task   # UUID of task
    ):
    """
    Find the UUID of the first run of the specified task, returning
    None if none exists.
    """

    # Let this throw what it may; callers are responsible for handling
    # exceptions.
    cursor = dbcursor_query("""
                SELECT run.uuid
                FROM
                  task
                  JOIN run ON run.task = task.id
                WHERE
                  task.uuid = %s
                ORDER BY run.times
                LIMIT 1
                """, [task])

    if cursor.rowcount == 0:
        return None
    else:
        return cursor.fetchone()[0]

@application.route("/tasks/<task>/runs/<run>", methods=['GET', 'PUT', 'DELETE'])
def tasks_uuid_runs_run(task, run):

    if task is None:
        return bad_request("Missing or invalid task")

    if run is None:
        return bad_request("Missing or invalid run")

    if request.method == 'GET':

        # Wait for there to be a local result
        wait_local = arg_boolean('wait-local')

        # Wait for there to be a merged result
        wait_merged = arg_boolean('wait-merged')

        if wait_local and wait_merged:
            return error("Cannot wait on local and merged results")

        # If asked for 'first', dig up the first run and use its UUID.

        if run == 'first':
            # 60 tries at 0.25s intervals == 15 sec.
            tries = 60 if (wait_local or wait_merged) else 1
            while tries > 0:
                try:
                    run = __runs_first_run(task)
                except Exception as ex:
                    log.exception()
                    return error(str(ex))
                if run is not None:
                    break
                time.sleep(0.25)
                tries -= 1

            if run is None:
                return not_found()

        # 40 tries at 0.25s intervals == 10 sec.
        tries = 40 if (wait_local or wait_merged) else 1

        while tries:

            try:
                cursor = dbcursor_query(
                    """
                    SELECT
                        lower(run.times),
                        upper(run.times),
                        upper(run.times) - lower(run.times),
                        task.participant,
                        task.nparticipants,
                        task.participants,
                        run.part_data,
                        run.part_data_full,
                        run.result,
                        run.result_full,
                        run.result_merged,
                        run_state.enum,
                        run_state.display,
                        run.errors,
                        run.clock_survey
                    FROM
                        run
                        JOIN task ON task.id = run.task
                        JOIN run_state ON run_state.id = run.state
                    WHERE
                        task.uuid = %s
                        AND run.uuid = %s
                    """, [task, run])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            if cursor.rowcount == 0:
                return not_found()

            row = cursor.fetchone()

            if not (wait_local or wait_merged):
                break
            else:
                if (wait_local and row[7] is None) \
                        or (wait_merged and row[9] is None):
                    time.sleep(0.25)
                    tries -= 1
                else:
                    break

        # Return a result Whether or not we timed out and let the
        # client sort it out.

        result = {}

        # This strips any query parameters and replaces the last item
        # with the run, which might be needed if the 'first' option
        # was used.

        href_path_parts = urlparse.urlparse(request.url).path.split('/')
        href_path_parts[-1] = run
        href_path = '/'.join(href_path_parts)
        href = urlparse.urljoin( request.url, href_path )

        result['href'] = href
        result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
        result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
        result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
        result['participant'] = row[3]
        result['participants'] = [
            participant if participant is not None
            else pscheduler.api_this_host()
            for participant in row[5]
            ]
        result['participant-data'] = row[6]
        result['participant-data-full'] = row[7]
        result['result'] = row[8]
        result['result-full'] = row[9]
        result['result-merged'] = row[10]
        result['state'] = row[11]
        result['state-display'] = row[12]
        result['errors'] = row[13]
        if row[14] is not None:
            result['clock-survey'] = row[14]
        result['task-href'] = root_url('tasks/' + task)
        result['result-href'] = href + '/result'

        return json_response(result)

    elif request.method == 'PUT':

        log.debug("Run PUT %s", request.url)

        # Get the JSON from the body
        try:
            run_data = pscheduler.json_load(request.data)
        except ValueError:
            log.exception()
            log.debug("Run data was %s", request.data)
            return error("Invalid or missing run data")

        # If the run doesn't exist, take the whole thing as if it were
        # a POST.

        try:
            cursor = dbcursor_query(
                "SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)",
                [run], onerow=True)
        except Exception as ex:
            log.exception()
            return error(str(ex))

        if not cursor.fetchone()[0]:

            log.debug("Record does not exist; full PUT.")

            try:
                start_time = \
                    pscheduler.iso8601_as_datetime(run_data['start-time'])
            except KeyError:
                return bad_request("Missing start time")
            except ValueError:
                return bad_request("Invalid start time")

            passed, diags = __evaluate_limits(task, start_time)

            try:
                cursor = dbcursor_query("SELECT api_run_post(%s, %s, %s)",
                               [task, start_time, run], onerow=True)
                log.debug("Full put of %s, got back %s", run, cursor.fetchone()[0])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            return ok()

        # For anything else, only one thing can be udated at a time,
        # and even that is a select subset.

        log.debug("Record exists; partial PUT.")

        if 'part-data-full' in run_data:

            log.debug("Updating part-data-full from %s", run_data)

            try:
                part_data_full = \
                    pscheduler.json_dump(run_data['part-data-full'])
            except KeyError:
                return bad_request("Missing part-data-full")
            except ValueError:
                return bad_request("Invalid part-data-full")

            log.debug("Full data is: %s", part_data_full)

            try:
                cursor = dbcursor_query("""
                              UPDATE
                                  run
                              SET
                                  part_data_full = %s
                              WHERE
                                  uuid = %s
                                  AND EXISTS (SELECT * FROM task WHERE UUID = %s)
                              """,
                           [ part_data_full, run, task])
            except Exception as ex:
                log.exception()
                return error(str(ex))
            if cursor.rowcount != 1:
                return not_found()

            log.debug("Full data updated")

            return ok()

        elif 'result-full' in run_data:

            log.debug("Updating result-full from %s", run_data)

            try:
                result_full = \
                    pscheduler.json_dump(run_data['result-full'])
            except KeyError:
                return bad_request("Missing result-full")
            except ValueError:
                return bad_request("Invalid result-full")

            log.debug("Updating result-full: JSON %s", result_full)
            log.debug("Updating result-full: Run  %s", run)
            log.debug("Updating result-full: Task %s", task)

            try:
                cursor = dbcursor_query("""
                              UPDATE
                                  run
                              SET
                                  result_full = %s
                              WHERE
                                  uuid = %s
                                  AND EXISTS (SELECT * FROM task WHERE UUID = %s)
                              """,
                               [ result_full, run, task ])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            if cursor.rowcount != 1:
                return not_found()

            return ok()

    elif request.method == 'DELETE':

        # TODO: If this is the lead, the run's counterparts on the
        # other participating nodes need to be removed as well.

        try:
            cursor = dbcursor_query("""
            DELETE FROM run
            WHERE
                task in (SELECT id FROM task WHERE uuid = %s)
                AND uuid = %s
            """, [task, run])
        except Exception as ex:
            log.exception()
            return error(str(ex))

        return ok() if cursor.rowcount == 1 else not_found()

    else:

        return not_allowed()

#
# Merged results, optionally formatted.
#

@application.route("/tasks/<task>/runs/<run>/result", methods=['GET'])
def tasks_uuid_runs_run_result(task, run):

    if task is None:
        return bad_request("Missing or invalid task")

    if run is None:
        return bad_request("Missing or invalid run")

    wait = arg_boolean('wait')

    format = request.args.get('format')

    if format is None:
        format = 'application/json'

    if format not in [ 'application/json', 'text/html', 'text/plain' ]:
        return bad_request("Unsupported format " + format)

    # If asked for 'first', dig up the first run and use its UUID.
    # This is more for debug convenience than anything else.
    if run == 'first':
        try:
            run = __runs_first_run(task)
        except Exception as ex:
            log.exception()
            return error(str(ex))
        if run is None:
            return not_found()

    #
    # Camp on the run for a result
    #

    # 40 tries at 0.25s intervals == 10 sec.
    tries = 40 if wait else 1

    while tries:

        try:
            cursor = dbcursor_query("""
                SELECT
                    test.name,
                    run.result_merged
                FROM
                    run
                    JOIN task ON task.id = run.task
                    JOIN test ON test.id = task.test
                WHERE
                    task.uuid = %s
                    AND run.uuid = %s
                """, [task, run])
        except Exception as ex:
            log.exception()
            return error(str(ex))

        if cursor.rowcount == 0:
            return not_found()

        # TODO: Make sure we got back one row with two columns.
        row = cursor.fetchone()

        if not wait and row[1] is None:
            time.sleep(0.25)
            tries -= 1
        else:
            break

    if tries == 0:
        return not_found()

    test_type, merged_result = row

    # JSON requires no formatting.
    if format == 'application/json':
        return ok_json(merged_result)

    if not merged_result['succeeded']:
        if format == 'text/plain':
            return ok("Test failed.", mimetype=format)
        elif format == 'text/html':
            return ok("
Test failed.
", mimetype=format)
        return error("Unsupported format " + format)

    returncode, stdout, stderr = pscheduler.run_program(
        [ "pscheduler", "internal", "invoke", "test", test_type,
          "result-format", format ],
        stdin = pscheduler.json_dump(merged_result)
        )

    if returncode != 0:
        return error("Failed to format result: " + stderr)

    return ok(stdout.rstrip(), mimetype=format)
<commit_msg>Don't subsititue the local host for null first participant if not the first participant.<commit_after>#
# Run-Related Pages
#

import pscheduler
import time

from pschedulerapiserver import application

from flask import request

from .dbcursor import dbcursor_query
from .json import *
from .limitproc import *
from .log import log
from .response import *
from .tasks import task_exists

# Proposed times for a task
@application.route("/tasks/<task>/runtimes", methods=['GET'])
def task_uuid_runtimes(task):
    try:
        range_start = arg_datetime('start')
        range_end   = arg_datetime('end')
    except ValueError:
        return bad_request('Invalid start or end time')
    if not task_exists(task):
        return not_found()

    # TODO: At some point, we will likely have to consider making the
    # proposals fall within the limits, which is going to be complex.

    return json_query_simple("""
        SELECT row_to_json(apt.*)
        FROM  api_proposed_times(%s, %s, %s) apt
        """, [task, range_start, range_end], empty_ok=True)

# Evaluate the limits for a run.
def __evaluate_limits(
    task,       # Task UUID
    start_time  # When the task should start
    ):

    log.debug("Applying limits")
    # Let this throw what it may; callers have to catch it.
    cursor = dbcursor_query(
        "SELECT json, duration, hints FROM task where uuid = %s", [task])
    if cursor.rowcount == 0:
        # TODO: This or bad_request when the task isn't there?
        return not_found()
    task_spec, duration, hints = cursor.fetchone()
    log.debug("Task is %s, duration is %s" % (task_spec, duration))

    limit_input = {
        'type': task_spec['test']['type'],
        'spec': task_spec['test']['spec'],
        'schedule': {
            'start': pscheduler.datetime_as_iso8601(start_time),
            'duration': pscheduler.timedelta_as_iso8601(duration)
            }
        }

    log.debug("Checking limits against %s" % str(limit_input))

    processor, whynot = limitprocessor()
    if processor is None:
        log.debug("Limit processor is not initialized. %s", whynot)
        return no_can_do("Limit processor is not initialized: %s" % whynot)

    # Don't pass hints since that would have been covered when the
    # task was submitted and only the scheduler will be submitting
    # runs.
    passed, diags = processor.process(limit_input, hints)

    log.debug("Passed: %s.  Diags: %s" % (passed, diags))

    # This prevents the run from being put in a non-starter state
    if passed:
        diags = None

    return passed, diags

# Established runs for a task
@application.route("/tasks/<task>/runs", methods=['GET', 'POST'])
def tasks_uuid_runs(task):

    if request.method == 'GET':

        query = "SELECT '" + base_url() + """/' || run.uuid
             FROM
                 run
                 JOIN task ON task.id = run.task
             WHERE
                task.uuid = %s"""
        args = [task]

        try:

            start_time = arg_datetime('start')
            if start_time is not None:
                query += " AND lower(times) >= %s"
                args.append(start_time)

            end_time = arg_datetime('end')
            if end_time is not None:
                query += " AND upper(times) <= %s"
                args.append(end_time)

            if arg_boolean('upcoming'):
                query += " AND lower(times) > now()"
                query += " AND state IN (run_state_pending(), run_state_on_deck(), run_state_running(), run_state_nonstart())"

            query += " ORDER BY times"

            limit = arg_cardinal('limit')
            if limit is not None:
                query += " LIMIT " + str(limit)

            # TODO: This should be exapandable

        except ValueError as ex:

            return bad_request(str(ex))

        return json_query_simple(query, args, empty_ok=True)

    elif request.method == 'POST':

        log.debug("Run POST: %s --> %s", request.url, request.data)

        try:
            data = pscheduler.json_load(request.data)
            start_time = pscheduler.iso8601_as_datetime(data['start-time'])
        except KeyError:
            return bad_request("Missing start time")
        except ValueError:
            return bad_request("Invalid JSON:" + request.data)

        try:
            passed, diags = __evaluate_limits(task, start_time)
        except Exception as ex:
            log.exception()
            return error(str(ex))

        try:
            log.debug("Posting run for task %s starting %s"
                      % (task, start_time))
            cursor = dbcursor_query("SELECT api_run_post(%s, %s, NULL, %s)",
                               [task, start_time, diags], onerow=True)
            uuid = cursor.fetchone()[0]
        except Exception as ex:
            log.exception()
            return error(str(ex))

        url = base_url() + '/' + uuid
        log.debug("New run posted to %s", url)
        return ok_json(url)

    else:

        return not_allowed()

def __runs_first_run(
    task   # UUID of task
    ):
    """
    Find the UUID of the first run of the specified task, returning
    None if none exists.
    """

    # Let this throw what it may; callers are responsible for handling
    # exceptions.
    cursor = dbcursor_query("""
                SELECT run.uuid
                FROM
                  task
                  JOIN run ON run.task = task.id
                WHERE
                  task.uuid = %s
                ORDER BY run.times
                LIMIT 1
                """, [task])

    if cursor.rowcount == 0:
        return None
    else:
        return cursor.fetchone()[0]

@application.route("/tasks/<task>/runs/<run>", methods=['GET', 'PUT', 'DELETE'])
def tasks_uuid_runs_run(task, run):

    if task is None:
        return bad_request("Missing or invalid task")

    if run is None:
        return bad_request("Missing or invalid run")

    if request.method == 'GET':

        # Wait for there to be a local result
        wait_local = arg_boolean('wait-local')

        # Wait for there to be a merged result
        wait_merged = arg_boolean('wait-merged')

        if wait_local and wait_merged:
            return error("Cannot wait on local and merged results")

        # If asked for 'first', dig up the first run and use its UUID.

        if run == 'first':
            # 60 tries at 0.25s intervals == 15 sec.
            tries = 60 if (wait_local or wait_merged) else 1
            while tries > 0:
                try:
                    run = __runs_first_run(task)
                except Exception as ex:
                    log.exception()
                    return error(str(ex))
                if run is not None:
                    break
                time.sleep(0.25)
                tries -= 1

            if run is None:
                return not_found()

        # 40 tries at 0.25s intervals == 10 sec.
        tries = 40 if (wait_local or wait_merged) else 1

        while tries:

            try:
                cursor = dbcursor_query(
                    """
                    SELECT
                        lower(run.times),
                        upper(run.times),
                        upper(run.times) - lower(run.times),
                        task.participant,
                        task.nparticipants,
                        task.participants,
                        run.part_data,
                        run.part_data_full,
                        run.result,
                        run.result_full,
                        run.result_merged,
                        run_state.enum,
                        run_state.display,
                        run.errors,
                        run.clock_survey
                    FROM
                        run
                        JOIN task ON task.id = run.task
                        JOIN run_state ON run_state.id = run.state
                    WHERE
                        task.uuid = %s
                        AND run.uuid = %s
                    """, [task, run])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            if cursor.rowcount == 0:
                return not_found()

            row = cursor.fetchone()

            if not (wait_local or wait_merged):
                break
            else:
                if (wait_local and row[7] is None) \
                        or (wait_merged and row[9] is None):
                    time.sleep(0.25)
                    tries -= 1
                else:
                    break

        # Return a result Whether or not we timed out and let the
        # client sort it out.

        result = {}

        # This strips any query parameters and replaces the last item
        # with the run, which might be needed if the 'first' option
        # was used.

        href_path_parts = urlparse.urlparse(request.url).path.split('/')
        href_path_parts[-1] = run
        href_path = '/'.join(href_path_parts)
        href = urlparse.urljoin( request.url, href_path )

        result['href'] = href
        result['start-time'] = pscheduler.datetime_as_iso8601(row[0])
        result['end-time'] = pscheduler.datetime_as_iso8601(row[1])
        result['duration'] = pscheduler.timedelta_as_iso8601(row[2])
        participant_num = row[3]
        result['participant'] = participant_num
        result['participants'] = [
            pscheduler.api_this_host()
            if participant is None and participant_num == 0
            else participant
            for participant in row[5]
            ]
        result['participant-data'] = row[6]
        result['participant-data-full'] = row[7]
        result['result'] = row[8]
        result['result-full'] = row[9]
        result['result-merged'] = row[10]
        result['state'] = row[11]
        result['state-display'] = row[12]
        result['errors'] = row[13]
        if row[14] is not None:
            result['clock-survey'] = row[14]
        result['task-href'] = root_url('tasks/' + task)
        result['result-href'] = href + '/result'

        return json_response(result)

    elif request.method == 'PUT':

        log.debug("Run PUT %s", request.url)

        # Get the JSON from the body
        try:
            run_data = pscheduler.json_load(request.data)
        except ValueError:
            log.exception()
            log.debug("Run data was %s", request.data)
            return error("Invalid or missing run data")

        # If the run doesn't exist, take the whole thing as if it were
        # a POST.

        try:
            cursor = dbcursor_query(
                "SELECT EXISTS (SELECT * FROM run WHERE uuid = %s)",
                [run], onerow=True)
        except Exception as ex:
            log.exception()
            return error(str(ex))

        if not cursor.fetchone()[0]:

            log.debug("Record does not exist; full PUT.")

            try:
                start_time = \
                    pscheduler.iso8601_as_datetime(run_data['start-time'])
            except KeyError:
                return bad_request("Missing start time")
            except ValueError:
                return bad_request("Invalid start time")

            passed, diags = __evaluate_limits(task, start_time)

            try:
                cursor = dbcursor_query("SELECT api_run_post(%s, %s, %s)",
                               [task, start_time, run], onerow=True)
                log.debug("Full put of %s, got back %s", run, cursor.fetchone()[0])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            return ok()

        # For anything else, only one thing can be udated at a time,
        # and even that is a select subset.

        log.debug("Record exists; partial PUT.")

        if 'part-data-full' in run_data:

            log.debug("Updating part-data-full from %s", run_data)

            try:
                part_data_full = \
                    pscheduler.json_dump(run_data['part-data-full'])
            except KeyError:
                return bad_request("Missing part-data-full")
            except ValueError:
                return bad_request("Invalid part-data-full")

            log.debug("Full data is: %s", part_data_full)

            try:
                cursor = dbcursor_query("""
                              UPDATE
                                  run
                              SET
                                  part_data_full = %s
                              WHERE
                                  uuid = %s
                                  AND EXISTS (SELECT * FROM task WHERE UUID = %s)
                              """,
                           [ part_data_full, run, task])
            except Exception as ex:
                log.exception()
                return error(str(ex))
            if cursor.rowcount != 1:
                return not_found()

            log.debug("Full data updated")

            return ok()

        elif 'result-full' in run_data:

            log.debug("Updating result-full from %s", run_data)

            try:
                result_full = \
                    pscheduler.json_dump(run_data['result-full'])
            except KeyError:
                return bad_request("Missing result-full")
            except ValueError:
                return bad_request("Invalid result-full")

            log.debug("Updating result-full: JSON %s", result_full)
            log.debug("Updating result-full: Run  %s", run)
            log.debug("Updating result-full: Task %s", task)

            try:
                cursor = dbcursor_query("""
                              UPDATE
                                  run
                              SET
                                  result_full = %s
                              WHERE
                                  uuid = %s
                                  AND EXISTS (SELECT * FROM task WHERE UUID = %s)
                              """,
                               [ result_full, run, task ])
            except Exception as ex:
                log.exception()
                return error(str(ex))

            if cursor.rowcount != 1:
                return not_found()

            return ok()

    elif request.method == 'DELETE':

        # TODO: If this is the lead, the run's counterparts on the
        # other participating nodes need to be removed as well.

        try:
            cursor = dbcursor_query("""
            DELETE FROM run
            WHERE
                task in (SELECT id FROM task WHERE uuid = %s)
                AND uuid = %s
            """, [task, run])
        except Exception as ex:
            log.exception()
            return error(str(ex))

        return ok() if cursor.rowcount == 1 else not_found()

    else:

        return not_allowed()

#
# Merged results, optionally formatted.
#

@application.route("/tasks/<task>/runs/<run>/result", methods=['GET'])
def tasks_uuid_runs_run_result(task, run):

    if task is None:
        return bad_request("Missing or invalid task")

    if run is None:
        return bad_request("Missing or invalid run")

    wait = arg_boolean('wait')

    format = request.args.get('format')

    if format is None:
        format = 'application/json'

    if format not in [ 'application/json', 'text/html', 'text/plain' ]:
        return bad_request("Unsupported format " + format)

    # If asked for 'first', dig up the first run and use its UUID.
    # This is more for debug convenience than anything else.
    if run == 'first':
        try:
            run = __runs_first_run(task)
        except Exception as ex:
            log.exception()
            return error(str(ex))
        if run is None:
            return not_found()

    #
    # Camp on the run for a result
    #

    # 40 tries at 0.25s intervals == 10 sec.
    tries = 40 if wait else 1

    while tries:

        try:
            cursor = dbcursor_query("""
                SELECT
                    test.name,
                    run.result_merged
                FROM
                    run
                    JOIN task ON task.id = run.task
                    JOIN test ON test.id = task.test
                WHERE
                    task.uuid = %s
                    AND run.uuid = %s
                """, [task, run])
        except Exception as ex:
            log.exception()
            return error(str(ex))

        if cursor.rowcount == 0:
            return not_found()

        # TODO: Make sure we got back one row with two columns.
        row = cursor.fetchone()

        if not wait and row[1] is None:
            time.sleep(0.25)
            tries -= 1
        else:
            break

    if tries == 0:
        return not_found()

    test_type, merged_result = row

    # JSON requires no formatting.
    if format == 'application/json':
        return ok_json(merged_result)

    if not merged_result['succeeded']:
        if format == 'text/plain':
            return ok("Test failed.", mimetype=format)
        elif format == 'text/html':
            return ok("
Test failed.
", mimetype=format)
        return error("Unsupported format " + format)

    returncode, stdout, stderr = pscheduler.run_program(
        [ "pscheduler", "internal", "invoke", "test", test_type,
          "result-format", format ],
        stdin = pscheduler.json_dump(merged_result)
        )

    if returncode != 0:
        return error("Failed to format result: " + stderr)

    return ok(stdout.rstrip(), mimetype=format)
<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import

import imageio
import numpy as np
from tqdm import tqdm
import warnings

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.image import imsave
import matplotlib.patheffects as path_effects
from matplotlib.colors import NoNorm

from astropy import log
from astropy import visualization
from astropy.wcs import WCS

import surveyquery

# Figure class
class K2Fig(object):
    """Figure showing K2 target pixel stamp and sky survey image."""

    def __init__(self,TPF):
        self.TPF = TPF
        self.verbose = self.TPF.verbose

    def cut_levels(self, min_percent=1., max_percent=95., data_col='FLUX'):
            """Determine the cut levels for contrast stretching.

            Returns
            -------
            vmin, vmax : float, float
                Min and max cut levels.
            """

            # Get co-added flux
            # update to use TPF
            sample = self.TPF.flux_binned()

            # Scale image
            with warnings.catch_warnings():
                warnings.filterwarnings('ignore', message="(.*)invalid value(.*)")
                vmin, vmax = np.percentile(sample[sample > 0],
                                           [min_percent, max_percent])
            return vmin, vmax

    # Set up the figure and axes using astropy WCS
    def create_figure(self, output_filename, survey, stretch='log', vmin=1, vmax=None, min_percent=1, max_percent=95,
                      cmap='gray', contour_color='red', data_col='FLUX'):
        """Returns a matplotlib Figure object that visualizes a frame.

        Parameters
        ----------

        vmin : float, optional
            Minimum cut level (default: 0).

        vmax : float, optional
            Maximum cut level (default: 5000).

        cmap : str, optional
            The matplotlib color map name.  The default is 'gray',
            can also be e.g. 'gist_heat'.

        raw : boolean, optional
            If `True`, show the raw pixel counts rather than
            the calibrated flux. Default: `False`.

        Returns
        -------
        image : array
            An array of unisgned integers of shape (x, y, 3),
            representing an RBG colour image x px wide and y px high.
        """
        # Get the flux data to visualize
        # Update to use TPF
        flx = self.TPF.flux_binned()
        # print(np.shape(flx))

        # calculate cut_levels
        if vmax is None:
            vmin, vmax = self.cut_levels(min_percent,max_percent,data_col)

        # Determine the figsize
        shape = list(flx.shape)
        # print(shape)
        # Create the figure and display the flux image using matshow
        fig = plt.figure(figsize=shape)
        # Display the image using matshow

        # Update to generate axes using WCS axes instead of plain axes
        ax = plt.subplot(projection=self.TPF.wcs)
        ax.set_xlabel('RA')
        ax.set_ylabel('Dec')

        if self.verbose:
            print('{} vmin/vmax = {}/{} (median={})'.format(data_col, vmin, vmax, np.nanmedian(flx)))

        if stretch == 'linear':
            stretch_fn = visualization.LinearStretch()
        elif stretch == 'sqrt':
            stretch_fn = visualization.SqrtStretch()
        elif stretch == 'power':
            stretch_fn = visualization.PowerStretch(1.0)
        elif stretch == 'log':
            stretch_fn = visualization.LogStretch()
        elif stretch == 'asinh':
            stretch_fn = visualization.AsinhStretch(0.1)
        else:
            raise ValueError('Unknown stretch: {0}.'.format(stretch))

        transform = (stretch_fn +
                     visualization.ManualInterval(vmin=vmin, vmax=vmax))
        ax.imshow((255*transform(flx)).astype(int), aspect='auto',
                   origin='lower', interpolation='nearest',
                   cmap=cmap, norm=NoNorm())
        ax.set_xticks([])
        ax.set_yticks([])

        current_ylims = ax.get_ylim()
        current_xlims = ax.get_xlim()

        pixels, header = surveyquery.getSVImg(self.TPF.position, survey)
        levels = np.linspace(np.min(pixels),np.percentile(pixels,95),10)
        ax.contour(pixels,transform=ax.get_transform(WCS(header)),
                    levels=levels,color=contour_color)

        ax.set_xlim(current_xlims)
        ax.set_ylim(current_ylims)

        fig.canvas.draw()
        plt.savefig(output_filename, bbox_inches='tight', dpi=300)
        return fig
<commit_msg>fixed contour color issue (keyword arg color->colors)<commit_after>from __future__ import division, print_function, absolute_import

import imageio
import numpy as np
from tqdm import tqdm
import warnings

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.image import imsave
import matplotlib.patheffects as path_effects
from matplotlib.colors import NoNorm

from astropy import log
from astropy import visualization
from astropy.wcs import WCS

import surveyquery

# Figure class
class K2Fig(object):
    """Figure showing K2 target pixel stamp and sky survey image."""

    def __init__(self,TPF):
        self.TPF = TPF
        self.verbose = self.TPF.verbose

    def cut_levels(self, min_percent=1., max_percent=95., data_col='FLUX'):
            """Determine the cut levels for contrast stretching.

            Returns
            -------
            vmin, vmax : float, float
                Min and max cut levels.
            """

            # Get co-added flux
            # update to use TPF
            sample = self.TPF.flux_binned()

            # Scale image
            with warnings.catch_warnings():
                warnings.filterwarnings('ignore', message="(.*)invalid value(.*)")
                vmin, vmax = np.percentile(sample[sample > 0],
                                           [min_percent, max_percent])
            return vmin, vmax

    # Set up the figure and axes using astropy WCS
    def create_figure(self, output_filename, survey, stretch='log', vmin=1, vmax=None, min_percent=1, max_percent=95,
                      cmap='gray', contour_color='red', data_col='FLUX'):
        """Returns a matplotlib Figure object that visualizes a frame.

        Parameters
        ----------

        vmin : float, optional
            Minimum cut level (default: 0).

        vmax : float, optional
            Maximum cut level (default: 5000).

        cmap : str, optional
            The matplotlib color map name.  The default is 'gray',
            can also be e.g. 'gist_heat'.

        raw : boolean, optional
            If `True`, show the raw pixel counts rather than
            the calibrated flux. Default: `False`.

        Returns
        -------
        image : array
            An array of unisgned integers of shape (x, y, 3),
            representing an RBG colour image x px wide and y px high.
        """
        # Get the flux data to visualize
        # Update to use TPF
        flx = self.TPF.flux_binned()
        # print(np.shape(flx))

        # calculate cut_levels
        if vmax is None:
            vmin, vmax = self.cut_levels(min_percent,max_percent,data_col)

        # Determine the figsize
        shape = list(flx.shape)
        # print(shape)
        # Create the figure and display the flux image using matshow
        fig = plt.figure(figsize=shape)
        # Display the image using matshow

        # Update to generate axes using WCS axes instead of plain axes
        ax = plt.subplot(projection=self.TPF.wcs)
        ax.set_xlabel('RA')
        ax.set_ylabel('Dec')

        if self.verbose:
            print('{} vmin/vmax = {}/{} (median={})'.format(data_col, vmin, vmax, np.nanmedian(flx)))

        if stretch == 'linear':
            stretch_fn = visualization.LinearStretch()
        elif stretch == 'sqrt':
            stretch_fn = visualization.SqrtStretch()
        elif stretch == 'power':
            stretch_fn = visualization.PowerStretch(1.0)
        elif stretch == 'log':
            stretch_fn = visualization.LogStretch()
        elif stretch == 'asinh':
            stretch_fn = visualization.AsinhStretch(0.1)
        else:
            raise ValueError('Unknown stretch: {0}.'.format(stretch))

        transform = (stretch_fn +
                     visualization.ManualInterval(vmin=vmin, vmax=vmax))
        ax.imshow((255*transform(flx)).astype(int), aspect='auto',
                   origin='lower', interpolation='nearest',
                   cmap=cmap, norm=NoNorm())
        ax.set_xticks([])
        ax.set_yticks([])

        current_ylims = ax.get_ylim()
        current_xlims = ax.get_xlim()

        pixels, header = surveyquery.getSVImg(self.TPF.position, survey)
        levels = np.linspace(np.min(pixels),np.percentile(pixels,95),10)
        ax.contour(pixels,transform=ax.get_transform(WCS(header)),
                    levels=levels,colors=contour_color)

        ax.set_xlim(current_xlims)
        ax.set_ylim(current_ylims)

        fig.canvas.draw()
        plt.savefig(output_filename, bbox_inches='tight', dpi=300)
        return fig
<|endoftext|>


<commit_before><commit_msg>one more bug...<commit_after><|endoftext|>


<commit_before>from collections import OrderedDict
import maya.cmds as cmds
from blur3d import api
from blur3d.api.abstract.abstractscenemodel import AbstractSceneModel

resolutionAttr = 'Active_Resolution'

#------------------------------------------------------------------------

class MayaSceneModel(AbstractSceneModel):
	def _attrName(self):
		return '{node}.{attr}'.format(node=self._nativeName(), attr=resolutionAttr)

	def __init__(self, scene, nativeObject):
		super(MayaSceneModel, self).__init__(scene, nativeObject)

	def addResolution(self, name='', path='', load=False):
		if self.isReferenced():
			reses = self.userProps().get('Resolutions', OrderedDict())
			reses[name] = path
			self.userProps()['Resolutions'] = reses
			# Update the enum
			cmds.addAttr(self._attrName(), edit=True, enumName=':'.join(self.resolutions()))
			if load:
				self.setResolution(name)
			return True
		return False

	def isReferenced(self):
		# Check if it already has referenceing set up
		ret = resolutionAttr in cmds.listAttr(self._nativeName())
		if not ret:
			props = self.userProps()
			# Check if this object should support referencing
			ret = props.get('Referenced')
			if ret:
				# Object should support referencing, but referencing hasn't been setup, so create
				# the structure.
				cmds.addAttr(
						self._nativeName(),
						longName=resolutionAttr,
						attributeType="enum",
						enumName="Offloaded")
				# Make the attribute viewable, but not keyable in the channelBox
				cmds.setAttr(self._attrName(), keyable=False, channelBox=True)
				# Remove the temp reference user prop
				del(props['Referenced'])
				# Create the user props for the reference
				props['Resolutions'] = OrderedDict(Offloaded='')
		return ret

	def offload(self):
		return False

	def offloaded(self):
		return self.resolution() == 'Offloaded'

	def removeResolution(self, name):
		return False

	def resolution(self):
		if self.isReferenced():
			return cmds.getAttr(self._attrName(), asString=True)
		return ''

	def resolutionPath(self, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.userProps().get('Resolutions', {})
			if resolution in resolutions:
				return resolutions[resolution]
		return ''

	def resolutions(self):
		return self.userProps().get('Resolutions', {}).keys()

	def resolutionsPaths(self):
		return [self.resolutionPath(resolution) for resolution in self.resolutions() if resolution != 'Offloaded']

	def _referenceNodeName(self):
		filename = self.userProps().get('Loaded_Reference')
		if filename:
			return cmds.referenceQuery(filename, referenceNode=True)
		return None

	def setResolution(self, resolution):
		if self.isReferenced():
			# handles qstrings
			resolution = unicode(resolution)
			# If I dont re-initialize a model object it does not return me the right resolutions.
			resolutions = self.resolutions()
			for i, res in enumerate(resolutions):
				# Handles cases sensitivity (crappy, but needed because resolution
				# names often come from filenames).
				if res.lower() != resolution.lower():
					continue
				path = self.resolutionPath(res)
				# Update the existing reference if it exists
				nodeName = self._referenceNodeName()
				if nodeName:
					cmds.file(path, loadReference=nodeName)
					filename = cmds.referenceQuery(nodeName, filename=True)
				else:
					# We use the filename to replace/unload the reference later. If c:\test.ma is
					# referenced twice the second uses the filename c:\test.ma{1}.
					filename = cmds.file(path, reference=True, namespace=":")
					# Find all top level items that were referenced into the scene
					topLevel = []
					# Use the namespace to find all of the nodes we just referenced into the scene
					objects = cmds.referenceQuery(filename, showDagPath=True, nodes=True)
					for obj in objects:
						if cmds.nodeType(obj) == 'transform' and not cmds.listRelatives(obj, parent=True, path=True):
									topLevel.append(obj)
					# Reparent the top level nodes under the model locator
					topLevel.append(self.name())
					cmds.parent(*topLevel)
				# Update the Active_Resolution property
				self.setProperty(resolutionAttr, i)
				# Store the loaded reference filename so we can remove it later
				self.userProps()['Loaded_Reference'] = filename
				return True
		else:
			self.userProps()['resolution'] = resolution
		return False

	def setResolutionPath(self, path, name=''):
		return False

	def update(self):
		return False

# register the symbol
api.registerSymbol('SceneModel', MayaSceneModel)<commit_msg>MayaSceneModel.setResolution now supports offloaded. It also supports offloading resolutions, and reparenting dagContainers containing "unparentable" nodes.<commit_after>from collections import OrderedDict
import maya.cmds as cmds
from blur3d import api
from blur3d.api.abstract.abstractscenemodel import AbstractSceneModel

resolutionAttr = 'Active_Resolution'

#------------------------------------------------------------------------

class MayaSceneModel(AbstractSceneModel):
	def _attrName(self):
		return '{node}.{attr}'.format(node=self._nativeName(), attr=resolutionAttr)

	def __init__(self, scene, nativeObject):
		super(MayaSceneModel, self).__init__(scene, nativeObject)

	def addResolution(self, name='', path='', load=False):
		if self.isReferenced():
			reses = self.userProps().get('Resolutions', OrderedDict())
			reses[name] = path
			self.userProps()['Resolutions'] = reses
			# Update the enum
			cmds.addAttr(self._attrName(), edit=True, enumName=':'.join(self.resolutions()))
			if load:
				self.setResolution(name)
			return True
		return False

	def isReferenced(self):
		# Check if it already has referenceing set up
		ret = resolutionAttr in cmds.listAttr(self._nativeName())
		if not ret:
			props = self.userProps()
			# Check if this object should support referencing
			ret = props.get('Referenced')
			if ret:
				# Object should support referencing, but referencing hasn't been setup, so create
				# the structure.
				cmds.addAttr(
						self._nativeName(),
						longName=resolutionAttr,
						attributeType="enum",
						enumName="Offloaded")
				# Make the attribute viewable, but not keyable in the channelBox
				cmds.setAttr(self._attrName(), keyable=False, channelBox=True)
				# Remove the temp reference user prop
				del(props['Referenced'])
				# Create the user props for the reference
				props['Resolutions'] = OrderedDict(Offloaded='')
		return ret

	def offload(self):
		return False

	def offloaded(self):
		return self.resolution() == 'Offloaded'

	def removeResolution(self, name):
		return False

	def resolution(self):
		if self.isReferenced():
			return cmds.getAttr(self._attrName(), asString=True)
		return ''

	def resolutionPath(self, resolution=''):
		if self.isReferenced():
			if not resolution:
				resolution = self.resolution()
			resolutions = self.userProps().get('Resolutions', {})
			if resolution in resolutions:
				return resolutions[resolution]
		return ''

	def resolutions(self):
		return self.userProps().get('Resolutions', {}).keys()

	def resolutionsPaths(self):
		return [self.resolutionPath(resolution) for resolution in self.resolutions() if resolution != 'Offloaded']

	def _referenceNodeName(self):
		filename = self.userProps().get('Loaded_Reference')
		if filename:
			return cmds.referenceQuery(filename, referenceNode=True)
		return None

	def setResolution(self, resolution):
		if self.isReferenced():
			# handles qstrings
			resolution = unicode(resolution)
			# If I dont re-initialize a model object it does not return me the right resolutions.
			resolutions = self.resolutions()
			for i, res in enumerate(resolutions):
				# Handles cases sensitivity (crappy, but needed because resolution
				# names often come from filenames).
				if res.lower() != resolution.lower():
					continue
				path = self.resolutionPath(res)
				nodeName = self._referenceNodeName()
				if res.lower() == 'offloaded':
					# If offloaded, unload the reference
					if nodeName:
						cmds.file(unloadReference=nodeName)
					return True
				# If the reference is already loaded, switch it with the requested one while
				# preserving any updated properties like animation.
				elif nodeName:
					cmds.file(path, loadReference=nodeName)
					filename = cmds.referenceQuery(nodeName, filename=True)
				else:
					# We use the filename to replace/unload the reference later. If c:\test.ma is
					# referenced twice the second uses the filename c:\test.ma{1}.
					filename = cmds.file(path, reference=True, namespace=":")
				# Find all top level items that were referenced into the scene and reparent them
				# under the model node
				objects = cmds.referenceQuery(filename, dagPath=True, nodes=True)
				topLevel = []
				for obj in objects:
					if cmds.nodeType(obj) in ('transform', 'dagContainer') and \
							not cmds.listRelatives(obj, parent=True, path=True):
								topLevel.append(obj)
				# Reparent the top level nodes under the model locator
				topLevel.append(self.name())
				if len(topLevel) > 1:
					cmds.parent(*topLevel)
				# Update the Active_Resolution property
				self.setProperty(resolutionAttr, i)
				# Store the loaded reference filename so we can remove it later
				self.userProps()['Loaded_Reference'] = filename
				return True
		else:
			self.userProps()['resolution'] = resolution
		return False

	def setResolutionPath(self, path, name=''):
		return False

	def update(self):
		return False

# register the symbol
api.registerSymbol('SceneModel', MayaSceneModel)<|endoftext|>


<commit_before># Copyright (c) 2012, 2015 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2005-2008 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
#          Andreas Hansson

from MemObject import MemObject
from System import System
from m5.params import *
from m5.proxy import *
from m5.SimObject import SimObject

class BaseXBar(MemObject):
    type = 'BaseXBar'
    abstract = True
    cxx_header = "mem/xbar.hh"

    slave = VectorSlavePort("Vector port for connecting masters")
    master = VectorMasterPort("Vector port for connecting slaves")

    # Latencies governing the time taken for the variuos paths a
    # packet has through the crossbar. Note that the crossbar itself
    # does not add the latency due to assumptions in the coherency
    # mechanism. Instead the latency is annotated on the packet and
    # left to the neighbouring modules.
    #
    # A request incurs the frontend latency, possibly snoop filter
    # lookup latency, and forward latency. A response incurs the
    # response latency. Frontend latency encompasses arbitration and
    # deciding what to do when a request arrives. the forward latency
    # is the latency involved once a decision is made to forward the
    # request. The response latency, is similar to the forward
    # latency, but for responses rather than requests.
    frontend_latency = Param.Cycles("Frontend latency")
    forward_latency = Param.Cycles("Forward latency")
    response_latency = Param.Cycles("Response latency")

    # Width governing the throughput of the crossbar
    width = Param.Unsigned("Datapath width per port (bytes)")

    # The default port can be left unconnected, or be used to connect
    # a default slave port
    default = MasterPort("Port for connecting an optional default slave")

    # The default port can be used unconditionally, or based on
    # address range, in which case it may overlap with other
    # ports. The default range is always checked first, thus creating
    # a two-level hierarchical lookup. This is useful e.g. for the PCI
    # xbar configuration.
    use_default_range = Param.Bool(False, "Perform address mapping for " \
                                       "the default port")

class NoncoherentXBar(BaseXBar):
    type = 'NoncoherentXBar'
    cxx_header = "mem/noncoherent_xbar.hh"

class CoherentXBar(BaseXBar):
    type = 'CoherentXBar'
    cxx_header = "mem/coherent_xbar.hh"

    # The coherent crossbar additionally has snoop responses that are
    # forwarded after a specific latency.
    snoop_response_latency = Param.Cycles("Snoop response latency")

    # An optional snoop filter
    snoop_filter = Param.SnoopFilter(NULL, "Selected snoop filter")

    system = Param.System(Parent.any, "System that the crossbar belongs to.")

class SnoopFilter(SimObject):
    type = 'SnoopFilter'
    cxx_header = "mem/snoop_filter.hh"

    # Lookup latency of the snoop filter, added to requests that pass
    # through a coherent crossbar.
    lookup_latency = Param.Cycles(1, "Lookup latency")

    system = Param.System(Parent.any, "System that the crossbar belongs to.")

# We use a coherent crossbar to connect multiple masters to the L2
# caches. Normally this crossbar would be part of the cache itself.
class L2XBar(CoherentXBar):
    # 256-bit crossbar by default
    width = 32

    # Assume that most of this is covered by the cache latencies, with
    # no more than a single pipeline stage for any packet.
    frontend_latency = 1
    forward_latency = 0
    response_latency = 1
    snoop_response_latency = 1

# One of the key coherent crossbar instances is the system
# interconnect, tying together the CPU clusters, GPUs, and any I/O
# coherent masters, and DRAM controllers.
class SystemXBar(CoherentXBar):
    # 128-bit crossbar by default
    width = 16

    # A handful pipeline stages for each portion of the latency
    # contributions.
    frontend_latency = 3
    forward_latency = 4
    response_latency = 2
    snoop_response_latency = 4

# In addition to the system interconnect, we typically also have one
# or more on-chip I/O crossbars. Note that at some point we might want
# to also define an off-chip I/O crossbar such as PCIe.
class IOXBar(NoncoherentXBar):
    # 128-bit crossbar by default
    width = 16

    # Assume a simpler datapath than a coherent crossbar, incuring
    # less pipeline stages for decision making and forwarding of
    # requests.
    frontend_latency = 2
    forward_latency = 1
    response_latency = 2
<commit_msg>Add the class of virtual crossbar for logic connection purpose<commit_after># Copyright (c) 2012, 2015 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Copyright (c) 2005-2008 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Nathan Binkert
#          Andreas Hansson

from MemObject import MemObject
from System import System
from m5.params import *
from m5.proxy import *
from m5.SimObject import SimObject

class BaseXBar(MemObject):
    type = 'BaseXBar'
    abstract = True
    cxx_header = "mem/xbar.hh"

    slave = VectorSlavePort("Vector port for connecting masters")
    master = VectorMasterPort("Vector port for connecting slaves")

    # Latencies governing the time taken for the variuos paths a
    # packet has through the crossbar. Note that the crossbar itself
    # does not add the latency due to assumptions in the coherency
    # mechanism. Instead the latency is annotated on the packet and
    # left to the neighbouring modules.
    #
    # A request incurs the frontend latency, possibly snoop filter
    # lookup latency, and forward latency. A response incurs the
    # response latency. Frontend latency encompasses arbitration and
    # deciding what to do when a request arrives. the forward latency
    # is the latency involved once a decision is made to forward the
    # request. The response latency, is similar to the forward
    # latency, but for responses rather than requests.
    frontend_latency = Param.Cycles("Frontend latency")
    forward_latency = Param.Cycles("Forward latency")
    response_latency = Param.Cycles("Response latency")

    # Width governing the throughput of the crossbar
    width = Param.Unsigned("Datapath width per port (bytes)")

    # The default port can be left unconnected, or be used to connect
    # a default slave port
    default = MasterPort("Port for connecting an optional default slave")

    # The default port can be used unconditionally, or based on
    # address range, in which case it may overlap with other
    # ports. The default range is always checked first, thus creating
    # a two-level hierarchical lookup. This is useful e.g. for the PCI
    # xbar configuration.
    use_default_range = Param.Bool(False, "Perform address mapping for " \
                                       "the default port")

class NoncoherentXBar(BaseXBar):
    type = 'NoncoherentXBar'
    cxx_header = "mem/noncoherent_xbar.hh"

class CoherentXBar(BaseXBar):
    type = 'CoherentXBar'
    cxx_header = "mem/coherent_xbar.hh"

    # The coherent crossbar additionally has snoop responses that are
    # forwarded after a specific latency.
    snoop_response_latency = Param.Cycles("Snoop response latency")

    # An optional snoop filter
    snoop_filter = Param.SnoopFilter(NULL, "Selected snoop filter")

    system = Param.System(Parent.any, "System that the crossbar belongs to.")

class SnoopFilter(SimObject):
    type = 'SnoopFilter'
    cxx_header = "mem/snoop_filter.hh"

    # Lookup latency of the snoop filter, added to requests that pass
    # through a coherent crossbar.
    lookup_latency = Param.Cycles(1, "Lookup latency")

    system = Param.System(Parent.any, "System that the crossbar belongs to.")

# We use a coherent crossbar to connect multiple masters to the L2
# caches. Normally this crossbar would be part of the cache itself.
class L2XBar(CoherentXBar):
    # 256-bit crossbar by default
    width = 32

    # Assume that most of this is covered by the cache latencies, with
    # no more than a single pipeline stage for any packet.
    frontend_latency = 1
    forward_latency = 0
    response_latency = 1
    snoop_response_latency = 1

# One of the key coherent crossbar instances is the system
# interconnect, tying together the CPU clusters, GPUs, and any I/O
# coherent masters, and DRAM controllers.
class SystemXBar(CoherentXBar):
    # 128-bit crossbar by default
    width = 16

    # A handful pipeline stages for each portion of the latency
    # contributions.
    frontend_latency = 3
    forward_latency = 4
    response_latency = 2
    snoop_response_latency = 4

# In addition to the system interconnect, we typically also have one
# or more on-chip I/O crossbars. Note that at some point we might want
# to also define an off-chip I/O crossbar such as PCIe.
class IOXBar(NoncoherentXBar):
    # 128-bit crossbar by default
    width = 16

    # Assume a simpler datapath than a coherent crossbar, incuring
    # less pipeline stages for decision making and forwarding of
    # requests.
    frontend_latency = 2
    forward_latency = 1
    response_latency = 2

# For logic connections. We can use this to organize components.
class VirtualXBar(NoncoherentXBar):
    # 128-bit crossbar by default
    width = 16

    # Assume no latency. Only for connection purpose.
    frontend_latency = 0
    forward_latency = 0
    response_latency = 0
<|endoftext|>


<commit_before>import json
from invisibleroads_macros_log import format_path
from logging import getLogger

from ..constants import (
    MODE_CODE_BY_NAME,
    MODE_ROUTE,
    VARIABLE_ROUTE)
from ..exceptions import (
    CrossComputeDataError)
from ..macros.iterable import find_item
from .interface import Batch
from .variable import (
    load_variable_data)

class DiskBatch(Batch):

    def __init__(self, automation_definition, batch_definition):
        self.automation_definition = automation_definition
        self.batch_definition = batch_definition
        self.folder = automation_definition.folder / batch_definition.folder

    def get_variable_configuration(self, variable_definition):
        variable_configuration = variable_definition.configuration
        if 'path' in variable_configuration:
            mode_name = variable_definition.mode_name
            path = self.folder / mode_name / variable_configuration['path']
            try:
                variable_configuration.update(json.load(open(path, 'rt')))
            except OSError:
                L.error('path not found %s', format_path(path))
            except json.JSONDecodeError:
                L.error('must be json %s', format_path(path))
            except TypeError:
                L.error('must contain a dictionary %s', format_path(path))
        return variable_configuration

    def get_data(self, variable_definition):
        variable_path = variable_definition.path
        if variable_path == 'ENVIRONMENT':
            return {}
        mode_name = variable_definition.mode_name
        variable_id = variable_definition.id
        path = self.folder / mode_name / variable_path
        try:
            variable_data = load_variable_data(path, variable_id)
        except CrossComputeDataError as e:
            L.error(e)
            return {'error': e}
        return variable_data

    def get_data_uri(self, variable_definition, element):
        base_uri = element.base_uri
        automation_uri = self.automation_definition.uri
        batch_uri = self.batch_definition.uri
        mode_code = MODE_CODE_BY_NAME[variable_definition.mode_name]
        mode_uri = MODE_ROUTE.format(mode_code=mode_code)
        variable_uri = VARIABLE_ROUTE.format(
            variable_id=variable_definition.id)
        return base_uri + automation_uri + batch_uri + mode_uri + variable_uri

    def is_match(self, payload):
        variable_definitions = self.automation_definition.variable_definitions
        for name, value in payload.items():
            try:
                variable_definition = find_item(
                    variable_definitions, 'id', name)
            except StopIteration:
                continue
            data = self.get_data(variable_definition)
            variable_value = data.get('value')
            if isinstance(value, list):
                if variable_value not in value:
                    return False
            else:
                if variable_value != value:
                    return False
        return True

L = getLogger(__name__)
<commit_msg>Use input variables<commit_after>import json
from invisibleroads_macros_log import format_path
from logging import getLogger

from ..constants import (
    MODE_CODE_BY_NAME,
    MODE_ROUTE,
    VARIABLE_ROUTE)
from ..exceptions import (
    CrossComputeDataError)
from ..macros.iterable import find_item
from .interface import Batch
from .variable import (
    load_variable_data)

class DiskBatch(Batch):

    def __init__(self, automation_definition, batch_definition):
        self.automation_definition = automation_definition
        self.batch_definition = batch_definition
        self.folder = automation_definition.folder / batch_definition.folder

    def get_variable_configuration(self, variable_definition):
        variable_configuration = variable_definition.configuration
        if 'path' in variable_configuration:
            mode_name = variable_definition.mode_name
            path = self.folder / mode_name / variable_configuration['path']
            try:
                variable_configuration.update(json.load(open(path, 'rt')))
            except OSError:
                L.error('path not found %s', format_path(path))
            except json.JSONDecodeError:
                L.error('must be json %s', format_path(path))
            except TypeError:
                L.error('must contain a dictionary %s', format_path(path))
        return variable_configuration

    def get_data(self, variable_definition):
        variable_path = variable_definition.path
        if variable_path == 'ENVIRONMENT':
            return {}
        mode_name = variable_definition.mode_name
        variable_id = variable_definition.id
        path = self.folder / mode_name / variable_path
        try:
            variable_data = load_variable_data(path, variable_id)
        except CrossComputeDataError as e:
            L.error(e)
            return {'error': e}
        return variable_data

    def get_data_uri(self, variable_definition, element):
        base_uri = element.base_uri
        automation_uri = self.automation_definition.uri
        batch_uri = self.batch_definition.uri
        mode_code = MODE_CODE_BY_NAME[variable_definition.mode_name]
        mode_uri = MODE_ROUTE.format(mode_code=mode_code)
        variable_uri = VARIABLE_ROUTE.format(
            variable_id=variable_definition.id)
        return base_uri + automation_uri + batch_uri + mode_uri + variable_uri

    def is_match(self, payload):
        automation_definition = self.automation_definition
        variable_definitions = automation_definition.get_variable_definitions(
            'input')
        for name, value in payload.items():
            try:
                variable_definition = find_item(
                    variable_definitions, 'id', name)
            except StopIteration:
                continue
            data = self.get_data(variable_definition)
            variable_value = data.get('value')
            if isinstance(value, list):
                if variable_value not in value:
                    return False
            else:
                if variable_value != value:
                    return False
        return True

L = getLogger(__name__)
<|endoftext|>


<commit_before>#from malleefowl.process import WorkerProcess
import malleefowl.process
import subprocess
from malleefowl import tokenmgr, utils
from malleefowl import wpslogging as logging
logger = logging.getLogger(__name__)

class IndicesProcess(malleefowl.process.WorkerProcess):
    """This process calculates the relative humidity"""

    def __init__(self):
        # definition of this process
        malleefowl.process.WorkerProcess.__init__(self,
            identifier = "de.csc.indice",
            title="Climate indices",
            version = "0.1",
            #storeSupported = "true",   # async
            #statusSupported = "true",  # retrieve status, needs to be true for async
            ## TODO: what can i do with this?
            metadata=[],
            abstract="Just testing a python script to test icclim",
            #extra_metadata={
                  #'esgfilter': 'variable:tas, variable:evspsblpot, variable:huss, variable:ps, variable:pr, variable:sftlf, time_frequency:day',
                  #'esgquery': 'data_node:esg-dn1.nsc.liu.se'
                  #},
            extra_metadata={
                  'esgfilter': 'variable:tasmax, variable:tasmin, variable:tas, variable:pr, project:CMIP5, project:CORDEX',
                  'esgquery': ' time_frequency:day'
                  },
            )

        # Literal Input Data
        # ------------------

        self.token = self.addLiteralInput(
            identifier = "token",
            title = "Token",
            abstract = "Your unique token to recieve data",
            minOccurs = 1,
            maxOccurs = 1,
            type = type('')
            )

        self.TG = self.addLiteralInput(
            identifier="TG",
            title="TG",
            abstract="Mean of mean temperatur (tas files as input files)",
            type=type(False),
            default=False,
            minOccurs=0,
            maxOccurs=0,
            )

        self.TX = self.addLiteralInput(
            identifier="TX",
            title="TX",
            abstract="mean of max temperatur (tasmax files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TN = self.addLiteralInput(
            identifier="TN",
            title="TN",
            abstract="Mean over min temperatur (tasmin files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR = self.addLiteralInput(
            identifier="RR",
            title="RR",
            abstract="precipitation sum (pr files as input files) ",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TG_5to9 = self.addLiteralInput(
            identifier="TG_5to9",
            title="TG_5to9",
            abstract="mean temperature (K) form Mai to September",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TG_6to8 = self.addLiteralInput(
            identifier="TG_6to8",
            title="TG_6to8",
            abstract="mean temperature (K) form Juni to August",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR_5to9 = self.addLiteralInput(
            identifier="RR_5to9",
            title="RR_5to9",
            abstract="precipitation sum (mm) form Mai to September",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR_6to8 = self.addLiteralInput(
            identifier="RR_6to8",
            title="RR_6to8",
            abstract="precipitation sum (mm) form Juni to August",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.SU = self.addLiteralInput(
            identifier="SU",
            title="SU",
            abstract="Nr of summer days (tasmax files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        # complex output
        # -------------
        self.output = self.addComplexOutput(
            identifier="output",
            title="indice log",
            abstract="indice log",
            metadata=[],
            formats=[{"mimeType":"text/plain"}],
            asReference=True,
            )
    def execute(self):

        import csc
        import os

        token = self.token.getValue()
        userid = tokenmgr.get_userid(tokenmgr.sys_token(), token)
        outdir = os.path.join(self.files_path, userid)
        utils.mkdir(outdir)
       #result = publish.to_local_store(files=self.get_nc_files(),basename=self.basename.getValue(),userid=userid)

        self.show_status('starting calcualtion of icclim indices', 5)

        result = csc.tools.indices(outdir, self.get_nc_files(), self.TG.getValue(), self.TX.getValue(), self.TN.getValue(), self.RR.getValue(), self.TG_5to9.getValue(), self.TG_6to8.getValue(), self.RR_5to9.getValue(), self.RR_6to8.getValue(), self.SU.getValue())

        outfile = self.mktempfile(suffix='.txt')

        with open(outfile, 'w') as fp:
             fp.write(result)

        self.output.setValue( outfile )

        self.show_status("processing done", 100)

<commit_msg>adapted indice_worker to new malleefowl<commit_after>from malleefowl.process import WPSProcess
import subprocess
from malleefowl import utils
from malleefowl import wpslogging as logging
logger = logging.getLogger(__name__)

class IndicesProcess(WPSProcess):
    """This process calculates the relative humidity"""

    def __init__(self):
        # definition of this process
        WPSProcess.__init__(self,
            identifier = "indice",
            title="Climate indices",
            version = "0.1",
            metadata=[],
            abstract="Just testing a python script to test icclim",
            # TODO: filter are configured in phoenix
            #extra_metadata={
                  #'esgfilter': 'variable:tas, variable:evspsblpot, variable:huss, variable:ps, variable:pr, variable:sftlf, time_frequency:day',
                  #'esgquery': 'data_node:esg-dn1.nsc.liu.se'
                  #},
            ## extra_metadata={
            ##       'esgfilter': 'variable:tasmax, variable:tasmin, variable:tas, variable:pr, project:CMIP5, project:CORDEX',
            ##       'esgquery': ' time_frequency:day'
            ##       },
            )

        self.netcdf_file = self.addComplexInput(
            identifier="netcdf_file",
            title="NetCDF File",
            abstract="NetCDF File",
            minOccurs=1,
            maxOccurs=100,
            maxmegabites=5000,
            formats=[{"mimeType":"application/x-netcdf"}],
            )

        self.TG = self.addLiteralInput(
            identifier="TG",
            title="TG",
            abstract="Mean of mean temperatur (tas files as input files)",
            type=type(False),
            default=False,
            minOccurs=0,
            maxOccurs=0,
            )

        self.TX = self.addLiteralInput(
            identifier="TX",
            title="TX",
            abstract="mean of max temperatur (tasmax files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TN = self.addLiteralInput(
            identifier="TN",
            title="TN",
            abstract="Mean over min temperatur (tasmin files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR = self.addLiteralInput(
            identifier="RR",
            title="RR",
            abstract="precipitation sum (pr files as input files) ",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TG_5to9 = self.addLiteralInput(
            identifier="TG_5to9",
            title="TG_5to9",
            abstract="mean temperature (K) form Mai to September",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.TG_6to8 = self.addLiteralInput(
            identifier="TG_6to8",
            title="TG_6to8",
            abstract="mean temperature (K) form Juni to August",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR_5to9 = self.addLiteralInput(
            identifier="RR_5to9",
            title="RR_5to9",
            abstract="precipitation sum (mm) form Mai to September",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.RR_6to8 = self.addLiteralInput(
            identifier="RR_6to8",
            title="RR_6to8",
            abstract="precipitation sum (mm) form Juni to August",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        self.SU = self.addLiteralInput(
            identifier="SU",
            title="SU",
            abstract="Nr of summer days (tasmax files as input files)",
            default=False,
            type=type(False),
            minOccurs=0,
            maxOccurs=0,
            )

        # complex output
        # -------------
        self.output = self.addComplexOutput(
            identifier="output",
            title="indice log",
            abstract="indice log",
            metadata=[],
            formats=[{"mimeType":"text/plain"}],
            asReference=True,
            )
    def execute(self):
        import csc
        import os

        self.show_status('starting calcualtion of icclim indices', 0)

        ncfiles = self.getInputValues(identifier='netcdf_file')
        result = csc.tools.indices(os.curdir, ncfiles,
                                   self.TG.getValue(),
                                   self.TX.getValue(),
                                   self.TN.getValue(),
                                   self.RR.getValue(),
                                   self.TG_5to9.getValue(),
                                   self.TG_6to8.getValue(),
                                   self.RR_5to9.getValue(),
                                   self.RR_6to8.getValue(),
                                   self.SU.getValue())

        outfile = self.mktempfile(suffix='.txt')

        with open(outfile, 'w') as fp:
             fp.write(result)

        self.output.setValue( outfile )

        self.show_status("processing done", 100)

<|endoftext|>


<commit_before>from copy import deepcopy

import networkx as nx

def signed_edges_to_signed_nodes(graph, prune_nodes=True,
                                 edge_signs={'pos': 0, 'neg': 1},
                                 copy_edge_data=False):
    """Convert a graph with signed edges to a graph with signed nodes.

    Each pair of nodes linked by an edge in an input graph are represented
    as four nodes and two edges in the new graph. For example, an edge (a,
    b, 0), where a and b are nodes and 0 is a sign of an edge (positive),
    will be represented as edges ((a, 0), (b, 0)) and ((a, 1), (b, 1)),
    where (a, 0), (a, 1), (b, 0), (b, 1) are signed nodes. An edge (a, b,
    1) with sign 1 (negative) will be represented as edges ((a, 0), (b,
    1)) and ((a, 1), (b, 0)).

    Parameters
    ----------
    graph : networkx.MultiDiGraph
        Graph with signed edges to convert. Can have multiple edges between
        a pair of nodes.
    prune_nodes : Optional[bool]
        If True, iteratively prunes negative (with sign 1) nodes without
        predecessors.
    edge_signs : dict
        A dictionary representing the signing policy of incoming graph. The
        dictionary should have strings 'pos' and 'neg' as keys and integers
        as values.
    copy_edge_data : bool|set(keys)
        Option for copying edge data as well from graph. If False (default),
        no edge data is copied (except sign). If True, all edge data is
        copied. If a set of keys is provided, only the keys appearing in the
        set will be copied, assuming the key is part of a nested dictionary.

    Returns
    -------
    signed_nodes_graph : networkx.DiGraph
    """
    signed_nodes_graph = nx.DiGraph()
    nodes = []
    for node, node_data in graph.nodes(data=True):
        nodes.append(((node, 0), node_data))
        nodes.append(((node, 1), node_data))
    signed_nodes_graph.add_nodes_from(nodes)
    edges = []
    for u, v, edge_data in graph.edges(data=True):
        copy_dict = deepcopy(edge_data)
        edge_sign = copy_dict.pop('sign', None)
        if edge_sign is None:
            continue
        edge_dict = copy_dict if copy_edge_data == True else \
            ({k: v for k, v in copy_dict.items() if k in copy_edge_data} if
             isinstance(copy_edge_data, set) else {})
        if edge_sign == edge_signs['pos']:
            edges.append(((u, 0), (v, 0), edge_dict))
            edges.append(((u, 1), (v, 1), edge_dict))
        elif edge_sign == edge_signs['neg']:
            edges.append(((u, 0), (v, 1), edge_dict))
            edges.append(((u, 1), (v, 0), edge_dict))
    signed_nodes_graph.add_edges_from(edges)
    if prune_nodes:
        signed_nodes_graph = prune_signed_nodes(signed_nodes_graph)
    return signed_nodes_graph

def prune_signed_nodes(graph):
    """Prune nodes with sign (1) if they do not have predecessors."""
    nodes_to_prune = [node for node, in_deg
                      in graph.in_degree()
                      if in_deg == 0 and node[1] == 1]
    while nodes_to_prune:
        graph.remove_nodes_from(nodes_to_prune)
        # Make a list of nodes whose in degree is now 0
        nodes_to_prune = [node for node, in_deg
                          in graph.in_degree()
                          if in_deg == 0 and node[1] == 1]
    return graph<commit_msg>Immigrate sign helpers from depmap_analysis<commit_after>import logging
from copy import deepcopy

import networkx as nx

logger = logging.getLogger(__name__)

def signed_edges_to_signed_nodes(graph, prune_nodes=True,
                                 edge_signs={'pos': 0, 'neg': 1},
                                 copy_edge_data=False):
    """Convert a graph with signed edges to a graph with signed nodes.

    Each pair of nodes linked by an edge in an input graph are represented
    as four nodes and two edges in the new graph. For example, an edge (a,
    b, 0), where a and b are nodes and 0 is a sign of an edge (positive),
    will be represented as edges ((a, 0), (b, 0)) and ((a, 1), (b, 1)),
    where (a, 0), (a, 1), (b, 0), (b, 1) are signed nodes. An edge (a, b,
    1) with sign 1 (negative) will be represented as edges ((a, 0), (b,
    1)) and ((a, 1), (b, 0)).

    Parameters
    ----------
    graph : networkx.MultiDiGraph
        Graph with signed edges to convert. Can have multiple edges between
        a pair of nodes.
    prune_nodes : Optional[bool]
        If True, iteratively prunes negative (with sign 1) nodes without
        predecessors.
    edge_signs : dict
        A dictionary representing the signing policy of incoming graph. The
        dictionary should have strings 'pos' and 'neg' as keys and integers
        as values.
    copy_edge_data : bool|set(keys)
        Option for copying edge data as well from graph. If False (default),
        no edge data is copied (except sign). If True, all edge data is
        copied. If a set of keys is provided, only the keys appearing in the
        set will be copied, assuming the key is part of a nested dictionary.

    Returns
    -------
    signed_nodes_graph : networkx.DiGraph
    """
    signed_nodes_graph = nx.DiGraph()
    nodes = []
    for node, node_data in graph.nodes(data=True):
        nodes.append(((node, 0), node_data))
        nodes.append(((node, 1), node_data))
    signed_nodes_graph.add_nodes_from(nodes)
    edges = []
    for u, v, edge_data in graph.edges(data=True):
        copy_dict = deepcopy(edge_data)
        edge_sign = copy_dict.pop('sign', None)
        if edge_sign is None:
            continue
        edge_dict = copy_dict if copy_edge_data == True else \
            ({k: v for k, v in copy_dict.items() if k in copy_edge_data} if
             isinstance(copy_edge_data, set) else {})
        if edge_sign == edge_signs['pos']:
            edges.append(((u, 0), (v, 0), edge_dict))
            edges.append(((u, 1), (v, 1), edge_dict))
        elif edge_sign == edge_signs['neg']:
            edges.append(((u, 0), (v, 1), edge_dict))
            edges.append(((u, 1), (v, 0), edge_dict))
    signed_nodes_graph.add_edges_from(edges)
    if prune_nodes:
        signed_nodes_graph = prune_signed_nodes(signed_nodes_graph)
    return signed_nodes_graph

def prune_signed_nodes(graph):
    """Prune nodes with sign (1) if they do not have predecessors."""
    nodes_to_prune = [node for node, in_deg
                      in graph.in_degree()
                      if in_deg == 0 and node[1] == 1]
    while nodes_to_prune:
        graph.remove_nodes_from(nodes_to_prune)
        # Make a list of nodes whose in degree is now 0
        nodes_to_prune = [node for node, in_deg
                          in graph.in_degree()
                          if in_deg == 0 and node[1] == 1]
    return graph

def path_sign_to_signed_nodes(source, target, edge_sign):
    """Translates a signed edge or path to valid signed nodes

    Pairs with a negative source node are filtered out.

    Paramters
    ---------
    source : str|int
        The source node
    target : str|int
        The target node
    edge_sign : int
        The sign of the edge

    Returns
    -------
    sign_tuple : (a, sign), (b, sign)
        Tuple of tuples of the valid combination of signed nodes
    """
    # Sign definitions: + == 0, - == 1
    # + path -> (a+, b+)
    # - path -> (a+, b-)
    # (a-, b-) and (a-, b+) are also technically valid but not in this context
    try:
        if int(edge_sign) == 0:
            return (source, 0), (target, 0)
        else:
            return (source, 1), (target, 0)
    except ValueError:
        logger.warning('Invalid sign %s when translating edge sign to int'
                       % edge_sign)
        return (None, None), (None, None)

def signed_nodes_to_signed_edge(source, target):
    """Create the triple (node, node, sign) from a pair of signed nodes

    Assuming source, target forms an edge of signed nodes:
    edge = (a, sign), (b, sign), return the corresponding signed edge triple
    """
    # Sign definitions: + == 0, - == 1
    # + edge/path -> (a+, b+) and (a-, b-)
    # - edge/path -> (a-, b+) and (a+, b-)
    source_name, source_sign = source
    target_name, target_sign = target
    try:
        if int(source_sign) == int(target_sign):
            return source_name, target_name, 0
        else:
            return source_name, target_name, 1
    except ValueError:
        logger.warning('Error translating signed nodes to signed edge using '
                       '(%s, %s)' % (source, target))
        return None, None, None<|endoftext|>