<commit_before>import logging
import tempfile
import threading

import os
import copy
import uuid
import datetime
import time
from abc import ABCMeta, abstractmethod
import prov.model as prov
from typing import Dict, Text, Any, Tuple, Set, List

from .builder import Builder
from .errors import WorkflowException
from .mutation import MutationManager
from .job import JobBase
from .process import relocateOutputs, cleanIntermediate, Process, shortname, uniquename, get_overrides
from . import loghandler
from schema_salad.sourceline import SourceLine

_logger = logging.getLogger("cwltool")

class JobExecutor(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        # type: (...) -> None
        self.final_output = []  # type: List
        self.final_status = []  # type: List
        self.output_dirs = set()  # type: Set

    def __call__(self, *args, **kwargs):
        return self.execute(*args, **kwargs)

    def output_callback(self, out, processStatus):
        self.final_status.append(processStatus)
        self.final_output.append(out)

    @abstractmethod
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 provDoc,
                 logger,
                 **kwargs  # type: Any
                 ):
        pass

    def execute(self, t,  # type: Process
                job_order_object,  # type: Dict[Text, Any]
                logger=_logger,
                provDoc=None,
                engineID=None,
                WorkflowID=None,
                **kwargs  # type: Any
                ):
        # type: (...) -> Tuple[Dict[Text, Any], Text]

        if "basedir" not in kwargs:
            raise WorkflowException("Must provide 'basedir' in kwargs")

        finaloutdir = os.path.abspath(kwargs.get("outdir")) if kwargs.get("outdir") else None
        kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs["tmp_outdir_prefix"]) if kwargs.get(
            "tmp_outdir_prefix") else tempfile.mkdtemp()
        self.output_dirs.add(kwargs["outdir"])
        kwargs["mutation_manager"] = MutationManager()

        jobReqs = None
        if "cwl:requirements" in job_order_object:
            jobReqs = job_order_object["cwl:requirements"]
        elif ("cwl:defaults" in t.metadata and "cwl:requirements" in t.metadata["cwl:defaults"]):
            jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"]
        if jobReqs:
            for req in jobReqs:
                t.requirements.append(req)

        self.run_jobs(t, job_order_object, provDoc, engineID, WorkflowID, logger, **kwargs)

        if self.final_output and self.final_output[0] and finaloutdir:
            self.final_output[0] = relocateOutputs(self.final_output[0], finaloutdir,
                                                   self.output_dirs, kwargs.get("move_outputs"),
                                                   kwargs["make_fs_access"](""),
                                                   kwargs["compute_checksum"])

        if kwargs.get("rm_tmpdir"):
            cleanIntermediate(self.output_dirs)

        if self.final_output and self.final_status:
            return (self.final_output[0], self.final_status[0])
        else:
            return (None, "permanentFail")

class SingleJobExecutor(JobExecutor):
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 document,
                 engineUUID,
                 WorkflowRunID,
                 logger,
                 **kwargs  # type: Any
                 ):
        reference_locations={}
        ProvActivity_dict={}
        jobiter = t.job(job_order_object,
                        self.output_callback,
                        **kwargs)
        try:
            ro = kwargs.get("ro")
            for r in jobiter:
                if r:
                    builder = kwargs.get("builder", None)  # type: Builder

                    if builder is not None:
                        r.builder = builder
                    if r.outdir:
                        self.output_dirs.add(r.outdir)
                    if ro:
                        #here we are recording provenance of each subprocess of the workflow
                        if ".cwl" in getattr(r, "name"): #for prospective provenance
                            steps=[]
                            for s in r.steps:
                                stepname="wf:main/"+str(s.name)[5:]
                                steps.append(stepname)
                                print("step name is: ", stepname)
                                document.entity(stepname, {prov.PROV_TYPE: "wfdesc:Process", "prov:type": "prov:Plan"})
                            #create prospective provenance recording for the workflow
                            document.entity("wf:main", {prov.PROV_TYPE: "wfdesc:Process", "prov:type": "prov:Plan", "wfdesc:hasSubProcess=":str(steps),  "prov:label":"Prospective provenance"})
                            customised_job={} #new job object for RO
                            for e, i in enumerate(r.tool["inputs"]):
                                with SourceLine(r.tool["inputs"], e, WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
                                    iid = shortname(i["id"])
                                    if iid in job_order_object:
                                        customised_job[iid]= copy.deepcopy(job_order_object[iid]) #add the input element in dictionary for provenance
                                    elif "default" in i:
                                        customised_job[iid]= copy.deepcopy(i["default"]) #add the defualt elements in the dictionary for provenance
                                    else:
                                        raise WorkflowException(
                                            u"Input '%s' not in input object and does not have a default value." % (i["id"]))
                            ##create master-job.json and returns a dictionary with workflow level identifiers as keys and locations or actual values of the attributes as values.
                            relativised_input_object=ro.create_job(customised_job, kwargs) #call the method to generate a file with customised job
                            for key, value in relativised_input_object.items():
                                strvalue=str(value)
                                if "data" in strvalue:
                                    shahash="data:"+value.split("/")[-1]
                                    rel_path=value[3:]
                                    reference_locations[job_order_object[key]["location"]]=relativised_input_object[key][11:]
                                    document.entity(shahash, {prov.PROV_TYPE:"wfprov:Artifact"})
                                    #document.specializationOf(rel_path, shahash) NOTE:THIS NEEDS FIXING as it required both params as entities.
                                else:
                                    ArtefactValue="data:"+strvalue
                                    document.entity(ArtefactValue, {prov.PROV_TYPE:"wfprov:Artifact"})
                    if ".cwl" not in getattr(r, "name"):
                        if ro:
                            ProcessRunID="run:"+str(uuid.uuid4())
                            #each subprocess is defined as an activity()
                            provLabel="Run of workflow/packed.cwl#main/"+str(r.name)
                            ProcessProvActivity = document.activity(ProcessRunID, None, None, {prov.PROV_TYPE: "wfprov:ProcessRun", "prov:label": provLabel})
                            if hasattr(r, 'name') and ".cwl" not in getattr(r, "name"):
                                document.wasAssociatedWith(ProcessRunID, engineUUID, str("wf:main/"+r.name))
                            document.wasStartedBy(ProcessRunID, None, WorkflowRunID, datetime.datetime.now(), None, None)
                            #this is where you run each step. so start and end time for the step
                            r.run(document, WorkflowRunID, ProcessProvActivity, reference_locations, **kwargs)
                        else:
                            r.run(**kwargs)
                        #capture workflow level outputs in the prov doc
                    if ro:
                        for eachOutput in self.final_output:
                            for key, value in eachOutput.items():
                                outputProvRole="wf:main"+"/"+str(key)
                                output_checksum="data:"+str(value["checksum"][5:])
                                document.entity(output_checksum, {prov.PROV_TYPE:"wfprov:Artifact"})
                                document.wasGeneratedBy(output_checksum, WorkflowRunID, datetime.datetime.now(), None, {"prov:role":outputProvRole })
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break
        except WorkflowException:
            raise
        except Exception as e:
            logger.exception("Got workflow error")
            raise WorkflowException(Text(e))

class MultithreadedJobExecutor(JobExecutor):
    def __init__(self):
        super(MultithreadedJobExecutor, self).__init__()
        self.threads = set()
        self.exceptions = []

    def run_job(self,
                job, # type:  JobBase
                **kwargs  # type:  Any
                ):
        # type: (...) -> None
        def runner():
            try:
                job.run(**kwargs)
            except WorkflowException as e:
                self.exceptions.append(e)
            except Exception as e:
                self.exceptions.append(WorkflowException(Text(e)))

            self.threads.remove(thread)

        thread = threading.Thread(target=runner)
        thread.daemon = True
        self.threads.add(thread)
        thread.start()

    def wait_for_next_completion(self):  # type: () -> None
        if self.exceptions:
            raise self.exceptions[0]

    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 logger,
                 **kwargs  # type: Any
                 ):

        jobiter = t.job(job_order_object, self.output_callback, **kwargs)

        for r in jobiter:
            if r:
                builder = kwargs.get("builder", None)  # type: Builder
                if builder is not None:
                    r.builder = builder
                if r.outdir:
                    self.output_dirs.add(r.outdir)
                self.run_job(r, **kwargs)
            else:
                if len(self.threads):
                    self.wait_for_next_completion()
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break

        while len(self.threads) > 0:
            self.wait_for_next_completion()
<commit_msg>fixed run_jobs() call<commit_after>import logging
import tempfile
import threading

import os
import copy
import uuid
import datetime
import time
from abc import ABCMeta, abstractmethod
import prov.model as prov
from typing import Dict, Text, Any, Tuple, Set, List

from .builder import Builder
from .errors import WorkflowException
from .mutation import MutationManager
from .job import JobBase
from .process import relocateOutputs, cleanIntermediate, Process, shortname, uniquename, get_overrides
from . import loghandler
from schema_salad.sourceline import SourceLine

_logger = logging.getLogger("cwltool")

class JobExecutor(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        # type: (...) -> None
        self.final_output = []  # type: List
        self.final_status = []  # type: List
        self.output_dirs = set()  # type: Set

    def __call__(self, *args, **kwargs):
        return self.execute(*args, **kwargs)

    def output_callback(self, out, processStatus):
        self.final_status.append(processStatus)
        self.final_output.append(out)

    @abstractmethod
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 document,
                 engineUUID,
                 WorkflowRunID,
                 logger,
                 **kwargs  # type: Any
                 ):
        pass

    def execute(self, t,  # type: Process
                job_order_object,  # type: Dict[Text, Any]
                logger=_logger,
                provDoc=None,
                engineID=None,
                WorkflowID=None,
                **kwargs  # type: Any
                ):
        # type: (...) -> Tuple[Dict[Text, Any], Text]

        if "basedir" not in kwargs:
            raise WorkflowException("Must provide 'basedir' in kwargs")

        finaloutdir = os.path.abspath(kwargs.get("outdir")) if kwargs.get("outdir") else None
        kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs["tmp_outdir_prefix"]) if kwargs.get(
            "tmp_outdir_prefix") else tempfile.mkdtemp()
        self.output_dirs.add(kwargs["outdir"])
        kwargs["mutation_manager"] = MutationManager()

        jobReqs = None
        if "cwl:requirements" in job_order_object:
            jobReqs = job_order_object["cwl:requirements"]
        elif ("cwl:defaults" in t.metadata and "cwl:requirements" in t.metadata["cwl:defaults"]):
            jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"]
        if jobReqs:
            for req in jobReqs:
                t.requirements.append(req)

        self.run_jobs(t, job_order_object, provDoc, engineID, WorkflowID, logger, **kwargs)

        if self.final_output and self.final_output[0] and finaloutdir:
            self.final_output[0] = relocateOutputs(self.final_output[0], finaloutdir,
                                                   self.output_dirs, kwargs.get("move_outputs"),
                                                   kwargs["make_fs_access"](""),
                                                   kwargs["compute_checksum"])

        if kwargs.get("rm_tmpdir"):
            cleanIntermediate(self.output_dirs)

        if self.final_output and self.final_status:
            return (self.final_output[0], self.final_status[0])
        else:
            return (None, "permanentFail")

class SingleJobExecutor(JobExecutor):
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 document,
                 engineUUID,
                 WorkflowRunID,
                 logger,
                 **kwargs  # type: Any
                 ):
        reference_locations={}
        ProvActivity_dict={}
        jobiter = t.job(job_order_object,
                        self.output_callback,
                        **kwargs)
        try:
            ro = kwargs.get("ro")
            for r in jobiter:
                if r:
                    builder = kwargs.get("builder", None)  # type: Builder

                    if builder is not None:
                        r.builder = builder
                    if r.outdir:
                        self.output_dirs.add(r.outdir)
                    if ro:
                        #here we are recording provenance of each subprocess of the workflow
                        if ".cwl" in getattr(r, "name"): #for prospective provenance
                            steps=[]
                            for s in r.steps:
                                stepname="wf:main/"+str(s.name)[5:]
                                steps.append(stepname)
                                print("step name is: ", stepname)
                                document.entity(stepname, {prov.PROV_TYPE: "wfdesc:Process", "prov:type": "prov:Plan"})
                            #create prospective provenance recording for the workflow
                            document.entity("wf:main", {prov.PROV_TYPE: "wfdesc:Process", "prov:type": "prov:Plan", "wfdesc:hasSubProcess=":str(steps),  "prov:label":"Prospective provenance"})
                            customised_job={} #new job object for RO
                            for e, i in enumerate(r.tool["inputs"]):
                                with SourceLine(r.tool["inputs"], e, WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
                                    iid = shortname(i["id"])
                                    if iid in job_order_object:
                                        customised_job[iid]= copy.deepcopy(job_order_object[iid]) #add the input element in dictionary for provenance
                                    elif "default" in i:
                                        customised_job[iid]= copy.deepcopy(i["default"]) #add the defualt elements in the dictionary for provenance
                                    else:
                                        raise WorkflowException(
                                            u"Input '%s' not in input object and does not have a default value." % (i["id"]))
                            ##create master-job.json and returns a dictionary with workflow level identifiers as keys and locations or actual values of the attributes as values.
                            relativised_input_object=ro.create_job(customised_job, kwargs) #call the method to generate a file with customised job
                            for key, value in relativised_input_object.items():
                                strvalue=str(value)
                                if "data" in strvalue:
                                    shahash="data:"+value.split("/")[-1]
                                    rel_path=value[3:]
                                    reference_locations[job_order_object[key]["location"]]=relativised_input_object[key][11:]
                                    document.entity(shahash, {prov.PROV_TYPE:"wfprov:Artifact"})
                                    #document.specializationOf(rel_path, shahash) NOTE:THIS NEEDS FIXING as it required both params as entities.
                                else:
                                    ArtefactValue="data:"+strvalue
                                    document.entity(ArtefactValue, {prov.PROV_TYPE:"wfprov:Artifact"})
                    #if ".cwl" not in getattr(r, "name"):
                    else:
                        if ro:
                            ProcessRunID="run:"+str(uuid.uuid4())
                            #each subprocess is defined as an activity()
                            provLabel="Run of workflow/packed.cwl#main/"+str(r.name)
                            ProcessProvActivity = document.activity(ProcessRunID, None, None, {prov.PROV_TYPE: "wfprov:ProcessRun", "prov:label": provLabel})
                            if hasattr(r, 'name') and ".cwl" not in getattr(r, "name"):
                                document.wasAssociatedWith(ProcessRunID, engineUUID, str("wf:main/"+r.name))
                            document.wasStartedBy(ProcessRunID, None, WorkflowRunID, datetime.datetime.now(), None, None)
                            #this is where you run each step. so start and end time for the step
                            r.run(document, WorkflowRunID, ProcessProvActivity, reference_locations, **kwargs)
                        else:
                            r.run(**kwargs)
                        #capture workflow level outputs in the prov doc
                    if ro:
                        for eachOutput in self.final_output:
                            for key, value in eachOutput.items():
                                outputProvRole="wf:main"+"/"+str(key)
                                output_checksum="data:"+str(value["checksum"][5:])
                                document.entity(output_checksum, {prov.PROV_TYPE:"wfprov:Artifact"})
                                document.wasGeneratedBy(output_checksum, WorkflowRunID, datetime.datetime.now(), None, {"prov:role":outputProvRole })
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break
        except WorkflowException:
            raise
        except Exception as e:
            logger.exception("Got workflow error")
            raise WorkflowException(Text(e))

class MultithreadedJobExecutor(JobExecutor):
    def __init__(self):
        super(MultithreadedJobExecutor, self).__init__()
        self.threads = set()
        self.exceptions = []

    def run_job(self,
                job, # type:  JobBase
                **kwargs  # type:  Any
                ):
        # type: (...) -> None
        def runner():
            try:
                job.run(**kwargs)
            except WorkflowException as e:
                self.exceptions.append(e)
            except Exception as e:
                self.exceptions.append(WorkflowException(Text(e)))

            self.threads.remove(thread)

        thread = threading.Thread(target=runner)
        thread.daemon = True
        self.threads.add(thread)
        thread.start()

    def wait_for_next_completion(self):  # type: () -> None
        if self.exceptions:
            raise self.exceptions[0]

    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 document,
                 engineUUID,
                 WorkflowRunID,
                 logger,
                 **kwargs  # type: Any
                 ):

        jobiter = t.job(job_order_object, self.output_callback, **kwargs)

        for r in jobiter:
            if r:
                builder = kwargs.get("builder", None)  # type: Builder
                if builder is not None:
                    r.builder = builder
                if r.outdir:
                    self.output_dirs.add(r.outdir)
                self.run_job(r, **kwargs)
            else:
                if len(self.threads):
                    self.wait_for_next_completion()
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break

        while len(self.threads) > 0:
            self.wait_for_next_completion()
<|endoftext|>


<commit_before>import logging
import tempfile
import threading

import os
import copy
import uuid
import datetime
import time
from abc import ABCMeta, abstractmethod
import prov.model as prov
from typing import Dict, Text, Any, Tuple, Set, List
from schema_salad.validate import ValidationException

from .builder import Builder
from .errors import WorkflowException
from .mutation import MutationManager
from .job import JobBase
from .process import relocateOutputs, cleanIntermediate, Process, shortname, uniquename, get_overrides
from . import loghandler
from schema_salad.sourceline import SourceLine

_logger = logging.getLogger("cwltool")

class JobExecutor(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        # type: (...) -> None
        self.final_output = []  # type: List
        self.final_status = []  # type: List
        self.output_dirs = set()  # type: Set

    def __call__(self, *args, **kwargs):
        return self.execute(*args, **kwargs)

    def output_callback(self, out, processStatus):
        self.final_status.append(processStatus)
        self.final_output.append(out)

    @abstractmethod
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):
        pass

    def execute(self,
                t,                 # type: Process
                job_order_object,  # type: Dict[Text, Any]
                engineID=None,
                logger=None,
                makeTool=None,
                select_resources=None,
                provDoc=None,
                WorkflowID=None,
                make_fs_access=None,
                secret_store=None,
                logger=_logger,
                **kwargs           # type: Any
                ):  # type: (...) -> Tuple[Dict[Text, Any], Text]

        if "basedir" not in kwargs:
            raise WorkflowException("Must provide 'basedir' in kwargs")
        finaloutdir = os.path.abspath(kwargs.get("outdir")) if kwargs.get("outdir") else None
        kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs.get("tmp_outdir_prefix"))
        self.output_dirs.add(kwargs["outdir"])
        kwargs["mutation_manager"] = MutationManager()
        kwargs["toplevel"] = True

        jobReqs = None
        if "cwl:requirements" in job_order_object:
            jobReqs = job_order_object["cwl:requirements"]
        elif ("cwl:defaults" in t.metadata and "cwl:requirements" in t.metadata["cwl:defaults"]):
            jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"]
        if jobReqs:
            for req in jobReqs:
                t.requirements.append(req)
        self.run_jobs(t, job_order_object, engineID, provDoc, WorkflowID, logger, make_fs_access, **kwargs)

        if self.final_output and self.final_output[0] and finaloutdir:
            self.final_output[0] = relocateOutputs(
                self.final_output[0], finaloutdir, self.output_dirs,
                kwargs.get("move_outputs"), make_fs_access(""),
                kwargs.get("compute_checksum", True))

        if kwargs.get("rm_tmpdir"):
            cleanIntermediate(self.output_dirs)

        if self.final_output and self.final_status:
            return (self.final_output[0], self.final_status[0])
        else:
            return (None, "permanentFail")

class SingleJobExecutor(JobExecutor):
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):
        reference_locations={} # type: Dict[Text, Any]
        ProcessProvActivity=None
        jobiter = t.job(job_order_object,
                        self.output_callback,
                        **kwargs)
        try:
            research_obj = kwargs.get("research_obj")
            for r in jobiter:
                if r:
                    builder = kwargs.get("builder", None)  # type: Builder

                    if builder is not None:
                        r.builder = builder
                    if r.outdir:
                        self.output_dirs.add(r.outdir)
                    if research_obj:
                        if not hasattr(t, "steps"): #record provenance of an independent commandline tool execution
                            research_obj.prospective_prov(document, r)
                            customised_job=research_obj.copy_job_order(r, job_order_object)
                            relativised_input_object, reference_locations =research_obj.create_job(customised_job, make_fs_access, kwargs)
                            research_obj.declare_artefact(relativised_input_object, document, job_order_object)
                            ProcessProvActivity = research_obj.startProcess(r, document, engUUID)
                        elif hasattr(r, "workflow"): #record provenance for the workflow execution
                            research_obj.prospective_prov(document, r)
                            customised_job=research_obj.copy_job_order(r, job_order_object)
                            relativised_input_object, reference_locations =research_obj.create_job(customised_job, make_fs_access, kwargs)
                            research_obj.declare_artefact(relativised_input_object, document, job_order_object)
                        else: #in case of commandline tool execution as part of workflow
                            ProcessProvActivity = research_obj.startProcess(r, document, engUUID, WorkflowRunID)
                        r.run(document, WorkflowRunID, ProcessProvActivity, reference_locations, **kwargs)
                        #capture workflow level outputs in the prov doc
                        if self.final_output:
                            ProcessRunID=None
                            research_obj.generate_outputProv(self.final_output[0], document, WorkflowRunID, ProcessRunID)
                    else:
                        r.run(**kwargs)
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break
        except (ValidationException, WorkflowException):
            raise
        except Exception as e:
            logger.exception("Got workflow error")
            raise WorkflowException(Text(e))

class MultithreadedJobExecutor(JobExecutor):
    def __init__(self):
        super(MultithreadedJobExecutor, self).__init__()
        self.threads = set()
        self.exceptions = []

    def run_job(self,
                job,      # type: JobBase
                **kwargs  # type: Any
                ):
        # type: (...) -> None
        def runner():
            try:
                job.run(**kwargs)
            except WorkflowException as e:
                self.exceptions.append(e)
            except Exception as e:
                self.exceptions.append(WorkflowException(Text(e)))

            self.threads.remove(thread)

        thread = threading.Thread(target=runner)
        thread.daemon = True
        self.threads.add(thread)
        thread.start()

    def wait_for_next_completion(self):  # type: () -> None
        if self.exceptions:
            raise self.exceptions[0]

    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):

        jobiter = t.job(job_order_object, self.output_callback, **kwargs)

        for r in jobiter:
            if r:
                builder = kwargs.get("builder", None)  # type: Builder
                if builder is not None:
                    r.builder = builder
                if r.outdir:
                    self.output_dirs.add(r.outdir)
                self.run_job(r, **kwargs)
            else:
                if len(self.threads):
                    self.wait_for_next_completion()
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break

        while len(self.threads) > 0:
            self.wait_for_next_completion()<commit_msg>remove duplicate logger arg<commit_after>import logging
import tempfile
import threading

import os
import copy
import uuid
import datetime
import time
from abc import ABCMeta, abstractmethod
import prov.model as prov
from typing import Dict, Text, Any, Tuple, Set, List
from schema_salad.validate import ValidationException

from .builder import Builder
from .errors import WorkflowException
from .mutation import MutationManager
from .job import JobBase
from .process import relocateOutputs, cleanIntermediate, Process, shortname, uniquename, get_overrides
from . import loghandler
from schema_salad.sourceline import SourceLine

_logger = logging.getLogger("cwltool")

class JobExecutor(object):
    __metaclass__ = ABCMeta

    def __init__(self):
        # type: (...) -> None
        self.final_output = []  # type: List
        self.final_status = []  # type: List
        self.output_dirs = set()  # type: Set

    def __call__(self, *args, **kwargs):
        return self.execute(*args, **kwargs)

    def output_callback(self, out, processStatus):
        self.final_status.append(processStatus)
        self.final_output.append(out)

    @abstractmethod
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):
        pass

    def execute(self,
                t,                 # type: Process
                job_order_object,  # type: Dict[Text, Any]
                engineID=None,
                makeTool=None,
                select_resources=None,
                provDoc=None,
                WorkflowID=None,
                make_fs_access=None,
                secret_store=None,
                logger=_logger,
                **kwargs           # type: Any
                ):  # type: (...) -> Tuple[Dict[Text, Any], Text]

        if "basedir" not in kwargs:
            raise WorkflowException("Must provide 'basedir' in kwargs")
        finaloutdir = os.path.abspath(kwargs.get("outdir")) if kwargs.get("outdir") else None
        kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs.get("tmp_outdir_prefix"))
        self.output_dirs.add(kwargs["outdir"])
        kwargs["mutation_manager"] = MutationManager()
        kwargs["toplevel"] = True

        jobReqs = None
        if "cwl:requirements" in job_order_object:
            jobReqs = job_order_object["cwl:requirements"]
        elif ("cwl:defaults" in t.metadata and "cwl:requirements" in t.metadata["cwl:defaults"]):
            jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"]
        if jobReqs:
            for req in jobReqs:
                t.requirements.append(req)
        self.run_jobs(t, job_order_object, engineID, provDoc, WorkflowID, logger, make_fs_access, **kwargs)

        if self.final_output and self.final_output[0] and finaloutdir:
            self.final_output[0] = relocateOutputs(
                self.final_output[0], finaloutdir, self.output_dirs,
                kwargs.get("move_outputs"), make_fs_access(""),
                kwargs.get("compute_checksum", True))

        if kwargs.get("rm_tmpdir"):
            cleanIntermediate(self.output_dirs)

        if self.final_output and self.final_status:
            return (self.final_output[0], self.final_status[0])
        else:
            return (None, "permanentFail")

class SingleJobExecutor(JobExecutor):
    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):
        reference_locations={} # type: Dict[Text, Any]
        ProcessProvActivity=None
        jobiter = t.job(job_order_object,
                        self.output_callback,
                        **kwargs)
        try:
            research_obj = kwargs.get("research_obj")
            for r in jobiter:
                if r:
                    builder = kwargs.get("builder", None)  # type: Builder

                    if builder is not None:
                        r.builder = builder
                    if r.outdir:
                        self.output_dirs.add(r.outdir)
                    if research_obj:
                        if not hasattr(t, "steps"): #record provenance of an independent commandline tool execution
                            research_obj.prospective_prov(document, r)
                            customised_job=research_obj.copy_job_order(r, job_order_object)
                            relativised_input_object, reference_locations =research_obj.create_job(customised_job, make_fs_access, kwargs)
                            research_obj.declare_artefact(relativised_input_object, document, job_order_object)
                            ProcessProvActivity = research_obj.startProcess(r, document, engUUID)
                        elif hasattr(r, "workflow"): #record provenance for the workflow execution
                            research_obj.prospective_prov(document, r)
                            customised_job=research_obj.copy_job_order(r, job_order_object)
                            relativised_input_object, reference_locations =research_obj.create_job(customised_job, make_fs_access, kwargs)
                            research_obj.declare_artefact(relativised_input_object, document, job_order_object)
                        else: #in case of commandline tool execution as part of workflow
                            ProcessProvActivity = research_obj.startProcess(r, document, engUUID, WorkflowRunID)
                        r.run(document, WorkflowRunID, ProcessProvActivity, reference_locations, **kwargs)
                        #capture workflow level outputs in the prov doc
                        if self.final_output:
                            ProcessRunID=None
                            research_obj.generate_outputProv(self.final_output[0], document, WorkflowRunID, ProcessRunID)
                    else:
                        r.run(**kwargs)
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break
        except (ValidationException, WorkflowException):
            raise
        except Exception as e:
            logger.exception("Got workflow error")
            raise WorkflowException(Text(e))

class MultithreadedJobExecutor(JobExecutor):
    def __init__(self):
        super(MultithreadedJobExecutor, self).__init__()
        self.threads = set()
        self.exceptions = []

    def run_job(self,
                job,      # type: JobBase
                **kwargs  # type: Any
                ):
        # type: (...) -> None
        def runner():
            try:
                job.run(**kwargs)
            except WorkflowException as e:
                self.exceptions.append(e)
            except Exception as e:
                self.exceptions.append(WorkflowException(Text(e)))

            self.threads.remove(thread)

        thread = threading.Thread(target=runner)
        thread.daemon = True
        self.threads.add(thread)
        thread.start()

    def wait_for_next_completion(self):  # type: () -> None
        if self.exceptions:
            raise self.exceptions[0]

    def run_jobs(self,
                 t,  # type: Process
                 job_order_object,  # type: Dict[Text, Any]
                 engUUID,
                 document,
                 WorkflowRunID,
                 logger,
                 make_fs_access,
                 **kwargs  # type: Any
                 ):

        jobiter = t.job(job_order_object, self.output_callback, **kwargs)

        for r in jobiter:
            if r:
                builder = kwargs.get("builder", None)  # type: Builder
                if builder is not None:
                    r.builder = builder
                if r.outdir:
                    self.output_dirs.add(r.outdir)
                self.run_job(r, **kwargs)
            else:
                if len(self.threads):
                    self.wait_for_next_completion()
                else:
                    logger.error("Workflow cannot make any more progress.")
                    break

        while len(self.threads) > 0:
            self.wait_for_next_completion()<|endoftext|>


<commit_before>"""Loads a CWL document."""
from __future__ import absolute_import

import hashlib
import logging
import os
import re
import uuid
from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
                    Optional, Tuple, Union, cast)

import requests.sessions
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad import schema
from schema_salad.ref_resolver import (ContextType,  # pylint: disable=unused-import
                                       Fetcher, Loader, file_uri, SubLoader)
from schema_salad.sourceline import SourceLine, cmap
from schema_salad.validate import ValidationException
from six import itervalues, string_types
from six.moves import urllib
from typing_extensions import Text  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from . import process, update
from .context import LoadingContext  # pylint: disable=unused-import
from .errors import WorkflowException
from .loghandler import _logger
from .process import (Process, get_schema,  # pylint: disable=unused-import
                      shortname)
from .software_requirements import (  # pylint: disable=unused-import
    DependenciesConfiguration)
from .update import ALLUPDATES
from .utils import json_dumps

jobloaderctx = {
    u"cwl": "https://w3id.org/cwl/cwl#",
    u"cwltool": "http://commonwl.org/cwltool#",
    u"path": {u"@type": u"@id"},
    u"location": {u"@type": u"@id"},
    u"format": {u"@type": u"@id"},
    u"id": u"@id"
}  # type: ContextType

overrides_ctx = {
    u"overrideTarget": {u"@type": u"@id"},
    u"cwltool": "http://commonwl.org/cwltool#",
    u"http://commonwl.org/cwltool#overrides": {
        "@id": "cwltool:overrides",
        "mapSubject": "overrideTarget",
    },
    "requirements": {
        "@id": "https://w3id.org/cwl/cwl#requirements",
        "mapSubject": "class"
    }
}  # type: ContextType

FetcherConstructorType = Callable[
    [Dict[Text, Union[Text, bool]], requests.sessions.Session], Fetcher]
ResolverType = Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]

def default_loader(fetcher_constructor=None):
    # type: (Optional[FetcherConstructorType]) -> Loader
    return Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)

def resolve_tool_uri(argsworkflow,  # type: Text
                     resolver=None,  # type: ResolverType
                     fetcher_constructor=None,  # type: FetcherConstructorType
                     document_loader=None  # type: Loader
                    ):  # type: (...) -> Tuple[Text, Text]

    uri = None  # type: Optional[Text]
    split = urllib.parse.urlsplit(argsworkflow)
    # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that
    if split.scheme and split.scheme in [u'http', u'https', u'file']:
        uri = argsworkflow
    elif os.path.exists(os.path.abspath(argsworkflow)):
        uri = file_uri(str(os.path.abspath(argsworkflow)))
    elif resolver is not None:
        if document_loader is None:
            document_loader = default_loader(fetcher_constructor)  # type: ignore
        uri = resolver(document_loader, argsworkflow)

    if uri is None:
        raise ValidationException("Not found: '%s'" % argsworkflow)

    if argsworkflow != uri:
        _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)

    fileuri = urllib.parse.urldefrag(uri)[0]
    return uri, fileuri

def fetch_document(argsworkflow,        # type: Union[Text, Dict[Text, Any]]
                   loadingContext=None  # type: Optional[LoadingContext]
                  ):  # type: (...) -> Tuple[LoadingContext, CommentedMap, Text]
    """Retrieve a CWL document."""

    if loadingContext is None:
        loadingContext = LoadingContext()
        loadingContext.loader = default_loader()
    else:
        loadingContext = loadingContext.copy()
        if loadingContext.loader is None:
            loadingContext.loader = default_loader(loadingContext.fetcher_constructor)

    if isinstance(argsworkflow, string_types):
        uri, fileuri = resolve_tool_uri(argsworkflow,
                                        resolver=loadingContext.resolver,
                                        document_loader=loadingContext.loader)
        workflowobj = loadingContext.loader.fetch(fileuri)
        return loadingContext, workflowobj, uri
    if isinstance(argsworkflow, dict):
        uri = argsworkflow["id"] if argsworkflow.get("id") else "_:" + Text(uuid.uuid4())
        workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri))
        loadingContext.loader.idx[uri] = workflowobj
        return loadingContext, workflowobj, uri
    raise ValidationException("Must be URI or object: '%s'" % argsworkflow)

def _convert_stdstreams_to_files(workflowobj):
    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None

    if isinstance(workflowobj, MutableMapping):
        if workflowobj.get('class') == 'CommandLineTool':
            with SourceLine(workflowobj, "outputs", ValidationException,
                            _logger.isEnabledFor(logging.DEBUG)):
                outputs = workflowobj.get('outputs', [])
                if not isinstance(outputs, CommentedSeq):
                    raise ValidationException('"outputs" section is not '
                                              'valid.')
                for out in workflowobj.get('outputs', []):
                    if not isinstance(out, CommentedMap):
                        raise ValidationException(
                            "Output '{}' is not a valid "
                            "OutputParameter.".format(out))
                    for streamtype in ['stdout', 'stderr']:
                        if out.get('type') == streamtype:
                            if 'outputBinding' in out:
                                raise ValidationException(
                                    "Not allowed to specify outputBinding when"
                                    " using %s shortcut." % streamtype)
                            if streamtype in workflowobj:
                                filename = workflowobj[streamtype]
                            else:
                                filename = Text(
                                    hashlib.sha1(  # nosec
                                        json_dumps(workflowobj, sort_keys=True
                                                  ).encode('utf-8')).hexdigest())
                                workflowobj[streamtype] = filename
                            out['type'] = 'File'
                            out['outputBinding'] = cmap({'glob': filename})
            for inp in workflowobj.get('inputs', []):
                if inp.get('type') == 'stdin':
                    if 'inputBinding' in inp:
                        raise ValidationException(
                            "Not allowed to specify inputBinding when"
                            " using stdin shortcut.")
                    if 'stdin' in workflowobj:
                        raise ValidationException(
                            "Not allowed to specify stdin path when"
                            " using stdin type shortcut.")
                    else:
                        workflowobj['stdin'] = \
                            "$(inputs.%s.path)" % \
                            inp['id'].rpartition('#')[2]
                        inp['type'] = 'File'
        else:
            for entry in itervalues(workflowobj):
                _convert_stdstreams_to_files(entry)
    if isinstance(workflowobj, MutableSequence):
        for entry in workflowobj:
            _convert_stdstreams_to_files(entry)

def _add_blank_ids(workflowobj):
    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None

    if isinstance(workflowobj, MutableMapping):
        if ("run" in workflowobj and
                isinstance(workflowobj["run"], MutableMapping) and
                "id" not in workflowobj["run"] and
                "$import" not in workflowobj["run"]):
            workflowobj["run"]["id"] = Text(uuid.uuid4())
        for entry in itervalues(workflowobj):
            _add_blank_ids(entry)
    if isinstance(workflowobj, MutableSequence):
        for entry in workflowobj:
            _add_blank_ids(entry)

def resolve_and_validate_document(loadingContext,
                      workflowobj,
                      uri,
                      preprocess_only=False,     # type: bool
                      skip_schemas=None,         # type: bool
                     ):
    # type: (...) -> Tuple[LoadingContext, Text]
    """Validate a CWL document."""

    loadingContext = loadingContext.copy()

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate)
        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        workflowobj = fetch_document(uri, loadingContext)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    cwlVersion = loadingContext.metadata.get("cwlVersion")
    if not cwlVersion:
        cwlVersion = workflowobj.get("cwlVersion")
    if not cwlVersion and fileuri != uri:
        # The tool we're loading is a fragment of a bigger file.  Get
        # the document root element and look for cwlVersion there
        metadata = fetch_document(fileuri, loadingContext)[1]
        cwlVersion = metadata.get("cwlVersion")
    if not cwlVersion:
        raise ValidationException(
            "No cwlVersion found. "
            "Use the following syntax in your CWL document to declare "
            "the version: cwlVersion: <version>.\n"
            "Note: if this is a CWL draft-2 (pre v1.0) document then it "
            "will need to be upgraded first.")

    if not isinstance(cwlVersion, string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(cwlVersion)))
    # strip out version
    cwlVersion = re.sub(
        r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
        cwlVersion)
    if cwlVersion not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
        loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri))
        del jobobj["http://commonwl.org/cwltool#overrides"]

    if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
        if cwlVersion not in ("v1.1.0-dev1",):
            raise ValidationException(
                    "`cwl:requirements` in the input object is not part of CWL "
                    "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                    "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
                    "--enable-dev.")
        loadingContext.overrides_list.append({"overrideTarget": uri,
                                              "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]})
        del jobobj["https://w3id.org/cwl/cwl#requirements"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(cwlVersion)[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=loadingContext.loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=loadingContext.fetcher_constructor,
                             skip_schemas=skip_schemas)

    if cwlVersion == "v1.0":
        _add_blank_ids(workflowobj)

    processobj, metadata = document_loader.resolve_all(workflowobj, fileuri)
    if loadingContext.metadata:
        metadata = loadingContext.metadata
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
    if not isinstance(metadata, CommentedMap):
        raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return loadingContext, uri

    if loadingContext.do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)

    # None means default behavior (do update)
    if loadingContext.do_update in (True, None):
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, loadingContext.enable_dev, metadata)))
        if isinstance(processobj, MutableMapping):
            document_loader.idx[processobj["id"]] = processobj
        elif isinstance(processobj, MutableSequence):
            document_loader.idx[metadata["id"]] = metadata
            for po in processobj:
                document_loader.idx[po["id"]] = po
        else:
            raise Exception("Whoops processobj was not MutableMapping or MutableSequence %s", type(processobj))

    if jobobj is not None:
        loadingContext.jobdefaults = jobobj

    loadingContext.loader = document_loader
    loadingContext.avsc_names = avsc_names
    loadingContext.metadata = metadata

    return loadingContext, uri

def make_tool(uri,                # type: Union[Text, CommentedMap, CommentedSeq]
              loadingContext      # type: LoadingContext
             ):  # type: (...) -> Process
    """Make a Python CWL object."""
    if loadingContext.loader is None:
        raise ValueError("loadingContext must have a loader")
    resolveduri, metadata = loadingContext.loader.resolve_ref(uri)

    processobj = None
    if isinstance(resolveduri, MutableSequence):
        for obj in resolveduri:
            if obj['id'].endswith('#main'):
                processobj = obj
                break
        if not processobj:
            raise WorkflowException(
                u"Tool file contains graph of multiple objects, must specify "
                "one of #%s" % ", #".join(
                    urllib.parse.urldefrag(i["id"])[1] for i in resolveduri
                    if "id" in i))
    elif isinstance(resolveduri, MutableMapping):
        processobj = resolveduri
    else:
        raise Exception("Must resolve to list or dict")

    tool = loadingContext.construct_tool_object(processobj, loadingContext)

    if loadingContext.jobdefaults:
        jobobj = loadingContext.jobdefaults
        for inp in tool.tool["inputs"]:
            if shortname(inp["id"]) in jobobj:
                inp["default"] = jobobj[shortname(inp["id"])]

    return tool

def load_tool(argsworkflow,         # type: Union[Text, Dict[Text, Any]]
              loadingContext=None   # type: Optional[LoadingContext]
             ):  # type: (...) -> Process

    loadingContext, workflowobj, uri = fetch_document(
        argsworkflow, loadingContext)

    loadingContext, uri = resolve_and_validate_document(
        loadingContext, workflowobj, uri)

    return make_tool(uri,
                     loadingContext)

def resolve_overrides(ov,      # Type: CommentedMap
                      ov_uri,  # Type: Text
                      baseurl  # type: Text
                     ):  # type: (...) -> List[Dict[Text, Any]]
    ovloader = Loader(overrides_ctx)
    ret, _ = ovloader.resolve_all(ov, baseurl)
    if not isinstance(ret, CommentedMap):
        raise Exception("Expected CommentedMap, got %s" % type(ret))
    cwl_docloader = get_schema("v1.0")[0]
    cwl_docloader.resolve_all(ret, ov_uri)
    return ret["http://commonwl.org/cwltool#overrides"]

def load_overrides(ov, base_url):  # type: (Text, Text) -> List[Dict[Text, Any]]
    ovloader = Loader(overrides_ctx)
    return resolve_overrides(ovloader.fetch(ov), ov, base_url)
<commit_msg>resolve_all() base_url should be uri not fileuri<commit_after>"""Loads a CWL document."""
from __future__ import absolute_import

import hashlib
import logging
import os
import re
import uuid
from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
                    Optional, Tuple, Union, cast)

import requests.sessions
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from schema_salad import schema
from schema_salad.ref_resolver import (ContextType,  # pylint: disable=unused-import
                                       Fetcher, Loader, file_uri, SubLoader)
from schema_salad.sourceline import SourceLine, cmap
from schema_salad.validate import ValidationException
from six import itervalues, string_types
from six.moves import urllib
from typing_extensions import Text  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from . import process, update
from .context import LoadingContext  # pylint: disable=unused-import
from .errors import WorkflowException
from .loghandler import _logger
from .process import (Process, get_schema,  # pylint: disable=unused-import
                      shortname)
from .software_requirements import (  # pylint: disable=unused-import
    DependenciesConfiguration)
from .update import ALLUPDATES
from .utils import json_dumps

jobloaderctx = {
    u"cwl": "https://w3id.org/cwl/cwl#",
    u"cwltool": "http://commonwl.org/cwltool#",
    u"path": {u"@type": u"@id"},
    u"location": {u"@type": u"@id"},
    u"format": {u"@type": u"@id"},
    u"id": u"@id"
}  # type: ContextType

overrides_ctx = {
    u"overrideTarget": {u"@type": u"@id"},
    u"cwltool": "http://commonwl.org/cwltool#",
    u"http://commonwl.org/cwltool#overrides": {
        "@id": "cwltool:overrides",
        "mapSubject": "overrideTarget",
    },
    "requirements": {
        "@id": "https://w3id.org/cwl/cwl#requirements",
        "mapSubject": "class"
    }
}  # type: ContextType

FetcherConstructorType = Callable[
    [Dict[Text, Union[Text, bool]], requests.sessions.Session], Fetcher]
ResolverType = Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]

def default_loader(fetcher_constructor=None):
    # type: (Optional[FetcherConstructorType]) -> Loader
    return Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)

def resolve_tool_uri(argsworkflow,  # type: Text
                     resolver=None,  # type: ResolverType
                     fetcher_constructor=None,  # type: FetcherConstructorType
                     document_loader=None  # type: Loader
                    ):  # type: (...) -> Tuple[Text, Text]

    uri = None  # type: Optional[Text]
    split = urllib.parse.urlsplit(argsworkflow)
    # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that
    if split.scheme and split.scheme in [u'http', u'https', u'file']:
        uri = argsworkflow
    elif os.path.exists(os.path.abspath(argsworkflow)):
        uri = file_uri(str(os.path.abspath(argsworkflow)))
    elif resolver is not None:
        if document_loader is None:
            document_loader = default_loader(fetcher_constructor)  # type: ignore
        uri = resolver(document_loader, argsworkflow)

    if uri is None:
        raise ValidationException("Not found: '%s'" % argsworkflow)

    if argsworkflow != uri:
        _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)

    fileuri = urllib.parse.urldefrag(uri)[0]
    return uri, fileuri

def fetch_document(argsworkflow,        # type: Union[Text, Dict[Text, Any]]
                   loadingContext=None  # type: Optional[LoadingContext]
                  ):  # type: (...) -> Tuple[LoadingContext, CommentedMap, Text]
    """Retrieve a CWL document."""

    if loadingContext is None:
        loadingContext = LoadingContext()
        loadingContext.loader = default_loader()
    else:
        loadingContext = loadingContext.copy()
        if loadingContext.loader is None:
            loadingContext.loader = default_loader(loadingContext.fetcher_constructor)

    if isinstance(argsworkflow, string_types):
        uri, fileuri = resolve_tool_uri(argsworkflow,
                                        resolver=loadingContext.resolver,
                                        document_loader=loadingContext.loader)
        workflowobj = loadingContext.loader.fetch(fileuri)
        return loadingContext, workflowobj, uri
    if isinstance(argsworkflow, dict):
        uri = argsworkflow["id"] if argsworkflow.get("id") else "_:" + Text(uuid.uuid4())
        workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri))
        loadingContext.loader.idx[uri] = workflowobj
        return loadingContext, workflowobj, uri
    raise ValidationException("Must be URI or object: '%s'" % argsworkflow)

def _convert_stdstreams_to_files(workflowobj):
    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None

    if isinstance(workflowobj, MutableMapping):
        if workflowobj.get('class') == 'CommandLineTool':
            with SourceLine(workflowobj, "outputs", ValidationException,
                            _logger.isEnabledFor(logging.DEBUG)):
                outputs = workflowobj.get('outputs', [])
                if not isinstance(outputs, CommentedSeq):
                    raise ValidationException('"outputs" section is not '
                                              'valid.')
                for out in workflowobj.get('outputs', []):
                    if not isinstance(out, CommentedMap):
                        raise ValidationException(
                            "Output '{}' is not a valid "
                            "OutputParameter.".format(out))
                    for streamtype in ['stdout', 'stderr']:
                        if out.get('type') == streamtype:
                            if 'outputBinding' in out:
                                raise ValidationException(
                                    "Not allowed to specify outputBinding when"
                                    " using %s shortcut." % streamtype)
                            if streamtype in workflowobj:
                                filename = workflowobj[streamtype]
                            else:
                                filename = Text(
                                    hashlib.sha1(  # nosec
                                        json_dumps(workflowobj, sort_keys=True
                                                  ).encode('utf-8')).hexdigest())
                                workflowobj[streamtype] = filename
                            out['type'] = 'File'
                            out['outputBinding'] = cmap({'glob': filename})
            for inp in workflowobj.get('inputs', []):
                if inp.get('type') == 'stdin':
                    if 'inputBinding' in inp:
                        raise ValidationException(
                            "Not allowed to specify inputBinding when"
                            " using stdin shortcut.")
                    if 'stdin' in workflowobj:
                        raise ValidationException(
                            "Not allowed to specify stdin path when"
                            " using stdin type shortcut.")
                    else:
                        workflowobj['stdin'] = \
                            "$(inputs.%s.path)" % \
                            inp['id'].rpartition('#')[2]
                        inp['type'] = 'File'
        else:
            for entry in itervalues(workflowobj):
                _convert_stdstreams_to_files(entry)
    if isinstance(workflowobj, MutableSequence):
        for entry in workflowobj:
            _convert_stdstreams_to_files(entry)

def _add_blank_ids(workflowobj):
    # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None

    if isinstance(workflowobj, MutableMapping):
        if ("run" in workflowobj and
                isinstance(workflowobj["run"], MutableMapping) and
                "id" not in workflowobj["run"] and
                "$import" not in workflowobj["run"]):
            workflowobj["run"]["id"] = Text(uuid.uuid4())
        for entry in itervalues(workflowobj):
            _add_blank_ids(entry)
    if isinstance(workflowobj, MutableSequence):
        for entry in workflowobj:
            _add_blank_ids(entry)

def resolve_and_validate_document(loadingContext,
                      workflowobj,
                      uri,
                      preprocess_only=False,     # type: bool
                      skip_schemas=None,         # type: bool
                     ):
    # type: (...) -> Tuple[LoadingContext, Text]
    """Validate a CWL document."""

    loadingContext = loadingContext.copy()

    if not isinstance(workflowobj, MutableMapping):
        raise ValueError("workflowjobj must be a dict, got '{}': {}".format(
            type(workflowobj), workflowobj))

    jobobj = None
    if "cwl:tool" in workflowobj:
        jobobj, _ = loadingContext.loader.resolve_all(workflowobj, uri, checklinks=loadingContext.do_validate)
        uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"])
        del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]

        workflowobj = fetch_document(uri, loadingContext)[1]

    fileuri = urllib.parse.urldefrag(uri)[0]

    cwlVersion = loadingContext.metadata.get("cwlVersion")
    if not cwlVersion:
        cwlVersion = workflowobj.get("cwlVersion")
    if not cwlVersion and fileuri != uri:
        # The tool we're loading is a fragment of a bigger file.  Get
        # the document root element and look for cwlVersion there
        metadata = fetch_document(fileuri, loadingContext)[1]
        cwlVersion = metadata.get("cwlVersion")
    if not cwlVersion:
        raise ValidationException(
            "No cwlVersion found. "
            "Use the following syntax in your CWL document to declare "
            "the version: cwlVersion: <version>.\n"
            "Note: if this is a CWL draft-2 (pre v1.0) document then it "
            "will need to be upgraded first.")

    if not isinstance(cwlVersion, string_types):
        with SourceLine(workflowobj, "cwlVersion", ValidationException):
            raise ValidationException("'cwlVersion' must be a string, "
                                      "got {}".format(
                                          type(cwlVersion)))
    # strip out version
    cwlVersion = re.sub(
        r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "",
        cwlVersion)
    if cwlVersion not in list(ALLUPDATES):
        # print out all the Supported Versions of cwlVersion
        versions = []
        for version in list(ALLUPDATES):
            if "dev" in version:
                version += " (with --enable-dev flag only)"
            versions.append(version)
        versions.sort()
        raise ValidationException(
            "The CWL reference runner no longer supports pre CWL v1.0 "
            "documents. Supported versions are: "
            "\n{}".format("\n".join(versions)))

    if isinstance(jobobj, CommentedMap) and "http://commonwl.org/cwltool#overrides" in jobobj:
        loadingContext.overrides_list.extend(resolve_overrides(jobobj, uri, uri))
        del jobobj["http://commonwl.org/cwltool#overrides"]

    if isinstance(jobobj, CommentedMap) and "https://w3id.org/cwl/cwl#requirements" in jobobj:
        if cwlVersion not in ("v1.1.0-dev1",):
            raise ValidationException(
                    "`cwl:requirements` in the input object is not part of CWL "
                    "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                    "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
                    "--enable-dev.")
        loadingContext.overrides_list.append({"overrideTarget": uri,
                                              "requirements": jobobj["https://w3id.org/cwl/cwl#requirements"]})
        del jobobj["https://w3id.org/cwl/cwl#requirements"]

    (sch_document_loader, avsc_names) = \
        process.get_schema(cwlVersion)[:2]

    if isinstance(avsc_names, Exception):
        raise avsc_names

    processobj = None  # type: Union[CommentedMap, CommentedSeq, Text, None]
    document_loader = Loader(sch_document_loader.ctx,
                             schemagraph=sch_document_loader.graph,
                             idx=loadingContext.loader.idx,
                             cache=sch_document_loader.cache,
                             fetcher_constructor=loadingContext.fetcher_constructor,
                             skip_schemas=skip_schemas)

    if cwlVersion == "v1.0":
        _add_blank_ids(workflowobj)

    processobj, metadata = document_loader.resolve_all(workflowobj, uri)
    if loadingContext.metadata:
        metadata = loadingContext.metadata
    if not isinstance(processobj, (CommentedMap, CommentedSeq)):
        raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
    if not isinstance(metadata, CommentedMap):
        raise ValidationException("metadata must be a CommentedMap, was %s" % type(metadata))

    _convert_stdstreams_to_files(workflowobj)

    if preprocess_only:
        return loadingContext, uri

    if loadingContext.do_validate:
        schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)

    # None means default behavior (do update)
    if loadingContext.do_update in (True, None):
        processobj = cast(CommentedMap, cmap(update.update(
            processobj, document_loader, fileuri, loadingContext.enable_dev, metadata)))
        if isinstance(processobj, MutableMapping):
            document_loader.idx[processobj["id"]] = processobj
        elif isinstance(processobj, MutableSequence):
            document_loader.idx[metadata["id"]] = metadata
            for po in processobj:
                document_loader.idx[po["id"]] = po
        else:
            raise Exception("'processobj' was not MutableMapping or MutableSequence %s" % type(processobj))

    if jobobj is not None:
        loadingContext.jobdefaults = jobobj

    loadingContext.loader = document_loader
    loadingContext.avsc_names = avsc_names
    loadingContext.metadata = metadata

    return loadingContext, uri

def make_tool(uri,                # type: Union[Text, CommentedMap, CommentedSeq]
              loadingContext      # type: LoadingContext
             ):  # type: (...) -> Process
    """Make a Python CWL object."""
    if loadingContext.loader is None:
        raise ValueError("loadingContext must have a loader")
    resolveduri, metadata = loadingContext.loader.resolve_ref(uri)

    processobj = None
    if isinstance(resolveduri, MutableSequence):
        for obj in resolveduri:
            if obj['id'].endswith('#main'):
                processobj = obj
                break
        if not processobj:
            raise WorkflowException(
                u"Tool file contains graph of multiple objects, must specify "
                "one of #%s" % ", #".join(
                    urllib.parse.urldefrag(i["id"])[1] for i in resolveduri
                    if "id" in i))
    elif isinstance(resolveduri, MutableMapping):
        processobj = resolveduri
    else:
        raise Exception("Must resolve to list or dict")

    tool = loadingContext.construct_tool_object(processobj, loadingContext)

    if loadingContext.jobdefaults:
        jobobj = loadingContext.jobdefaults
        for inp in tool.tool["inputs"]:
            if shortname(inp["id"]) in jobobj:
                inp["default"] = jobobj[shortname(inp["id"])]

    return tool

def load_tool(argsworkflow,         # type: Union[Text, Dict[Text, Any]]
              loadingContext=None   # type: Optional[LoadingContext]
             ):  # type: (...) -> Process

    loadingContext, workflowobj, uri = fetch_document(
        argsworkflow, loadingContext)

    loadingContext, uri = resolve_and_validate_document(
        loadingContext, workflowobj, uri)

    return make_tool(uri,
                     loadingContext)

def resolve_overrides(ov,      # Type: CommentedMap
                      ov_uri,  # Type: Text
                      baseurl  # type: Text
                     ):  # type: (...) -> List[Dict[Text, Any]]
    ovloader = Loader(overrides_ctx)
    ret, _ = ovloader.resolve_all(ov, baseurl)
    if not isinstance(ret, CommentedMap):
        raise Exception("Expected CommentedMap, got %s" % type(ret))
    cwl_docloader = get_schema("v1.0")[0]
    cwl_docloader.resolve_all(ret, ov_uri)
    return ret["http://commonwl.org/cwltool#overrides"]

def load_overrides(ov, base_url):  # type: (Text, Text) -> List[Dict[Text, Any]]
    ovloader = Loader(overrides_ctx)
    return resolve_overrides(ovloader.fetch(ov), ov, base_url)
<|endoftext|>


<commit_before>from django.contrib import admin
from django.db.models import Count, Max, Min

from .models import Build, Game, Title, Executable, XDKLibrary

class TitleInline(admin.StackedInline):
    model = Title
    extra = 0

class XDKLibraryInline(admin.StackedInline):
    model = Executable.xdk_libraries.through
    extra = 0

class GameAdmin(admin.ModelAdmin):
    list_display = ('name', 'titles', 'exes')
    search_fields = ('name',)

    inlines = [TitleInline]

    def titles(self, obj):
        return obj.titles

    titles.admin_order_field = 'titles'

    def exes(self, obj):
        return obj.exes

    exes.admin_order_field = 'exes'

    def get_queryset(self, request):
        qs = super(GameAdmin, self).get_queryset(request)
        qs = qs.annotate(titles=Count('title', distinct=True), exes=Count('title__executable'))
        return qs

class ExecutableInline(admin.TabularInline):
    model = Executable.xdk_libraries.through
    extra = 0

class XDKLibraryAdmin(admin.ModelAdmin):
    list_display = ('name', 'xdk_version', 'qfe_version', 'exes')
    list_filter = ('name', 'xdk_version')

    fields = ['name', 'xdk_version', 'qfe_version']

    inlines = [ExecutableInline]

    search_fields = ('name', 'xdk_version')

    def exes(self, obj):
        return obj.exes

    exes.admin_order_field = 'exes'

    def get_queryset(self, request):
        qs = super(XDKLibraryAdmin, self).get_queryset(request)
        qs = qs.annotate(exes=Count('executable'))
        return qs

class ExecutableAdmin(admin.ModelAdmin):
    list_display = ('executable', 'title_name', 'min_xdk', 'max_xdk', 'libraries')
    search_fields = ('file_name', 'title__game__name', 'title__title_id')

    fields = ['file_name', 'disk_path', 'signature', 'title']

    inlines = [XDKLibraryInline]

    def executable(self, obj):
        return '{1}{0}'.format(obj.file_name, obj.disk_path)

    def get_queryset(self, request):
        qs = super(ExecutableAdmin, self).get_queryset(request)
        qs = qs.annotate(
            libraries=Count('xdk_libraries'),
            max_version=Max('xdk_libraries__xdk_version'),
            min_version=Min('xdk_libraries__xdk_version')
        )
        return qs

    def min_xdk(self, obj):
        return obj.min_version

    def max_xdk(self, obj):
        return obj.max_version

    def libraries(self, obj):
        return obj.libraries

    def title_name(self, obj):
        return obj.title

    min_xdk.admin_order_field = 'min_version'
    max_xdk.admin_order_field = 'max_version'
    libraries.admin_order_field = 'libraries'
    title_name.admin_order_field = 'title__game__name'

class TitleAdmin(admin.ModelAdmin):
    list_display = ('title_id', 'game_name', 'exes')
    search_fields = ('title_id', 'game__name')

    def exes(self, obj):
        return obj.exes

    def game_name(self, obj):
        return obj.game

    def get_queryset(self, request):
        qs = super(TitleAdmin, self).get_queryset(request)
        qs = qs.annotate(exes=Count('executable'))
        return qs

    exes.admin_order_field = 'exes'
    game_name.admin_order_field = 'game__name'

admin.site.register(Build)
admin.site.register(Game, GameAdmin)
admin.site.register(Title, TitleAdmin)
admin.site.register(Executable, ExecutableAdmin)
admin.site.register(XDKLibrary, XDKLibraryAdmin)

<commit_msg>Defined verbose names for XDKLibraryInline<commit_after>from django.contrib import admin
from django.db.models import Count, Max, Min

from .models import Build, Game, Title, Executable, XDKLibrary

class TitleInline(admin.StackedInline):
    model = Title
    extra = 0

class XDKLibraryInline(admin.TabularInline):
    model = Executable.xdk_libraries.through
    extra = 0
    verbose_name = 'XDK Library'
    verbose_name_plural = 'XDK Libraries'

class GameAdmin(admin.ModelAdmin):
    list_display = ('name', 'titles', 'exes')
    search_fields = ('name',)

    inlines = [TitleInline]

    def titles(self, obj):
        return obj.titles

    titles.admin_order_field = 'titles'

    def exes(self, obj):
        return obj.exes

    exes.admin_order_field = 'exes'

    def get_queryset(self, request):
        qs = super(GameAdmin, self).get_queryset(request)
        qs = qs.annotate(titles=Count('title', distinct=True), exes=Count('title__executable'))
        return qs

class ExecutableInline(admin.TabularInline):
    model = Executable.xdk_libraries.through
    extra = 0

class XDKLibraryAdmin(admin.ModelAdmin):
    list_display = ('name', 'xdk_version', 'qfe_version', 'exes')
    list_filter = ('name', 'xdk_version')

    fields = ['name', 'xdk_version', 'qfe_version']

    inlines = [ExecutableInline]

    search_fields = ('name', 'xdk_version')

    def exes(self, obj):
        return obj.exes

    exes.admin_order_field = 'exes'

    def get_queryset(self, request):
        qs = super(XDKLibraryAdmin, self).get_queryset(request)
        qs = qs.annotate(exes=Count('executable'))
        return qs

class ExecutableAdmin(admin.ModelAdmin):
    list_display = ('executable', 'title_name', 'min_xdk', 'max_xdk', 'libraries')
    search_fields = ('file_name', 'title__game__name', 'title__title_id')

    fields = ['file_name', 'disk_path', 'signature', 'title']

    inlines = [XDKLibraryInline]

    def executable(self, obj):
        return '{1}{0}'.format(obj.file_name, obj.disk_path)

    def get_queryset(self, request):
        qs = super(ExecutableAdmin, self).get_queryset(request)
        qs = qs.annotate(
            libraries=Count('xdk_libraries'),
            max_version=Max('xdk_libraries__xdk_version'),
            min_version=Min('xdk_libraries__xdk_version')
        )
        return qs

    def min_xdk(self, obj):
        return obj.min_version

    def max_xdk(self, obj):
        return obj.max_version

    def libraries(self, obj):
        return obj.libraries

    def title_name(self, obj):
        return obj.title

    min_xdk.admin_order_field = 'min_version'
    max_xdk.admin_order_field = 'max_version'
    libraries.admin_order_field = 'libraries'
    title_name.admin_order_field = 'title__game__name'

class TitleAdmin(admin.ModelAdmin):
    list_display = ('title_id', 'game_name', 'exes')
    search_fields = ('title_id', 'game__name')

    def exes(self, obj):
        return obj.exes

    def game_name(self, obj):
        return obj.game

    def get_queryset(self, request):
        qs = super(TitleAdmin, self).get_queryset(request)
        qs = qs.annotate(exes=Count('executable'))
        return qs

    exes.admin_order_field = 'exes'
    game_name.admin_order_field = 'game__name'

admin.site.register(Build)
admin.site.register(Game, GameAdmin)
admin.site.register(Title, TitleAdmin)
admin.site.register(Executable, ExecutableAdmin)
admin.site.register(XDKLibrary, XDKLibraryAdmin)

<|endoftext|>