<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from deps import deps

NO_FLAG_MAGIC = '-*none*-'

class DepTable:
    def __init__(self, db):
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if 'Dependencies' not in tables:
            cu.execute("""CREATE TABLE Dependencies(depId integer primary key,
                                                    class integer,
                                                    name str,
                                                    flag str
                                                    )""")
            cu.execute("""CREATE INDEX DependenciesIdx ON Dependencies(
                                                    class, name, flag)""")

class DepUser:
    def __init__(self, db, name):
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if name not in tables:
            cu.execute("""CREATE TABLE %s(instanceId integer,
                                          depId integer
                                         )""" % name)
            cu.execute("CREATE INDEX %sIdx ON %s(instanceId)" % (name, name))
            cu.execute("CREATE INDEX %sIdx2 ON %s(depId)" % (name, name))

class DependencyTables:

    def _createTmpTable(self, cu, name):
        cu.execute("""CREATE TEMPORARY TABLE %s(
                                              troveId INT,
                                              depNum INT,
                                              flagCount INT,
                                              isProvides BOOL,
                                              class INTEGER,
                                              name STRING,
                                              flag STRING)""" % name)

    def _populateTmpTable(self, cu, name, depList, troveNum, requires,
                          provides):
        allDeps = []
        if requires:
            allDeps += [ (False, x) for x in
                            requires.getDepClasses().iteritems() ]
        if provides:
            allDeps += [ (True,  x) for x in
                            provides.getDepClasses().iteritems() ]

        for (isProvides, (classId, depClass)) in allDeps:
            for dep in depClass.getDeps():
                for (depName, flags) in zip(dep.getName(), dep.getFlags()):
                    if flags:
                        for flag in flags:
                            cu.execute("INSERT INTO %s VALUES(?, ?, ?, ?, "
                                                "?, ?, ?)" % name,
                                       (troveNum, len(depList), len(flags),
                                        isProvides, classId,
                                        depName, flag))
                    else:
                        cu.execute(    "INSERT INTO %s VALUES(?, ?, ?, ?, "
                                                "?, ?, ?)" % name,
                                       (troveNum, len(depList), 1,
                                        isProvides, classId,
                                        depName, NO_FLAG_MAGIC))

                if not isProvides:
                    depList.append((troveNum, classId, dep))

    def _mergeTmpTable(self, cu, tmpName, depTable, reqTable, provTable):
        substDict = { 'tmpName'   : tmpName,
                      'depTable'  : depTable,
                      'reqTable'  : reqTable,
                      'provTable' : provTable }

        cu = self.db.cursor()

        cu.execute("""INSERT INTO %(depTable)s
                        SELECT DISTINCT
                            NULL,
                            %(tmpName)s.class,
                            %(tmpName)s.name,
                            %(tmpName)s.flag
                        FROM %(tmpName)s LEFT OUTER JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(depTable)s.depId is NULL
                    """ % substDict)

        cu.execute("""INSERT INTO %(reqTable)s
                    SELECT %(tmpName)s.troveId, depId FROM
                        %(tmpName)s JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(tmpName)s.isProvides == 0""" % substDict)

        cu.execute("""INSERT INTO %(provTable)s SELECT
                            %(tmpName)s.troveId, depId FROM
                        %(tmpName)s JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(tmpName)s.isProvides == 1""" % substDict)

    def get(self, cu, trv, troveId):
        for (tblName, setFn) in (('Requires', trv.setRequires),
                                 ('Provides', trv.setProvides)):
            cu.execute("SELECT class, name, flag FROM %s NATURAL JOIN "
                       "Dependencies WHERE instanceId=? ORDER BY class, name"
                    % tblName, troveId)

            last = None
            flags = []
            depSet = deps.DependencySet()
            for (classId, name, flag) in cu:
                if (classId, name) == last:
                    flags.append(flag)
                else:
                    if last:
                        depSet.addDep(deps.dependencyClasses[last[0]],
                                      deps.Dependency(last[1], flags))

                    last = (classId, name)
                    flags = []
                    if flag != NO_FLAG_MAGIC:
                        flags.append(flag)

            if last:
                depSet.addDep(deps.dependencyClasses[last[0]],
                              deps.Dependency(last[1], flags))
                setFn(depSet)

    def add(self, cu, trove, troveId):
        self._createTmpTable(cu, "NeededDeps")
        self._populateTmpTable(cu, "NeededDeps", [], troveId,
                               trove.getRequires(), trove.getProvides())
        self._mergeTmpTable(cu, "NeededDeps", "Dependencies", "Requires",
                            "Provides")

        cu.execute("DROP TABLE NeededDeps")

    def delete(self, cu, troveId):
        cu.execute("CREATE TEMPORARY TABLE suspectDepsOrig(depId integer)")
        for tbl in ('Requires', 'Provides'):
            cu.execute("INSERT INTO suspectDepsOrig SELECT depId "
                       "FROM %s WHERE instanceId=%d" % (tbl, troveId))
            cu.execute("DELETE FROM %s WHERE instanceId=%d" % (tbl, troveId))

        cu.execute("CREATE TEMPORARY TABLE suspectDeps(depId integer)")
        cu.execute("INSERT INTO suspectDeps SELECT DISTINCT depId "
                   "FROM suspectDepsOrig")
        cu.execute("DROP TABLE suspectDepsOrig")

        cu.execute("""DELETE FROM Dependencies WHERE depId IN
                (SELECT DISTINCT suspectDeps.depId FROM suspectDeps
                 LEFT OUTER JOIN
                    (SELECT depId AS depId1,
                            instanceId AS instanceId1 FROM Requires UNION
                     SELECT depId AS depId1,
                            instanceId AS instanceId1 FROM Provides)
                    ON suspectDeps.depId = depId1
                 WHERE instanceId1 IS NULL)""")

        cu.execute("DROP TABLE suspectDeps")

    def _resolveStmt(self, providesTable = "Provides"):
        substTable = { 'provides' : providesTable }

        return """
                SELECT depCheck.depNum as depNum,
                        %(provides)s.instanceId as rsvInstanceId
                    FROM Requires LEFT OUTER JOIN %(provides)s ON
                        Requires.depId == %(provides)s.depId
                    JOIN Dependencies ON
                        Requires.depId == Dependencies.depId
                    JOIN DepCheck ON
                        Requires.instanceId == DepCheck.troveId AND
                        Dependencies.class == DepCheck.class AND
                        Dependencies.name == DepCheck.name AND
                        Dependencies.flag == DepCheck.flag
                    WHERE
                        Requires.instanceId < 0
                        AND %(provides)s.depId is not NULL
                        AND NOT DepCheck.isProvides
                    GROUP BY
                        DepCheck.depNum,
                        %(provides)s.instanceId
                    HAVING
                        COUNT(DepCheck.troveId) == DepCheck.flagCount
                """ % substTable

    def check(self, changeSet):
        # XXX
        # this ignores file dependencies for now

        self.db._begin()
        cu = self.db.cursor()

        self._createTmpTable(cu, "DepCheck")
        depList = []
        failedSets = []
        for i, trvCs in enumerate(changeSet.iterNewPackageList()):
            failedSets.append((trvCs.getName(), None, None))
            self._populateTmpTable(cu, "DepCheck", depList, -i - 1,
                                   trvCs.getRequires(),
                                   trvCs.getProvides())

            cu.execute("INSERT INTO DepCheck VALUES(?, ?, ?, ?, ?, ?, ?)",
                       (-i - 1, 0, 1, True, deps.DEP_CLASS_TROVES,
                        trvCs.getName(), NO_FLAG_MAGIC))

        if not failedSets:
            self.db.rollback()
            return (False, [])

        self._mergeTmpTable(cu, "DepCheck", "Dependencies", "Requires",
                            "Provides")
        cu.execute(self._resolveStmt())

        for (depNum, instanceId) in cu:
            depList[depNum] = None

        missingDeps = False
        for depInfo in depList:
            if depInfo is not None:
                (troveIndex, classId, dep) = depInfo

                if classId in [ deps.DEP_CLASS_ABI, deps.DEP_CLASS_FILES ]:
                    continue

                missingDeps = True
                troveIndex = -(troveIndex + 1)

                if failedSets[troveIndex][2] is None:
                    failedSets[troveIndex] = (failedSets[troveIndex][0],
                                              failedSets[troveIndex][1],
                                              deps.DependencySet())
                failedSets[troveIndex][2].addDep(
                                deps.dependencyClasses[classId], dep)

        failedList = []
        for (name, classId, depSet) in failedSets:
            if depSet is not None:
                failedList.append((name, depSet))

        # no need to drop the DepCheck table since we're rolling this whole
        # transaction back anyway
        self.db.rollback()

        return (missingDeps, failedList)

    def resolve(self, label, depSetList):
        cu = self.db.cursor()

        cu.execute("""
                CREATE VIEW providesBranch AS
                    SELECT provides.depId AS depId,
                           provides.instanceId AS instanceId FROM
                    LabelMap JOIN Nodes ON
                        LabelMap.itemId == Nodes.itemId AND
                        LabelMap.branchId == Nodes.branchId
                    JOIN Instances ON
                        Instances.itemId == Nodes.itemId AND
                        Instances.versionId == Nodes.versionId
                    JOIN Provides ON
                        Provides.instanceId == Instances.instanceId
                    WHERE
                        LabelMap.labelId ==
                            (SELECT labelId FROM Labels WHERE Label='%s')
        """ % label.asString())

        self._createTmpTable(cu, "DepCheck")

        depList = []
        for i, depSet in enumerate(depSetList):
            self._populateTmpTable(cu, "DepCheck", depList, -i - 1,
                                   depSet, None)

        self._mergeTmpTable(cu, "DepCheck", "Dependencies", "Requires",
                            "Provides")

        cu.execute("""SELECT depNum, Items.item, Versions.version FROM
                        (%s)
                      JOIN Instances ON
                        rsvInstanceId == Instances.instanceId
                      JOIN Items ON
                        Instances.itemId == Items.itemId
                      JOIN Versions ON
                        Instances.versionId == Versions.versionId
                      JOIN Nodes ON
                        Instances.itemId == Nodes.itemId AND
                        Instances.versionId == Nodes.versionId
                      ORDER BY
                        Nodes.finalTimestamp DESC
                    """ % self._resolveStmt(providesTable = "providesBranch"))
        result = {}
        handled = {}
        for (depId, troveName, versionStr) in cu:
            if handled.has_key(depId):
                continue

            handled[depId] = True

            depNum = depList[depId][0]
            depSet = depSetList[depNum]
            l = result.get(depSet, None)
            if not l:
                result[depSet] = [ (troveName, versionStr) ]
            elif (troveName, versionStr) not in l:
                l.append((troveName, versionStr))

        # no need to drop the DepCheck table since we're rolling this whole
        # transaction back anyway
        self.db.rollback()

        return result

    def __init__(self, db):
        self.db = db
        DepTable(db)
        DepUser(db, 'Provides')
        DepUser(db, 'Requires')
<commit_msg>operations on (possibly) temporary tables no longer start transactions<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from deps import deps

NO_FLAG_MAGIC = '-*none*-'

class DepTable:
    def __init__(self, db):
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if 'Dependencies' not in tables:
            cu.execute("""CREATE TABLE Dependencies(depId integer primary key,
                                                    class integer,
                                                    name str,
                                                    flag str
                                                    )""")
            cu.execute("""CREATE INDEX DependenciesIdx ON Dependencies(
                                                    class, name, flag)""")

class DepUser:
    def __init__(self, db, name):
        cu = db.cursor()
        cu.execute("SELECT tbl_name FROM sqlite_master WHERE type='table'")
        tables = [ x[0] for x in cu ]
        if name not in tables:
            cu.execute("""CREATE TABLE %s(instanceId integer,
                                          depId integer
                                         )""" % name)
            cu.execute("CREATE INDEX %sIdx ON %s(instanceId)" % (name, name))
            cu.execute("CREATE INDEX %sIdx2 ON %s(depId)" % (name, name))

class DependencyTables:

    def _createTmpTable(self, cu, name):
        cu.execute("""CREATE TEMPORARY TABLE %s(
                                              troveId INT,
                                              depNum INT,
                                              flagCount INT,
                                              isProvides BOOL,
                                              class INTEGER,
                                              name STRING,
                                              flag STRING)""" % name,
                   start_transaction = False)

    def _populateTmpTable(self, cu, name, depList, troveNum, requires,
                          provides):
        allDeps = []
        if requires:
            allDeps += [ (False, x) for x in
                            requires.getDepClasses().iteritems() ]
        if provides:
            allDeps += [ (True,  x) for x in
                            provides.getDepClasses().iteritems() ]

        for (isProvides, (classId, depClass)) in allDeps:
            for dep in depClass.getDeps():
                for (depName, flags) in zip(dep.getName(), dep.getFlags()):
                    if flags:
                        for flag in flags:
                            cu.execute("INSERT INTO %s VALUES(?, ?, ?, ?, "
                                                "?, ?, ?)" % name,
                                       (troveNum, len(depList), len(flags),
                                        isProvides, classId,
                                        depName, flag),
                                       start_transaction = False)
                    else:
                        cu.execute(    "INSERT INTO %s VALUES(?, ?, ?, ?, "
                                                "?, ?, ?)" % name,
                                       (troveNum, len(depList), 1,
                                        isProvides, classId,
                                        depName, NO_FLAG_MAGIC),
                                       startTransaction = False)

                if not isProvides:
                    depList.append((troveNum, classId, dep))

    def _mergeTmpTable(self, cu, tmpName, depTable, reqTable, provTable):
        substDict = { 'tmpName'   : tmpName,
                      'depTable'  : depTable,
                      'reqTable'  : reqTable,
                      'provTable' : provTable }

        cu = self.db.cursor()

        cu.execute("""INSERT INTO %(depTable)s
                        SELECT DISTINCT
                            NULL,
                            %(tmpName)s.class,
                            %(tmpName)s.name,
                            %(tmpName)s.flag
                        FROM %(tmpName)s LEFT OUTER JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(depTable)s.depId is NULL
                    """ % substDict, startTransaction = False)

        cu.execute("""INSERT INTO %(reqTable)s
                    SELECT %(tmpName)s.troveId, depId FROM
                        %(tmpName)s JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(tmpName)s.isProvides == 0""" % substDict,
                   start_transaction = False)

        cu.execute("""INSERT INTO %(provTable)s SELECT
                            %(tmpName)s.troveId, depId FROM
                        %(tmpName)s JOIN %(depTable)s ON
                            %(tmpName)s.class == %(depTable)s.class AND
                            %(tmpName)s.name == %(depTable)s.name AND
                            %(tmpName)s.flag == %(depTable)s.flag
                        WHERE
                            %(tmpName)s.isProvides == 1""" % substDict,
                   start_transaction = False)

    def get(self, cu, trv, troveId):
        for (tblName, setFn) in (('Requires', trv.setRequires),
                                 ('Provides', trv.setProvides)):
            cu.execute("SELECT class, name, flag FROM %s NATURAL JOIN "
                       "Dependencies WHERE instanceId=? ORDER BY class, name"
                    % tblName, troveId)

            last = None
            flags = []
            depSet = deps.DependencySet()
            for (classId, name, flag) in cu:
                if (classId, name) == last:
                    flags.append(flag)
                else:
                    if last:
                        depSet.addDep(deps.dependencyClasses[last[0]],
                                      deps.Dependency(last[1], flags))

                    last = (classId, name)
                    flags = []
                    if flag != NO_FLAG_MAGIC:
                        flags.append(flag)

            if last:
                depSet.addDep(deps.dependencyClasses[last[0]],
                              deps.Dependency(last[1], flags))
                setFn(depSet)

    def add(self, cu, trove, troveId):
        # XXX this should assert() that we're in a transaction, but I
        # need msw to tell me how to check that
        self._createTmpTable(cu, "NeededDeps")
        self._populateTmpTable(cu, "NeededDeps", [], troveId,
                               trove.getRequires(), trove.getProvides())
        self._mergeTmpTable(cu, "NeededDeps", "Dependencies", "Requires",
                            "Provides")

        cu.execute("DROP TABLE NeededDeps", start_transaction = False)

    def delete(self, cu, troveId):
        cu.execute("CREATE TEMPORARY TABLE suspectDepsOrig(depId integer)")
        for tbl in ('Requires', 'Provides'):
            cu.execute("INSERT INTO suspectDepsOrig SELECT depId "
                       "FROM %s WHERE instanceId=%d" % (tbl, troveId))
            cu.execute("DELETE FROM %s WHERE instanceId=%d" % (tbl, troveId))

        cu.execute("CREATE TEMPORARY TABLE suspectDeps(depId integer)")
        cu.execute("INSERT INTO suspectDeps SELECT DISTINCT depId "
                   "FROM suspectDepsOrig")
        cu.execute("DROP TABLE suspectDepsOrig")

        cu.execute("""DELETE FROM Dependencies WHERE depId IN
                (SELECT DISTINCT suspectDeps.depId FROM suspectDeps
                 LEFT OUTER JOIN
                    (SELECT depId AS depId1,
                            instanceId AS instanceId1 FROM Requires UNION
                     SELECT depId AS depId1,
                            instanceId AS instanceId1 FROM Provides)
                    ON suspectDeps.depId = depId1
                 WHERE instanceId1 IS NULL)""")

        cu.execute("DROP TABLE suspectDeps")

    def _resolveStmt(self, providesTable = "Provides"):
        substTable = { 'provides' : providesTable }

        return """
                SELECT depCheck.depNum as depNum,
                        %(provides)s.instanceId as rsvInstanceId
                    FROM Requires LEFT OUTER JOIN %(provides)s ON
                        Requires.depId == %(provides)s.depId
                    JOIN Dependencies ON
                        Requires.depId == Dependencies.depId
                    JOIN DepCheck ON
                        Requires.instanceId == DepCheck.troveId AND
                        Dependencies.class == DepCheck.class AND
                        Dependencies.name == DepCheck.name AND
                        Dependencies.flag == DepCheck.flag
                    WHERE
                        Requires.instanceId < 0
                        AND %(provides)s.depId is not NULL
                        AND NOT DepCheck.isProvides
                    GROUP BY
                        DepCheck.depNum,
                        %(provides)s.instanceId
                    HAVING
                        COUNT(DepCheck.troveId) == DepCheck.flagCount
                """ % substTable

    def check(self, changeSet):
        # XXX
        # this ignores file dependencies for now

        self.db._begin()
        cu = self.db.cursor()

        self._createTmpTable(cu, "DepCheck")
        depList = []
        failedSets = []
        for i, trvCs in enumerate(changeSet.iterNewPackageList()):
            failedSets.append((trvCs.getName(), None, None))
            self._populateTmpTable(cu, "DepCheck", depList, -i - 1,
                                   trvCs.getRequires(),
                                   trvCs.getProvides())

            cu.execute("INSERT INTO DepCheck VALUES(?, ?, ?, ?, ?, ?, ?)",
                       (-i - 1, 0, 1, True, deps.DEP_CLASS_TROVES,
                        trvCs.getName(), NO_FLAG_MAGIC),
                       start_transaction = False)

        if not failedSets:
            self.db.rollback()
            return (False, [])

        self._mergeTmpTable(cu, "DepCheck", "Dependencies", "Requires",
                            "Provides")
        cu.execute(self._resolveStmt(), start_transaction = False)

        for (depNum, instanceId) in cu:
            depList[depNum] = None

        missingDeps = False
        for depInfo in depList:
            if depInfo is not None:
                (troveIndex, classId, dep) = depInfo

                if classId in [ deps.DEP_CLASS_ABI, deps.DEP_CLASS_FILES ]:
                    continue

                missingDeps = True
                troveIndex = -(troveIndex + 1)

                if failedSets[troveIndex][2] is None:
                    failedSets[troveIndex] = (failedSets[troveIndex][0],
                                              failedSets[troveIndex][1],
                                              deps.DependencySet())
                failedSets[troveIndex][2].addDep(
                                deps.dependencyClasses[classId], dep)

        failedList = []
        for (name, classId, depSet) in failedSets:
            if depSet is not None:
                failedList.append((name, depSet))

        # no need to drop the DepCheck table since we're rolling this whole
        # transaction back anyway
        self.db.rollback()

        return (missingDeps, failedList)

    def resolve(self, label, depSetList):
        cu = self.db.cursor()

        cu.execute("""
                CREATE VIEW providesBranch AS
                    SELECT provides.depId AS depId,
                           provides.instanceId AS instanceId FROM
                    LabelMap JOIN Nodes ON
                        LabelMap.itemId == Nodes.itemId AND
                        LabelMap.branchId == Nodes.branchId
                    JOIN Instances ON
                        Instances.itemId == Nodes.itemId AND
                        Instances.versionId == Nodes.versionId
                    JOIN Provides ON
                        Provides.instanceId == Instances.instanceId
                    WHERE
                        LabelMap.labelId ==
                            (SELECT labelId FROM Labels WHERE Label='%s')
        """ % label.asString())

        self._createTmpTable(cu, "DepCheck")

        depList = []
        for i, depSet in enumerate(depSetList):
            self._populateTmpTable(cu, "DepCheck", depList, -i - 1,
                                   depSet, None)

        self._mergeTmpTable(cu, "DepCheck", "Dependencies", "Requires",
                            "Provides")

        cu.execute("""SELECT depNum, Items.item, Versions.version FROM
                        (%s)
                      JOIN Instances ON
                        rsvInstanceId == Instances.instanceId
                      JOIN Items ON
                        Instances.itemId == Items.itemId
                      JOIN Versions ON
                        Instances.versionId == Versions.versionId
                      JOIN Nodes ON
                        Instances.itemId == Nodes.itemId AND
                        Instances.versionId == Nodes.versionId
                      ORDER BY
                        Nodes.finalTimestamp DESC
                    """ % self._resolveStmt(providesTable = "providesBranch"))
        result = {}
        handled = {}
        for (depId, troveName, versionStr) in cu:
            if handled.has_key(depId):
                continue

            handled[depId] = True

            depNum = depList[depId][0]
            depSet = depSetList[depNum]
            l = result.get(depSet, None)
            if not l:
                result[depSet] = [ (troveName, versionStr) ]
            elif (troveName, versionStr) not in l:
                l.append((troveName, versionStr))

        # no need to drop the DepCheck table since we're rolling this whole
        # transaction back anyway
        self.db.rollback()

        return result

    def __init__(self, db):
        self.db = db
        DepTable(db)
        DepUser(db, 'Provides')
        DepUser(db, 'Requires')
<|endoftext|>


<commit_before>#!/usr/bin/python
"""
(c) William Stein, 2013

A git-aware version of "ls -l", with JSON output

For each file, we compute the following information:

    * name: name of the file
    * isdir: true if it is a directory
    * size: size in bytes (if it is not a directory)
    * mtime: timestamp of file itself
    * commit: the *most recent* git commit related to this file; only
      given if file is under git control
         - author
         - date
         - message
         - sha
"""

import json, os, subprocess, sys, uuid

def getmtime(name):
    try:
        return os.lstat(name).st_mtime   # use lstat instead of stat or getmtime so this works on broken symlinks!
    except:
        # This should never happen, but we include this just in case
        # to avoid really buggy UI behavior.
        return 0

def getsize(name):
    try:
        return os.lstat(name).st_size # same as above; use instead of os.path....
    except:
        return 0

def gitls(path, time, start, limit, hidden, directories_first, git_aware=True):
    if not os.path.exists(path):
        sys.stderr.write("error: no such path '%s'"%path)
        sys.exit(1)
    os.chdir(path)
    result = {}
    listdir = os.listdir('.')
    try:
        json.dumps(listdir)
    except:
        # Throw away filenames that can't be json'd, since they can't be JSON'd below, which would totally
        # lock user out of their project.
        listdir = []
        for x in os.listdir('.'):
            try:
                json.dumps(x)
                listdir.append(x)
            except:
                pass
    if not hidden:
        listdir = [x for x in listdir if not x.startswith('.')]
    if path.endswith('.snapshots') or path.endswith('.snapshots/'):
        all = [(x, getmtime(x)) for x in reversed(sorted(listdir)) if os.path.exists(x)]  # exists due to broken symlinks when snaps vanish
        files = dict([(name, {'name':name, 'mtime':mtime}) for name, mtime in all])
        sorted_names = [x[0] for x in all]
    elif time:
        # Get list of pairs (timestamp, name)
        all = [(getmtime(name), name) for name in listdir]
        # Sort
        all.sort()
        all.reverse()
        # Limit and convert to objects
        all = all[start:]
        if limit > 0 and len(all) > limit:
            result['more'] = True
            all = all[:limit]
        files = dict([(name, {'name':name, 'mtime':int(mtime)}) for mtime, name in all])
        sorted_names = [x[1] for x in all]
    else:
        # Get list of (name, timestamp) pairs
        all = [(name, int(getmtime(name))) for name in listdir]
        # Sort
        all.sort()
        # Limit and convert to objects
        all = all[start:]
        if limit > 0 and len(all) > limit:
            result['more'] = True
            all = all[:limit]
        files = dict([(name, {'name':name, 'mtime':mtime}) for name, mtime in all])
        sorted_names = [x[0] for x in all]

    # Fill in other OS information about each file
    #for obj in result:
    for name, info in files.iteritems():
        if os.path.isdir(name):
            info['isdir'] = True
        else:
            info['size'] = getsize(name)

    # Fill in git-related information about each file
    if git_aware:
        # First, obtain git history about files
        format = "--pretty=format:!%H|%an <%ae>|%ad|%s|"
        field_sep = str(uuid.uuid4())
        commit_sep = str(uuid.uuid4())
        format = format.replace('|',field_sep).replace("!",commit_sep)
        # Why 200 below: in tests, if there are tons of files, passing
        # them all on the command line ends up taking longer than just
        # getting info about everything in this tree
        if len(files) > 200:
            git_path = ['.']
        else:
            git_path = [name for name, info in files.iteritems() if not info.get('isdir',False)]
        log = subprocess.Popen(['git', 'log', '--date=raw', '--name-status', format] + git_path,
                               stdin=subprocess.PIPE, stdout = subprocess.PIPE,
                               stderr=subprocess.PIPE).stdout.read()
        v = {}
        for entry in log.split(commit_sep):
            if len(entry.strip()) == 0 : continue
            sha, author, date, message, modified_files= entry.split(field_sep)
            date = int(date.split()[0])
            # modified_files = set of filenames
            modified_files = [os.path.split(str(x[2:]).replace('\\\\"','"').replace('\\"',''))[-1] for x in modified_files.splitlines() if x]
            for name in modified_files:
                if name in files and not files[name].has_key('commit'):
                    files[name]['commit'] =  {'author':author, 'date':date, 'message':message, 'sha':sha}

    # Make ordered list of files, with directories first.
    if directories_first:
        result['files'] = ([files[name] for name in sorted_names if files[name].get('isdir',False)] + \
                           [files[name] for name in sorted_names if not files[name].get('isdir',False)])
    else:
        result['files'] = [files[name] for name in sorted_names]
    return result

def main():
    import argparse
    parser = argparse.ArgumentParser(description="git-ls -- like ls, but git *aware*, and JSON output; files not under git are still included.")
    parser.add_argument('--time', help='order by time instead of alphabetical order (default: False)', action='store_true')
    parser.add_argument('--start',  help='starting file number', default=0)
    parser.add_argument('--limit',  help='maximum number of files', default=0)  # 0 = no limit
    parser.add_argument('--git', help="actually be git aware (not the default)", action="store_true")
    parser.add_argument('--hidden', help="include files/directories that start with a dot", action="store_true")
    parser.add_argument('--directories_first', help="sort by putting directories first, then files (default: False)", action="store_true")
    parser.add_argument('path', nargs='?', help='return info about all files in this path', default='.')
    args = parser.parse_args()
    if isinstance(args.path, list):
        args.path = args.path[0]

    if os.path.abspath(args.path) == os.path.join(os.environ['HOME'], '.snapshots'):
        # When getting a listing for the .snapshots directory, update it to show the latest version.
        from update_snapshots import update_snapshots
        update_snapshots()

    r = gitls(path=args.path, time=args.time, start=int(args.start), limit=int(args.limit),
                hidden=args.hidden, directories_first=args.directories_first, git_aware=args.git)
    print json.dumps(r, separators=(',',':'))

if __name__ == "__main__":
    main()

<commit_msg>has_key is not allowed in py3<commit_after>#!/usr/bin/python
"""
(c) William Stein, 2013

A git-aware version of "ls -l", with JSON output

For each file, we compute the following information:

    * name: name of the file
    * isdir: true if it is a directory
    * size: size in bytes (if it is not a directory)
    * mtime: timestamp of file itself
    * commit: the *most recent* git commit related to this file; only
      given if file is under git control
         - author
         - date
         - message
         - sha
"""

import json, os, subprocess, sys, uuid

def getmtime(name):
    try:
        return os.lstat(name).st_mtime   # use lstat instead of stat or getmtime so this works on broken symlinks!
    except:
        # This should never happen, but we include this just in case
        # to avoid really buggy UI behavior.
        return 0

def getsize(name):
    try:
        return os.lstat(name).st_size # same as above; use instead of os.path....
    except:
        return 0

def gitls(path, time, start, limit, hidden, directories_first, git_aware=True):
    if not os.path.exists(path):
        sys.stderr.write("error: no such path '%s'"%path)
        sys.exit(1)
    os.chdir(path)
    result = {}
    listdir = os.listdir('.')
    try:
        json.dumps(listdir)
    except:
        # Throw away filenames that can't be json'd, since they can't be JSON'd below, which would totally
        # lock user out of their project.
        listdir = []
        for x in os.listdir('.'):
            try:
                json.dumps(x)
                listdir.append(x)
            except:
                pass
    if not hidden:
        listdir = [x for x in listdir if not x.startswith('.')]
    if path.endswith('.snapshots') or path.endswith('.snapshots/'):
        all = [(x, getmtime(x)) for x in reversed(sorted(listdir)) if os.path.exists(x)]  # exists due to broken symlinks when snaps vanish
        files = dict([(name, {'name':name, 'mtime':mtime}) for name, mtime in all])
        sorted_names = [x[0] for x in all]
    elif time:
        # Get list of pairs (timestamp, name)
        all = [(getmtime(name), name) for name in listdir]
        # Sort
        all.sort()
        all.reverse()
        # Limit and convert to objects
        all = all[start:]
        if limit > 0 and len(all) > limit:
            result['more'] = True
            all = all[:limit]
        files = dict([(name, {'name':name, 'mtime':int(mtime)}) for mtime, name in all])
        sorted_names = [x[1] for x in all]
    else:
        # Get list of (name, timestamp) pairs
        all = [(name, int(getmtime(name))) for name in listdir]
        # Sort
        all.sort()
        # Limit and convert to objects
        all = all[start:]
        if limit > 0 and len(all) > limit:
            result['more'] = True
            all = all[:limit]
        files = dict([(name, {'name':name, 'mtime':mtime}) for name, mtime in all])
        sorted_names = [x[0] for x in all]

    # Fill in other OS information about each file
    #for obj in result:
    for name, info in files.iteritems():
        if os.path.isdir(name):
            info['isdir'] = True
        else:
            info['size'] = getsize(name)

    # Fill in git-related information about each file
    if git_aware:
        # First, obtain git history about files
        format = "--pretty=format:!%H|%an <%ae>|%ad|%s|"
        field_sep = str(uuid.uuid4())
        commit_sep = str(uuid.uuid4())
        format = format.replace('|',field_sep).replace("!",commit_sep)
        # Why 200 below: in tests, if there are tons of files, passing
        # them all on the command line ends up taking longer than just
        # getting info about everything in this tree
        if len(files) > 200:
            git_path = ['.']
        else:
            git_path = [name for name, info in files.iteritems() if not info.get('isdir',False)]
        log = subprocess.Popen(['git', 'log', '--date=raw', '--name-status', format] + git_path,
                               stdin=subprocess.PIPE, stdout = subprocess.PIPE,
                               stderr=subprocess.PIPE).stdout.read()
        v = {}
        for entry in log.split(commit_sep):
            if len(entry.strip()) == 0 : continue
            sha, author, date, message, modified_files= entry.split(field_sep)
            date = int(date.split()[0])
            # modified_files = set of filenames
            modified_files = [os.path.split(str(x[2:]).replace('\\\\"','"').replace('\\"',''))[-1] for x in modified_files.splitlines() if x]
            for name in modified_files:
                if name in files and 'commit' not in files[name]:
                    files[name]['commit'] =  {'author':author, 'date':date, 'message':message, 'sha':sha}

    # Make ordered list of files, with directories first.
    if directories_first:
        result['files'] = ([files[name] for name in sorted_names if files[name].get('isdir',False)] + \
                           [files[name] for name in sorted_names if not files[name].get('isdir',False)])
    else:
        result['files'] = [files[name] for name in sorted_names]
    return result

def main():
    import argparse
    parser = argparse.ArgumentParser(description="git-ls -- like ls, but git *aware*, and JSON output; files not under git are still included.")
    parser.add_argument('--time', help='order by time instead of alphabetical order (default: False)', action='store_true')
    parser.add_argument('--start',  help='starting file number', default=0)
    parser.add_argument('--limit',  help='maximum number of files', default=0)  # 0 = no limit
    parser.add_argument('--git', help="actually be git aware (not the default)", action="store_true")
    parser.add_argument('--hidden', help="include files/directories that start with a dot", action="store_true")
    parser.add_argument('--directories_first', help="sort by putting directories first, then files (default: False)", action="store_true")
    parser.add_argument('path', nargs='?', help='return info about all files in this path', default='.')
    args = parser.parse_args()
    if isinstance(args.path, list):
        args.path = args.path[0]

    if os.path.abspath(args.path) == os.path.join(os.environ['HOME'], '.snapshots'):
        # When getting a listing for the .snapshots directory, update it to show the latest version.
        from update_snapshots import update_snapshots
        update_snapshots()

    r = gitls(path=args.path, time=args.time, start=int(args.start), limit=int(args.limit),
                hidden=args.hidden, directories_first=args.directories_first, git_aware=args.git)
    print json.dumps(r, separators=(',',':'))

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, print_function
from builtins import zip
from builtins import map
from builtins import range
import codecs
import os
import random
import unittest

# Python 3 compatibility imports
from six.moves import xrange

from toil.lib.bioio import getTempFile
from toil.job import Job
from toil.test import ToilTest, slow
import time
import sys
from threading import Thread, Event
import logging
logger = logging.getLogger( __name__ )
from unittest import skipIf
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
from toil.leader import FailedJobsException, DeadlockException

class JobServiceTest(ToilTest):
    """
    Tests testing the Job.Service class
    """

    @slow
    def testServiceSerialization(self):
        """
        Tests that a service can receive a promise without producing a serialization
        error.
        """
        job = Job()
        service = ToySerializableService("woot")
        startValue = job.addService(service) # Add a first service to job
        subService = ToySerializableService(startValue) # Now create a child of
        # that service that takes the start value promise from the parent service
        job.addService(subService, parentService=service) # This should work if
        # serialization on services is working correctly.

        self.runToil(job)

    @slow
    def testService(self, checkpoint=False):
        """
        Tests the creation of a Job.Service with random failures of the worker.
        """
        for test in range(2):
            outFile = getTempFile(rootDir=self._createTempDir()) # Temporary file
            messageInt = random.randint(1, sys.maxsize)
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTest, outFile, messageInt, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEqual(int(open(outFile, 'r').readline()), messageInt)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceDeadlock(self):
        """
        Creates a job with more services than maxServices, checks that deadlock is detected.
        """
        outFile = getTempFile(rootDir=self._createTempDir())
        try:
            def makeWorkflow():
                job = Job()
                r1 = job.addService(ToySerializableService("woot1"))
                r2 = job.addService(ToySerializableService("woot2"))
                r3 = job.addService(ToySerializableService("woot3"))
                job.addChildFn(fnTest, [ r1, r2, r3 ], outFile)
                return job

            # This should fail as too few services available
            try:
                self.runToil(makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5)
            except DeadlockException:
                print("Got expected deadlock exception")
            else:
                assert 0

            # This should pass, as adequate services available
            self.runToil(makeWorkflow(), maxServiceJobs=3)
            # Check we get expected output
            assert open(outFile, 'r').read() == "woot1 woot2 woot3"
        finally:
            os.remove(outFile)

    def testServiceWithCheckpoints(self):
        """
        Tests the creation of a Job.Service with random failures of the worker, making the root job use checkpointing to
        restart the subtree.
        """
        self.testService(checkpoint=True)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating a chain of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFile = getTempFile(rootDir=self._createTempDir())
            messages = [ random.randint(1, sys.maxsize) for i in range(3) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestRecursive, outFile, messages, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEqual(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceParallelRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating parallel chains of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFiles = [ getTempFile(rootDir=self._createTempDir()) for j in range(2) ]
            messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestParallelRecursive, outFiles, messageBundles, checkpoint=True)

                # Run the workflow repeatedly until success
                self.runToil(t, retryCount=2)

                # Check output
                for (messages, outFile) in zip(messageBundles, outFiles):
                    self.assertEqual(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                list(map(os.remove, outFiles))

    def runToil(self, rootJob, retryCount=1, badWorker=0.5, badWorkedFailInterval=0.05, maxServiceJobs=sys.maxsize, deadlockWait=60):
        # Create the runner for the workflow.
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = "DEBUG"

        options.retryCount = retryCount
        options.badWorker = badWorker
        options.badWorkerFailInterval = badWorkedFailInterval
        options.servicePollingInterval = 1
        options.maxServiceJobs = maxServiceJobs
        options.deadlockWait=deadlockWait

        # Run the workflow
        totalTrys = 0
        while True:
            try:
                Job.Runner.startToil(rootJob, options)
                break
            except FailedJobsException as e:
                i = e.numberOfFailedJobs
                if totalTrys > 40: #p(fail after this many restarts) = 0.5**32
                    self.fail() #Exceeded a reasonable number of restarts
                totalTrys += 1
                options.restart = True

def serviceTest(job, outFile, messageInt):
    """
    Creates one service and one accessing job, which communicate with two files to establish
    that both run concurrently.
    """
    #Clean out out-file
    open(outFile, 'w').close()
    randInt = random.randint(1, sys.maxsize) # We create a random number that is added to messageInt and subtracted by the serviceAccessor, to prove that
    # when service test is checkpointed and restarted there is never a connection made between an earlier service and later serviceAccessor, or vice versa.
    job.addChildJobFn(serviceAccessor, job.addService(ToyService(messageInt + randInt)), outFile, randInt)

def serviceTestRecursive(job, outFile, messages):
    """
    Creates a chain of services and accessing jobs, each paired together.
    """
    if len(messages) > 0:
        #Clean out out-file
        open(outFile, 'w').close()
        randInt = random.randint(1, sys.maxsize)
        service = ToyService(messages[0] + randInt)
        child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

        for i in range(1, len(messages)):
            randInt = random.randint(1, sys.maxsize)
            service2 = ToyService(messages[i] + randInt, cores=0.1)
            child = child.addChildJobFn(serviceAccessor,
                                        job.addService(service2, parentService=service),
                                        outFile, randInt, cores=0.1)
            service = service2

def serviceTestParallelRecursive(job, outFiles, messageBundles):
    """
    Creates multiple chains of services and accessing jobs.
    """
    for messages, outFile in zip(messageBundles, outFiles):
        #Clean out out-file
        open(outFile, 'w').close()
        if len(messages) > 0:
            randInt = random.randint(1, sys.maxsize)
            service = ToyService(messages[0] + randInt)
            child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

            for i in range(1, len(messages)):
                randInt = random.randint(1, sys.maxsize)
                service2 = ToyService(messages[i] + randInt, cores=0.1)
                child = child.addChildJobFn(serviceAccessor,
                                            job.addService(service2, parentService=service),
                                            outFile, randInt, cores=0.1)
                service = service2

class ToyService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        While established the service repeatedly:
             - reads an integer i from the inJobStoreFileID file
             - writes i and messageInt to the outJobStoreFileID file
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        assert self.disk is not None
        assert self.memory is not None
        assert self.cores is not None
        self.terminate = Event()
        self.error = Event()
        # Note that service jobs are special and do not necessarily have job.jobStoreID.
        # So we don't associate these files with this job.
        inJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        outJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        self.serviceThread = Thread(target=self.serviceWorker,
                                    args=(job.fileStore.jobStore, self.terminate, self.error,
                                          inJobStoreID, outJobStoreID,
                                          self.messageInt))
        self.serviceThread.start()
        return (inJobStoreID, outJobStoreID)

    def stop(self, job):
        self.terminate.set()
        self.serviceThread.join()

    def check(self):
        if self.error.isSet():
            raise RuntimeError("Service worker failed")
        return True

    @staticmethod
    def serviceWorker(jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt):
        try:
            while True:
                if terminate.isSet(): # Quit if we've got the terminate signal
                    logger.debug("Demo service worker being told to quit")
                    return

                time.sleep(0.2) # Sleep to avoid thrashing

                # Try reading a line from the input file
                try:
                    with jobStore.readFileStream(inJobStoreID) as fH:
                        fH = codecs.getreader('utf-8')(fH)
                        line = fH.readline()
                except:
                    logger.debug("Something went wrong reading a line")
                    raise

                # Try converting the input line into an integer
                try:
                    inputInt = int(line)
                except ValueError:
                    logger.debug("Tried casting input line to integer but got error: %s" % line)
                    continue

                # Write out the resulting read integer and the message
                with jobStore.updateFileStream(outJobStoreID) as fH:
                    fH.write(("%s %s\n" % (inputInt, messageInt)).encode('utf-8'))
        except:
            error.set()
            raise

def serviceAccessor(job, communicationFiles, outFile, randInt):
    """
    Writes a random integer iinto the inJobStoreFileID file, then tries 10 times reading
    from outJobStoreFileID to get a pair of integers, the first equal to i the second written into the outputFile.
    """
    inJobStoreFileID, outJobStoreFileID = communicationFiles

    # Get a random integer
    key = random.randint(1, sys.maxsize)

    # Write the integer into the file
    logger.debug("Writing key to inJobStoreFileID")
    with job.fileStore.jobStore.updateFileStream(inJobStoreFileID) as fH:
        fH.write(("%s\n" % key).encode('utf-8'))

    logger.debug("Trying to read key and message from outJobStoreFileID")
    for i in range(10): # Try 10 times over
        time.sleep(0.2) #Avoid thrashing

        # Try reading an integer from the input file and writing out the message
        with job.fileStore.jobStore.readFileStream(outJobStoreFileID) as fH:
            fH = codecs.getreader('utf-8')(fH)
            line = fH.readline()

        tokens = line.split()
        if len(tokens) != 2:
            continue

        key2, message = tokens

        if int(key2) == key:
            logger.debug("Matched key's: %s, writing message: %s with randInt: %s" % (key, int(message) - randInt, randInt))
            with open(outFile, 'a') as fH:
                fH.write("%s\n" % (int(message) - randInt))
            return

    assert 0 # Job failed to get info from the service

class ToySerializableService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        Trivial service for testing serialization.
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        return self.messageInt

    def stop(self, job):
        pass

    def check(self):
        return True

def fnTest(strings, outputFile):
    """
    Function concatenates the strings together and writes them to the output file
    """
    with open(outputFile, 'w') as fH:
        fH.write(" ".join(strings))

<commit_msg>Report more info from and robustify the ToyService (#2883)<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, print_function
from builtins import zip
from builtins import map
from builtins import range
import codecs
import os
import random
import traceback
import unittest

# Python 3 compatibility imports
from six.moves import xrange

from toil.lib.bioio import getTempFile
from toil.job import Job
from toil.test import ToilTest, slow
import time
import sys
from threading import Thread, Event
import logging
logger = logging.getLogger( __name__ )
from unittest import skipIf
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
from toil.leader import FailedJobsException, DeadlockException

class JobServiceTest(ToilTest):
    """
    Tests testing the Job.Service class
    """

    @slow
    def testServiceSerialization(self):
        """
        Tests that a service can receive a promise without producing a serialization
        error.
        """
        job = Job()
        service = ToySerializableService("woot")
        startValue = job.addService(service) # Add a first service to job
        subService = ToySerializableService(startValue) # Now create a child of
        # that service that takes the start value promise from the parent service
        job.addService(subService, parentService=service) # This should work if
        # serialization on services is working correctly.

        self.runToil(job)

    @slow
    def testService(self, checkpoint=False):
        """
        Tests the creation of a Job.Service with random failures of the worker.
        """
        for test in range(2):
            outFile = getTempFile(rootDir=self._createTempDir()) # Temporary file
            messageInt = random.randint(1, sys.maxsize)
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTest, outFile, messageInt, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEqual(int(open(outFile, 'r').readline()), messageInt)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceDeadlock(self):
        """
        Creates a job with more services than maxServices, checks that deadlock is detected.
        """
        outFile = getTempFile(rootDir=self._createTempDir())
        try:
            def makeWorkflow():
                job = Job()
                r1 = job.addService(ToySerializableService("woot1"))
                r2 = job.addService(ToySerializableService("woot2"))
                r3 = job.addService(ToySerializableService("woot3"))
                job.addChildFn(fnTest, [ r1, r2, r3 ], outFile)
                return job

            # This should fail as too few services available
            try:
                self.runToil(makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5)
            except DeadlockException:
                print("Got expected deadlock exception")
            else:
                assert 0

            # This should pass, as adequate services available
            self.runToil(makeWorkflow(), maxServiceJobs=3)
            # Check we get expected output
            assert open(outFile, 'r').read() == "woot1 woot2 woot3"
        finally:
            os.remove(outFile)

    def testServiceWithCheckpoints(self):
        """
        Tests the creation of a Job.Service with random failures of the worker, making the root job use checkpointing to
        restart the subtree.
        """
        self.testService(checkpoint=True)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating a chain of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFile = getTempFile(rootDir=self._createTempDir())
            messages = [ random.randint(1, sys.maxsize) for i in range(3) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestRecursive, outFile, messages, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEqual(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceParallelRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating parallel chains of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFiles = [ getTempFile(rootDir=self._createTempDir()) for j in range(2) ]
            messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestParallelRecursive, outFiles, messageBundles, checkpoint=True)

                # Run the workflow repeatedly until success
                self.runToil(t, retryCount=2)

                # Check output
                for (messages, outFile) in zip(messageBundles, outFiles):
                    self.assertEqual(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                list(map(os.remove, outFiles))

    def runToil(self, rootJob, retryCount=1, badWorker=0.5, badWorkedFailInterval=0.05, maxServiceJobs=sys.maxsize, deadlockWait=60):
        # Create the runner for the workflow.
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = "DEBUG"

        options.retryCount = retryCount
        options.badWorker = badWorker
        options.badWorkerFailInterval = badWorkedFailInterval
        options.servicePollingInterval = 1
        options.maxServiceJobs = maxServiceJobs
        options.deadlockWait=deadlockWait

        # Run the workflow
        totalTrys = 0
        while True:
            try:
                Job.Runner.startToil(rootJob, options)
                break
            except FailedJobsException as e:
                i = e.numberOfFailedJobs
                if totalTrys > 40: #p(fail after this many restarts) = 0.5**32
                    self.fail() #Exceeded a reasonable number of restarts
                totalTrys += 1
                options.restart = True

def serviceTest(job, outFile, messageInt):
    """
    Creates one service and one accessing job, which communicate with two files to establish
    that both run concurrently.
    """
    #Clean out out-file
    open(outFile, 'w').close()
    randInt = random.randint(1, sys.maxsize) # We create a random number that is added to messageInt and subtracted by the serviceAccessor, to prove that
    # when service test is checkpointed and restarted there is never a connection made between an earlier service and later serviceAccessor, or vice versa.
    job.addChildJobFn(serviceAccessor, job.addService(ToyService(messageInt + randInt)), outFile, randInt)

def serviceTestRecursive(job, outFile, messages):
    """
    Creates a chain of services and accessing jobs, each paired together.
    """
    if len(messages) > 0:
        #Clean out out-file
        open(outFile, 'w').close()
        randInt = random.randint(1, sys.maxsize)
        service = ToyService(messages[0] + randInt)
        child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

        for i in range(1, len(messages)):
            randInt = random.randint(1, sys.maxsize)
            service2 = ToyService(messages[i] + randInt, cores=0.1)
            child = child.addChildJobFn(serviceAccessor,
                                        job.addService(service2, parentService=service),
                                        outFile, randInt, cores=0.1)
            service = service2

def serviceTestParallelRecursive(job, outFiles, messageBundles):
    """
    Creates multiple chains of services and accessing jobs.
    """
    for messages, outFile in zip(messageBundles, outFiles):
        #Clean out out-file
        open(outFile, 'w').close()
        if len(messages) > 0:
            randInt = random.randint(1, sys.maxsize)
            service = ToyService(messages[0] + randInt)
            child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

            for i in range(1, len(messages)):
                randInt = random.randint(1, sys.maxsize)
                service2 = ToyService(messages[i] + randInt, cores=0.1)
                child = child.addChildJobFn(serviceAccessor,
                                            job.addService(service2, parentService=service),
                                            outFile, randInt, cores=0.1)
                service = service2

class ToyService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        While established the service repeatedly:
             - reads an integer i from the inJobStoreFileID file
             - writes i and messageInt to the outJobStoreFileID file
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        assert self.disk is not None
        assert self.memory is not None
        assert self.cores is not None
        self.terminate = Event()
        self.error = Event()
        # Note that service jobs are special and do not necessarily have job.jobStoreID.
        # So we don't associate these files with this job.
        inJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        outJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        self.serviceThread = Thread(target=self.serviceWorker,
                                    args=(job.fileStore.jobStore, self.terminate, self.error,
                                          inJobStoreID, outJobStoreID,
                                          self.messageInt))
        self.serviceThread.start()
        return (inJobStoreID, outJobStoreID)

    def stop(self, job):
        self.terminate.set()
        self.serviceThread.join()

    def check(self):
        if self.error.isSet():
            raise RuntimeError("Service worker failed")
        return True

    @staticmethod
    def serviceWorker(jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt):
        try:
            while True:
                if terminate.isSet(): # Quit if we've got the terminate signal
                    logger.debug("Service worker being told to quit")
                    return

                time.sleep(0.2) # Sleep to avoid thrashing

                # Try reading a line from the input file
                try:
                    with jobStore.readFileStream(inJobStoreID) as fH:
                        fH = codecs.getreader('utf-8')(fH)
                        line = fH.readline()
                except:
                    logger.debug("Something went wrong reading a line: %s", traceback.format_exc())
                    raise

                if len(line.strip()) == 0:
                    # Don't try and make an integer out of nothing
                    continue

                # Try converting the input line into an integer
                try:
                    inputInt = int(line)
                except ValueError:
                    logger.debug("Tried casting input line '%s' to integer but got error: %s", line, traceback.format_exc())
                    continue

                # Write out the resulting read integer and the message
                with jobStore.updateFileStream(outJobStoreID) as fH:
                    fH.write(("%s %s\n" % (inputInt, messageInt)).encode('utf-8'))

                logger.debug("Service worker did useful work")
        except:
            logger.debug("Error in service worker: %s", traceback.format_exc())
            error.set()
            raise

def serviceAccessor(job, communicationFiles, outFile, randInt):
    """
    Writes a random integer iinto the inJobStoreFileID file, then tries 10 times reading
    from outJobStoreFileID to get a pair of integers, the first equal to i the second written into the outputFile.
    """
    inJobStoreFileID, outJobStoreFileID = communicationFiles

    # Get a random integer
    key = random.randint(1, sys.maxsize)

    # Write the integer into the file
    logger.debug("Writing key to inJobStoreFileID")
    with job.fileStore.jobStore.updateFileStream(inJobStoreFileID) as fH:
        fH.write(("%s\n" % key).encode('utf-8'))

    logger.debug("Trying to read key and message from outJobStoreFileID")
    for i in range(10): # Try 10 times over
        time.sleep(0.2) #Avoid thrashing

        # Try reading an integer from the input file and writing out the message
        with job.fileStore.jobStore.readFileStream(outJobStoreFileID) as fH:
            fH = codecs.getreader('utf-8')(fH)
            line = fH.readline()

        tokens = line.split()
        if len(tokens) != 2:
            continue

        key2, message = tokens

        if int(key2) == key:
            logger.debug("Matched key's: %s, writing message: %s with randInt: %s" % (key, int(message) - randInt, randInt))
            with open(outFile, 'a') as fH:
                fH.write("%s\n" % (int(message) - randInt))
            return

    assert 0 # Job failed to get info from the service

class ToySerializableService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        Trivial service for testing serialization.
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        return self.messageInt

    def stop(self, job):
        pass

    def check(self):
        return True

def fnTest(strings, outputFile):
    """
    Function concatenates the strings together and writes them to the output file
    """
    with open(outputFile, 'w') as fH:
        fH.write(" ".join(strings))

<|endoftext|>


<commit_before>from __future__ import print_function
# Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mimetypes
import sys
import os

from toil import subprocess
from toil.test import ToilTest, slow
from toil.test.mesos import helloWorld

class RegularLogTest(ToilTest):

    def setUp(self):
        super(RegularLogTest, self).setUp()
        self.tempDir = self._createTempDir(purpose='tempDir')

    def _getFiles(self, dir):
        return [os.path.join(dir, f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]

    def _assertFileTypeExists(self, dir, extension, encoding=None):
        # an encoding of None implies no compression
        onlyFiles = self._getFiles(dir)
        print(os.listdir(dir))
        onlyLogs = [f for f in onlyFiles if f.endswith(extension)]
        assert onlyLogs
        for log in onlyLogs:
            with open(log, "r") as f:
                if encoding == "gzip":
                    # Check for gzip magic header '\x1f\x8b'
                    assert f.read().startswith('\x1f\x8b')
                else:
                    mime = mimetypes.guess_type(log)
                    self.assertEqual(mime[1], encoding)

    @slow
    def testLogToMaster(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert helloWorld.childMessage in toilOutput.decode('utf-8')

    def testWriteLogs(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=debug',
                                              '--writeLogs=%s' % self.tempDir],
                                             stderr=subprocess.STDOUT)
        self._assertFileTypeExists(self.tempDir, '.log')

    @slow
    def testWriteGzipLogs(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=debug',
                                              '--writeLogsGzip=%s' % self.tempDir],
                                              stderr=subprocess.STDOUT)
        self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')

    @slow
    def testMultipleLogToMaster(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert helloWorld.parentMessage in toilOutput.decode('utf-8')

    def testRegularLog(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--batchSystem=singleMachine',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert "single machine batch system" in toilOutput.decode('utf-8')
<commit_msg>Read and inspect gzip as binary<commit_after>from __future__ import print_function
# Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mimetypes
import sys
import os

from toil import subprocess
from toil.test import ToilTest, slow
from toil.test.mesos import helloWorld

class RegularLogTest(ToilTest):

    def setUp(self):
        super(RegularLogTest, self).setUp()
        self.tempDir = self._createTempDir(purpose='tempDir')

    def _getFiles(self, dir):
        return [os.path.join(dir, f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]

    def _assertFileTypeExists(self, dir, extension, encoding=None):
        # an encoding of None implies no compression
        onlyFiles = self._getFiles(dir)
        print(os.listdir(dir))
        onlyLogs = [f for f in onlyFiles if f.endswith(extension)]
        assert onlyLogs
        for log in onlyLogs:
            with open(log, "rb") as f:
                if encoding == "gzip":
                    # Check for gzip magic header '\x1f\x8b'
                    assert f.read().startswith(b'\x1f\x8b')
                else:
                    mime = mimetypes.guess_type(log)
                    self.assertEqual(mime[1], encoding)

    @slow
    def testLogToMaster(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert helloWorld.childMessage in toilOutput.decode('utf-8')

    def testWriteLogs(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=debug',
                                              '--writeLogs=%s' % self.tempDir],
                                             stderr=subprocess.STDOUT)
        self._assertFileTypeExists(self.tempDir, '.log')

    @slow
    def testWriteGzipLogs(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=debug',
                                              '--writeLogsGzip=%s' % self.tempDir],
                                              stderr=subprocess.STDOUT)
        self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')

    @slow
    def testMultipleLogToMaster(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert helloWorld.parentMessage in toilOutput.decode('utf-8')

    def testRegularLog(self):
        toilOutput = subprocess.check_output([sys.executable,
                                              '-m', helloWorld.__name__,
                                              './toilTest',
                                              '--clean=always',
                                              '--batchSystem=singleMachine',
                                              '--logLevel=info'], stderr=subprocess.STDOUT)
        assert "single machine batch system" in toilOutput.decode('utf-8')
<|endoftext|>