<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import os
from lib import util
import pickle

import conarycfg
import versions
import metadata
from local import database
from repository import repository
from repository import changeset
from repository.netclient import NetworkRepositoryClient

class ClientError(Exception):
    """Base class for client errors"""

class TroveNotFound(Exception):
    def __init__(self, troveName):
        self.troveName = troveName

    def __str__(self):
        return "trove not found: %s" % self.troveName

class UpdateError(ClientError):
    """Base class for update errors"""

class VersionSuppliedError(UpdateError):
    def __str__(self):
        return "version should not be specified when a Conary change set " \
               "is being installed"

class NoNewTrovesError(UpdateError):
    def __str__(self):
        return "no new troves were found"

class UpdateChangeSet(changeset.ReadOnlyChangeSet):

    def merge(self, cs, src = None):
        changeset.ReadOnlyChangeSet.merge(self, cs)
        if isinstance(cs, UpdateChangeSet):
            self.contents += cs.contents
        else:
            self.contents.append(src)
        self.empty = False

    def __init__(self, *args):
        changeset.ReadOnlyChangeSet.__init__(self, *args)
        self.contents = []
        self.empty = True

class ConaryClient:
    def __init__(self, cfg = None):
        if cfg == None:
            cfg = conarycfg.ConaryConfiguration()

        cfg.installLabel = cfg.installLabelPath[0]
        self.cfg = cfg
        self.db = database.Database(cfg.root, cfg.dbPath)
        self.repos = NetworkRepositoryClient(cfg.repositoryMap,
                                             localRepository = self.db)

    def _rootChangeSet(self, cs, keepExisting = False):
	troveList = [ (x.getName(), x.getNewVersion(),
		       x.getNewFlavor())
			    for x in cs.iterNewPackageList() ]

	if keepExisting:
	    outdated = None
	else:
	    # this ignores eraseList, just like we do when trove names
	    # are specified
	    outdated, eraseList = self.db.outdatedTroves(troveList)

	    for key, tup in outdated.items():
		outdated[key] = tup[1:3]

	cs.rootChangeSet(self.db, outdated)

    def _resolveDependencies(self, cs, keepExisting = None, recurse = True):
        pathIdx = 0
        foundSuggestions = False
        (depList, cannotResolve) = self.db.depCheck(cs)
        suggMap = {}

        while depList and True:
            sugg = self.repos.resolveDependencies(
                            self.cfg.installLabelPath[pathIdx],
                            [ x[1] for x in depList ])

            if sugg:
                troves = {}

                for (troveName, depSet) in depList:
                    if sugg.has_key(depSet):
                        if suggMap.has_key(troveName):
                            suggMap[troveName] += sugg[depSet]
                        else:
                            suggMap[troveName] = sugg[depSet]

			suggList = [ (x[0], x[1], None) for x in sugg[depSet] ]
			troves.update(dict.fromkeys(suggList))

                troves = troves.keys()
                newCs = self._updateChangeSet(troves,
                                              keepExisting = keepExisting)
                cs.merge(newCs)

                (depList, cannotResolve) = self.db.depCheck(cs)

            if sugg and recurse:
                pathIdx = 0
                foundSuggestions = False
            else:
                pathIdx += 1
                if sugg:
                    foundSuggestions = True
                if pathIdx == len(self.cfg.installLabelPath):
                    if not foundSuggestions or not recurse:
                        return (cs, depList, suggMap, cannotResolve)
                    pathIdx = 0
                    foundSuggestions = False

        return (cs, depList, suggMap, cannotResolve)

    def _updateChangeSet(self, itemList, keepExisting = None, test = False):
        """
        Updates a trove on the local system to the latest version
        in the respository that the trove was initially installed from.

        @param itemList: List specifying the changes to apply. Each item
        in the list must be a ChangeSetFromFile, the name of a trove to
        update, or a (name, versionString) tuple.
        @type itemList: list
        """
        changeSetList = []
        finalCs = UpdateChangeSet()
        for item in itemList:
            if isinstance(item, changeset.ChangeSetFromFile):
                if item.isAbsolute():
		    self._rootChangeSet(item, keepExisting = keepExisting)

                finalCs.merge(item, (changeset.ChangeSetFromFile, item))

                continue

            if type(item) == str:
                troveName = item
                versionStr = None
            else:
                troveName = item[0]
                versionStr = item[1]
                flavor = item[2]

            if versionStr and versionStr[0] == '/':
                # fully qualified versions don't need repository affinity
                # or the label search path
                try:
                    newList = self.repos.findTrove(None, troveName,
                                                   self.cfg.flavor, versionStr,
                                                   withFiles = False,
                                                   affinityDatabase = self.db,
                                                   flavor = flavor)

                except repository.TroveNotFound, e:
                    # we give an error for this later on
                    newList = []
            else:
                newList = self.repos.findTrove(self.cfg.installLabelPath,
                                               troveName,
                                               self.cfg.flavor, versionStr,
                                               withFiles = False,
                                               affinityDatabase = self.db,
                                               flavor = flavor)
                    # XXX where does this go now?
                    # updating locally cooked troves needs a label override
                    #if True in [isinstance(x, versions.CookBranch) or
                    #            isinstance(x, versions.EmergeBranch)
                    #            for x in labels]:
                    #    if not versionStr:
                    #        raise UpdateError, \
                    #         "Package %s cooked locally; version, branch, or " \
                    #         "label must be specified for update" % troveName
                    #    else:
                    #        labels = [ None ]
                    #
                    #    pass

            if keepExisting:
                for newTrove in newList:
                    changeSetList.append((newTrove.getName(), (None, None),
                                (newTrove.getVersion(), newTrove.getFlavor()),
                                0))
                eraseList = []
            else:
                newItems = []
                for newTrove in newList:
                    newItems.append((newTrove.getName(), newTrove.getVersion(),
                                     newTrove.getFlavor()))

                # everything which needs to be installed is in this list; if
                # it's not here, it's a duplicate
                outdated, eraseList = self.db.outdatedTroves(newItems)
                for (name, newVersion, newFlavor), \
                      (oldName, oldVersion, oldFlavor) in outdated.iteritems():
                    changeSetList.append((name, (oldVersion, oldFlavor),
                                                (newVersion, newFlavor), 0))

        if finalCs.empty  and not changeSetList:
            raise NoNewTrovesError

        if changeSetList:
            cs = self.repos.createChangeSet(changeSetList, withFiles = False)
            finalCs.merge(cs, (self.repos.createChangeSet, changeSetList))

        return finalCs

    def updateChangeSet(self, itemList, keepExisting = False,
                        recurse = True, resolveDeps = True, test = False):
        if not test:
            self._prepareRoot()

        finalCs = self._updateChangeSet(itemList, keepExisting = keepExisting)

        if not resolveDeps:
            return (finalCs, [], {}, [])

        return self._resolveDependencies(finalCs, keepExisting = keepExisting,
                                         recurse = recurse)

    def applyChangeSet(self, cs, replaceFiles = False, tagScript = None,
                       keepExisting = None):
	assert(0)
        assert(isinstance(cs, changeset.ChangeSet))

	assert(not cs.isAbsolute)
        self.db.commitChangeSet(cs, replaceFiles = replaceFiles,
                                tagScript = tagScript,
                                keepExisting = keepExisting)

    def applyUpdate(self, theCs, replaceFiles = False, tagScript = None,
                    keepExisting = None, test = False, justDatabase = False,
                    journal = None):
        assert(isinstance(theCs, changeset.ReadOnlyChangeSet))
        cs = changeset.ReadOnlyChangeSet()
        for (how, what) in theCs.contents:
            if how == self.repos.createChangeSet:
                newCs = self.repos.createChangeSet(what)
                cs.merge(newCs)
            else:
                assert(how == changeset.ChangeSetFromFile)
                cs.merge(what)

        self.db.commitChangeSet(cs, replaceFiles = replaceFiles,
                                tagScript = tagScript,
                                keepExisting = keepExisting,
                                test = test, justDatabase = justDatabase,
                                journal = journal)

    def eraseTrove(self, troveList, depCheck = True, tagScript = None,
                   test = False, justDatabase = False):
	cs = changeset.ChangeSet()

        for (troveName, versionStr, flavor) in troveList:
            troves = self.db.findTrove(troveName, versionStr)

            for outerTrove in troves:
                for trove in self.db.walkTroveSet(outerTrove,
                                                 ignoreMissing = True):
                    if flavor is None or flavor.satisfies(trove.getFlavor()):
                        cs.oldPackage(trove.getName(), trove.getVersion(),
                                      trove.getFlavor())

        if depCheck:
            (depList, cannotResolve) = self.db.depCheck(cs)
            assert(not depList)
            if cannotResolve:
                return cannotResolve

	self.db.commitChangeSet(cs, tagScript = tagScript, test = test,
                                justDatabase = justDatabase)

    def getMetadata(self, troveList, label, cacheFile = None,
                    cacheOnly = False, saveOnly = False):
        md = {}
        if cacheFile and not saveOnly:
            try:
                cacheFp = open(cacheFile, "r")
                cache = pickle.load(cacheFp)
                cacheFp.close()
            except IOError, EOFError:
                if cacheOnly:
                    return {}
            else:
                lStr = label.asString()

                t = troveList[:]
                for troveName, branch in t:
                    bStr = branch.asString()

                    if lStr in cache and\
                       bStr in cache[lStr] and\
                       troveName in cache[lStr][bStr]:
                        md[troveName] = metadata.Metadata(cache[lStr][bStr][troveName])
                        troveList.remove((troveName, branch))

        # if the cache missed any, grab from the repos
        if not cacheOnly and troveList:
            md .update(self.repos.getMetadata(troveList, label))
            if md and cacheFile:
                try:
                    cacheFp = open(cacheFile, "rw")
                    cache = pickle.load(cacheFp)
                    cacheFp.close()
                except IOError, EOFError:
                    cache = {}

                cacheFp = open(cacheFile, "w")

                # filter down troveList to only contain items for which we found metadata
                cacheTroves = [x for x in troveList if x[0] in md]

                lStr = label.asString()
                for troveName, branch in cacheTroves:
                    bStr = branch.asString()

                    if lStr not in cache:
                        cache[lStr] = {}
                    if bStr not in cache[lStr]:
                        cache[lStr][bStr] = {}

                    cache[lStr][bStr][troveName] = md[troveName].freeze()

                pickle.dump(cache, cacheFp)
                cacheFp.close()

        return md

    def createBranch(self, newBranch, where, troveList = []):
        cs = changeset.ChangeSet()

	troveList = [ (x, where) for x in troveList ]

	branchedTroves = {}
	branchedFiles = {}
        dupList = []
        needsCommit = False

	while troveList:
            leavesByLabelOps = {}

            for (troveName, location) in troveList:
                if branchedTroves.has_key(troveName): continue
                branchedTroves[troveName] = 1

                l = leavesByLabelOps.get(location, None)
                if l is None:
                    l = []
                    leavesByLabelOps[location] = l
                l.append(troveName)

            # reset for the next pass
            troveList = []

            verDict = {}
            for (location, l) in leavesByLabelOps.iteritems():
                if isinstance(location, versions.Version):
                    for name in l:
                        l = verDict.get(name, None)
                        if l is None:
                            l = [ location ]
                            verDict[name] = l
                        else:
                            verDict[name].append(location)
                else:
                    verDict.update(
                            self.repos.getTroveLeavesByLabel(l, location))

            del leavesByLabelOps

            flavors = self.repos.getAllTroveFlavors(verDict)
            del verDict

            fullList = []
            for troveName in flavors.iterkeys():
                for (version, theFlavors) in flavors[troveName].iteritems():
                    fullList += [ (troveName, version, x) for x in theFlavors ]
            del flavors

            troves = self.repos.getTroves(fullList)
            branchedTroves = {}

	    for trove in troves:
                troveName = trove.getName()
		branchedVersion = trove.getVersion().createBranch(newBranch,
							         withVerRel = 1)

                branchedTrove = trove.copy()
		branchedTrove.changeVersion(branchedVersion)

		for (name, version, flavor) in trove.iterTroveList():
		    troveList.append((name, version))

		    branchedVersion = version.createBranch(newBranch,
                                                           withVerRel = 1)
		    branchedTrove.delTrove(name, version, flavor,
                                           missingOkay = False)
		    branchedTrove.addTrove(name, branchedVersion, flavor)

                key = (troveName, branchedVersion, trove.getFlavor())
                branchedTroves[key] = branchedTrove.diff(None)[0]

            # check for duplicates - XXX this could be more efficient with
            # a better repository API
            queryDict = {}
            for (name, version, flavor) in branchedTroves.iterkeys():
                if queryDict.has_key(name):
                    queryDict[name].append(version)
                else:
                    queryDict[name] = [ version ]

            matches = self.repos.getAllTroveFlavors(queryDict)

            for (name, version, flavor), troveCs in branchedTroves.iteritems():
                if matches.has_key(name) and matches[name].has_key(version) \
                   and flavor in matches[name][version]:
                    # this trove has already been branched
                    dupList.append((trove.getName(),
                                    trove.getVersion().branch()))
                else:
                    cs.newPackage(troveCs)
                    cs.addPrimaryPackage(name, version, flavor)
                    needsCommit = True

        if needsCommit:
            self.repos.commitChangeSet(cs)

	return dupList

    def _prepareRoot(self):
        """
        Prepares the installation root for trove updates and change
        set applications.
        """
        if not os.path.exists(self.cfg.root):
            util.mkdirChain(self.cfg.root)
        if not self.db.writeAccess():
            raise UpdateError, \
                "Write permission denied on conary database %s" % self.db.dbpath

<commit_msg>set flavor to None<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import os
from lib import util
import pickle

import conarycfg
import versions
import metadata
from local import database
from repository import repository
from repository import changeset
from repository.netclient import NetworkRepositoryClient

class ClientError(Exception):
    """Base class for client errors"""

class TroveNotFound(Exception):
    def __init__(self, troveName):
        self.troveName = troveName

    def __str__(self):
        return "trove not found: %s" % self.troveName

class UpdateError(ClientError):
    """Base class for update errors"""

class VersionSuppliedError(UpdateError):
    def __str__(self):
        return "version should not be specified when a Conary change set " \
               "is being installed"

class NoNewTrovesError(UpdateError):
    def __str__(self):
        return "no new troves were found"

class UpdateChangeSet(changeset.ReadOnlyChangeSet):

    def merge(self, cs, src = None):
        changeset.ReadOnlyChangeSet.merge(self, cs)
        if isinstance(cs, UpdateChangeSet):
            self.contents += cs.contents
        else:
            self.contents.append(src)
        self.empty = False

    def __init__(self, *args):
        changeset.ReadOnlyChangeSet.__init__(self, *args)
        self.contents = []
        self.empty = True

class ConaryClient:
    def __init__(self, cfg = None):
        if cfg == None:
            cfg = conarycfg.ConaryConfiguration()

        cfg.installLabel = cfg.installLabelPath[0]
        self.cfg = cfg
        self.db = database.Database(cfg.root, cfg.dbPath)
        self.repos = NetworkRepositoryClient(cfg.repositoryMap,
                                             localRepository = self.db)

    def _rootChangeSet(self, cs, keepExisting = False):
	troveList = [ (x.getName(), x.getNewVersion(),
		       x.getNewFlavor())
			    for x in cs.iterNewPackageList() ]

	if keepExisting:
	    outdated = None
	else:
	    # this ignores eraseList, just like we do when trove names
	    # are specified
	    outdated, eraseList = self.db.outdatedTroves(troveList)

	    for key, tup in outdated.items():
		outdated[key] = tup[1:3]

	cs.rootChangeSet(self.db, outdated)

    def _resolveDependencies(self, cs, keepExisting = None, recurse = True):
        pathIdx = 0
        foundSuggestions = False
        (depList, cannotResolve) = self.db.depCheck(cs)
        suggMap = {}

        while depList and True:
            sugg = self.repos.resolveDependencies(
                            self.cfg.installLabelPath[pathIdx],
                            [ x[1] for x in depList ])

            if sugg:
                troves = {}

                for (troveName, depSet) in depList:
                    if sugg.has_key(depSet):
                        if suggMap.has_key(troveName):
                            suggMap[troveName] += sugg[depSet]
                        else:
                            suggMap[troveName] = sugg[depSet]

			suggList = [ (x[0], x[1], None) for x in sugg[depSet] ]
			troves.update(dict.fromkeys(suggList))

                troves = troves.keys()
                newCs = self._updateChangeSet(troves,
                                              keepExisting = keepExisting)
                cs.merge(newCs)

                (depList, cannotResolve) = self.db.depCheck(cs)

            if sugg and recurse:
                pathIdx = 0
                foundSuggestions = False
            else:
                pathIdx += 1
                if sugg:
                    foundSuggestions = True
                if pathIdx == len(self.cfg.installLabelPath):
                    if not foundSuggestions or not recurse:
                        return (cs, depList, suggMap, cannotResolve)
                    pathIdx = 0
                    foundSuggestions = False

        return (cs, depList, suggMap, cannotResolve)

    def _updateChangeSet(self, itemList, keepExisting = None, test = False):
        """
        Updates a trove on the local system to the latest version
        in the respository that the trove was initially installed from.

        @param itemList: List specifying the changes to apply. Each item
        in the list must be a ChangeSetFromFile, the name of a trove to
        update, or a (name, versionString) tuple.
        @type itemList: list
        """
        changeSetList = []
        finalCs = UpdateChangeSet()
        for item in itemList:
            if isinstance(item, changeset.ChangeSetFromFile):
                if item.isAbsolute():
		    self._rootChangeSet(item, keepExisting = keepExisting)

                finalCs.merge(item, (changeset.ChangeSetFromFile, item))

                continue

            if type(item) == str:
                troveName = item
                versionStr = None
                flavor = None
            else:
                troveName = item[0]
                versionStr = item[1]
                flavor = item[2]

            if versionStr and versionStr[0] == '/':
                # fully qualified versions don't need repository affinity
                # or the label search path
                try:
                    newList = self.repos.findTrove(None, troveName,
                                                   self.cfg.flavor, versionStr,
                                                   withFiles = False,
                                                   affinityDatabase = self.db,
                                                   flavor = flavor)

                except repository.TroveNotFound, e:
                    # we give an error for this later on
                    newList = []
            else:
                newList = self.repos.findTrove(self.cfg.installLabelPath,
                                               troveName,
                                               self.cfg.flavor, versionStr,
                                               withFiles = False,
                                               affinityDatabase = self.db,
                                               flavor = flavor)
                    # XXX where does this go now?
                    # updating locally cooked troves needs a label override
                    #if True in [isinstance(x, versions.CookBranch) or
                    #            isinstance(x, versions.EmergeBranch)
                    #            for x in labels]:
                    #    if not versionStr:
                    #        raise UpdateError, \
                    #         "Package %s cooked locally; version, branch, or " \
                    #         "label must be specified for update" % troveName
                    #    else:
                    #        labels = [ None ]
                    #
                    #    pass

            if keepExisting:
                for newTrove in newList:
                    changeSetList.append((newTrove.getName(), (None, None),
                                (newTrove.getVersion(), newTrove.getFlavor()),
                                0))
                eraseList = []
            else:
                newItems = []
                for newTrove in newList:
                    newItems.append((newTrove.getName(), newTrove.getVersion(),
                                     newTrove.getFlavor()))

                # everything which needs to be installed is in this list; if
                # it's not here, it's a duplicate
                outdated, eraseList = self.db.outdatedTroves(newItems)
                for (name, newVersion, newFlavor), \
                      (oldName, oldVersion, oldFlavor) in outdated.iteritems():
                    changeSetList.append((name, (oldVersion, oldFlavor),
                                                (newVersion, newFlavor), 0))

        if finalCs.empty  and not changeSetList:
            raise NoNewTrovesError

        if changeSetList:
            cs = self.repos.createChangeSet(changeSetList, withFiles = False)
            finalCs.merge(cs, (self.repos.createChangeSet, changeSetList))

        return finalCs

    def updateChangeSet(self, itemList, keepExisting = False,
                        recurse = True, resolveDeps = True, test = False):
        if not test:
            self._prepareRoot()

        finalCs = self._updateChangeSet(itemList, keepExisting = keepExisting)

        if not resolveDeps:
            return (finalCs, [], {}, [])

        return self._resolveDependencies(finalCs, keepExisting = keepExisting,
                                         recurse = recurse)

    def applyChangeSet(self, cs, replaceFiles = False, tagScript = None,
                       keepExisting = None):
	assert(0)
        assert(isinstance(cs, changeset.ChangeSet))

	assert(not cs.isAbsolute)
        self.db.commitChangeSet(cs, replaceFiles = replaceFiles,
                                tagScript = tagScript,
                                keepExisting = keepExisting)

    def applyUpdate(self, theCs, replaceFiles = False, tagScript = None,
                    keepExisting = None, test = False, justDatabase = False,
                    journal = None):
        assert(isinstance(theCs, changeset.ReadOnlyChangeSet))
        cs = changeset.ReadOnlyChangeSet()
        for (how, what) in theCs.contents:
            if how == self.repos.createChangeSet:
                newCs = self.repos.createChangeSet(what)
                cs.merge(newCs)
            else:
                assert(how == changeset.ChangeSetFromFile)
                cs.merge(what)

        self.db.commitChangeSet(cs, replaceFiles = replaceFiles,
                                tagScript = tagScript,
                                keepExisting = keepExisting,
                                test = test, justDatabase = justDatabase,
                                journal = journal)

    def eraseTrove(self, troveList, depCheck = True, tagScript = None,
                   test = False, justDatabase = False):
	cs = changeset.ChangeSet()

        for (troveName, versionStr, flavor) in troveList:
            troves = self.db.findTrove(troveName, versionStr)

            for outerTrove in troves:
                for trove in self.db.walkTroveSet(outerTrove,
                                                 ignoreMissing = True):
                    if flavor is None or flavor.satisfies(trove.getFlavor()):
                        cs.oldPackage(trove.getName(), trove.getVersion(),
                                      trove.getFlavor())

        if depCheck:
            (depList, cannotResolve) = self.db.depCheck(cs)
            assert(not depList)
            if cannotResolve:
                return cannotResolve

	self.db.commitChangeSet(cs, tagScript = tagScript, test = test,
                                justDatabase = justDatabase)

    def getMetadata(self, troveList, label, cacheFile = None,
                    cacheOnly = False, saveOnly = False):
        md = {}
        if cacheFile and not saveOnly:
            try:
                cacheFp = open(cacheFile, "r")
                cache = pickle.load(cacheFp)
                cacheFp.close()
            except IOError, EOFError:
                if cacheOnly:
                    return {}
            else:
                lStr = label.asString()

                t = troveList[:]
                for troveName, branch in t:
                    bStr = branch.asString()

                    if lStr in cache and\
                       bStr in cache[lStr] and\
                       troveName in cache[lStr][bStr]:
                        md[troveName] = metadata.Metadata(cache[lStr][bStr][troveName])
                        troveList.remove((troveName, branch))

        # if the cache missed any, grab from the repos
        if not cacheOnly and troveList:
            md .update(self.repos.getMetadata(troveList, label))
            if md and cacheFile:
                try:
                    cacheFp = open(cacheFile, "rw")
                    cache = pickle.load(cacheFp)
                    cacheFp.close()
                except IOError, EOFError:
                    cache = {}

                cacheFp = open(cacheFile, "w")

                # filter down troveList to only contain items for which we found metadata
                cacheTroves = [x for x in troveList if x[0] in md]

                lStr = label.asString()
                for troveName, branch in cacheTroves:
                    bStr = branch.asString()

                    if lStr not in cache:
                        cache[lStr] = {}
                    if bStr not in cache[lStr]:
                        cache[lStr][bStr] = {}

                    cache[lStr][bStr][troveName] = md[troveName].freeze()

                pickle.dump(cache, cacheFp)
                cacheFp.close()

        return md

    def createBranch(self, newBranch, where, troveList = []):
        cs = changeset.ChangeSet()

	troveList = [ (x, where) for x in troveList ]

	branchedTroves = {}
	branchedFiles = {}
        dupList = []
        needsCommit = False

	while troveList:
            leavesByLabelOps = {}

            for (troveName, location) in troveList:
                if branchedTroves.has_key(troveName): continue
                branchedTroves[troveName] = 1

                l = leavesByLabelOps.get(location, None)
                if l is None:
                    l = []
                    leavesByLabelOps[location] = l
                l.append(troveName)

            # reset for the next pass
            troveList = []

            verDict = {}
            for (location, l) in leavesByLabelOps.iteritems():
                if isinstance(location, versions.Version):
                    for name in l:
                        l = verDict.get(name, None)
                        if l is None:
                            l = [ location ]
                            verDict[name] = l
                        else:
                            verDict[name].append(location)
                else:
                    verDict.update(
                            self.repos.getTroveLeavesByLabel(l, location))

            del leavesByLabelOps

            flavors = self.repos.getAllTroveFlavors(verDict)
            del verDict

            fullList = []
            for troveName in flavors.iterkeys():
                for (version, theFlavors) in flavors[troveName].iteritems():
                    fullList += [ (troveName, version, x) for x in theFlavors ]
            del flavors

            troves = self.repos.getTroves(fullList)
            branchedTroves = {}

	    for trove in troves:
                troveName = trove.getName()
		branchedVersion = trove.getVersion().createBranch(newBranch,
							         withVerRel = 1)

                branchedTrove = trove.copy()
		branchedTrove.changeVersion(branchedVersion)

		for (name, version, flavor) in trove.iterTroveList():
		    troveList.append((name, version))

		    branchedVersion = version.createBranch(newBranch,
                                                           withVerRel = 1)
		    branchedTrove.delTrove(name, version, flavor,
                                           missingOkay = False)
		    branchedTrove.addTrove(name, branchedVersion, flavor)

                key = (troveName, branchedVersion, trove.getFlavor())
                branchedTroves[key] = branchedTrove.diff(None)[0]

            # check for duplicates - XXX this could be more efficient with
            # a better repository API
            queryDict = {}
            for (name, version, flavor) in branchedTroves.iterkeys():
                if queryDict.has_key(name):
                    queryDict[name].append(version)
                else:
                    queryDict[name] = [ version ]

            matches = self.repos.getAllTroveFlavors(queryDict)

            for (name, version, flavor), troveCs in branchedTroves.iteritems():
                if matches.has_key(name) and matches[name].has_key(version) \
                   and flavor in matches[name][version]:
                    # this trove has already been branched
                    dupList.append((trove.getName(),
                                    trove.getVersion().branch()))
                else:
                    cs.newPackage(troveCs)
                    cs.addPrimaryPackage(name, version, flavor)
                    needsCommit = True

        if needsCommit:
            self.repos.commitChangeSet(cs)

	return dupList

    def _prepareRoot(self):
        """
        Prepares the installation root for trove updates and change
        set applications.
        """
        if not os.path.exists(self.cfg.root):
            util.mkdirChain(self.cfg.root)
        if not self.db.writeAccess():
            raise UpdateError, \
                "Write permission denied on conary database %s" % self.db.dbpath

<|endoftext|>


<commit_before>#!/usr/bin/env python3

## this script process trust+ domain blacklist file into configuration files for unbound
## TODO: multithreading, checking if unbound is installed, erase temporary file
import sys, subprocess, shlex, tempfile, os
import tldextract
import ipaddress
ip_addrs = []
unique_domains = []
subdomain_groups = {}
if len(sys.argv) == 1:
    sys.stderr.write("usage: {} domains-file output-conf-dir\n".format(sys.argv[0]))
    exit(1)
if not os.path.isfile(os.path.realpath(sys.argv[1])):
    sys.stderr.write("input file doesn't exists\n")
    exit(1)
if not os.path.isdir(os.path.realpath(sys.argv[2])):
    sys.stderr.write("output conf dir doesn't exists. creating it\n")
    os.mkdir(os.path.realpath(sys.argv[2])
cmdline = shlex.split("sed -e '/^\*\./d' {} ".format(sys.argv[1]))
cmdout = tempfile.mkstemp()
cmd0 = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd1 = subprocess.Popen(["sort"], stdin=cmd0.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd2 = subprocess.Popen(["uniq"], stdin=cmd1.stdout, stdout=cmdout[0], stderr=subprocess.STDOUT)
cmd0.wait()
cmd1.wait()
cmd2.wait()

with open(cmdout[1], 'r') as infile:
    progres = []
    for line in infile:
        #if line.startswith('*.'):
        #    continue
        line = line.strip()
        p = line[0]
        if p not in progres:
            print(p)
            progres.append(p)
        try:
            ipaddress.ip_address(line)
            ip_addrs.append(line)
            continue
        except ValueError:
            pass
        line_dom = tldextract.extract(line)
        if line_dom.registered_domain not in unique_domains and \
           line_dom.registered_domain not in subdomain_groups.keys():
            unique_domains.append(line_dom.registered_domain)
            #print(line_dom.registered_domain + " unique")
        else:
            if line_dom.registered_domain in unique_domains:
                unique_domains.remove(line_dom.registered_domain)
            #    print("create dom grp", line_dom.registered_domain)
                subdomain_groups[line_dom.registered_domain] = []
            subdomain_groups[line_dom.registered_domain].append(line)
            #print("add to grp", line)
os.remove(cmdout[1])
g = open(os.path.join(os.path.realpath(sys.argv[2]), "unique_domains.conf"))
for d in unique_domains:
    g.write('local-zone: "' + d + '" redirect\n')
    g.write('local-data: "' + d + ' IN A ' + redir_ip + '"\n')

for k in subdomain_groups.keys():
    h = open(k + '.conf','w')
    h.write('local-zone: "' + k + '" transparent\n')
    for d in subdoms[k]:
        h.write('local-data: "' + d + ' IN A ' + redir_ip + '"\n')
    h.close()
_ = [print(d) for d in unique_domains]
_ = [print(d) for d in ip_addrs]
for k,v in subdomain_groups:
    print("{} subdoms:")
    for e in v:
        print(e)

<commit_msg>fix variable initializations, fix file openings<commit_after>#!/usr/bin/env python3

## this script process trust+ domain blacklist file into configuration files for unbound
## TODO: multithreading, checking if unbound is installed, erase temporary file
import sys, subprocess, shlex, tempfile, os
import tldextract
import ipaddress

ip_addrs = []
unique_domains = []
subdomain_groups = {}
redir_ip = "127.0.1.1"

if len(sys.argv) == 1:
    sys.stderr.write("usage: {} domains-file output-conf-dir <redirect-address>\n".format(sys.argv[0]))
    exit(1)
if not os.path.isfile(os.path.realpath(sys.argv[1])):
    sys.stderr.write("input file doesn't exists\n")
    exit(1)
if not os.path.isdir(os.path.realpath(sys.argv[2])):
    sys.stderr.write("output conf dir doesn't exists. creating it\n")
    os.mkdir(os.path.realpath(sys.argv[2])
if len(sys.argv) == 4:
    redir_ip = sys.argv[3]

cmdline = shlex.split("sed -e '/^\*\./d' {} ".format(sys.argv[1]))
cmdout = tempfile.mkstemp()
cmd0 = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd1 = subprocess.Popen(["sort"], stdin=cmd0.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
cmd2 = subprocess.Popen(["uniq"], stdin=cmd1.stdout, stdout=cmdout[0], stderr=subprocess.STDOUT)
cmd0.wait()
cmd1.wait()
cmd2.wait()

with open(cmdout[1], 'r') as infile:
    progres = []
    for line in infile:
        #if line.startswith('*.'):
        #    continue
        line = line.strip()
        p = line[0]
        if p not in progres:
            print(p)
            progres.append(p)
        try:
            ipaddress.ip_address(line)
            ip_addrs.append(line)
            continue
        except ValueError:
            pass
        line_dom = tldextract.extract(line)
        if line_dom.registered_domain not in unique_domains and \
           line_dom.registered_domain not in subdomain_groups.keys():
            unique_domains.append(line_dom.registered_domain)
            #print(line_dom.registered_domain + " unique")
        else:
            if line_dom.registered_domain in unique_domains:
                unique_domains.remove(line_dom.registered_domain)
            #    print("create dom grp", line_dom.registered_domain)
                subdomain_groups[line_dom.registered_domain] = []
            subdomain_groups[line_dom.registered_domain].append(line)
            #print("add to grp", line)
os.remove(cmdout[1])
g = open(os.path.join(os.path.realpath(sys.argv[2]), "unique_domains.conf"),'w')
for d in unique_domains:
    g.write('local-zone: "' + d + '" redirect\n')
    g.write('local-data: "' + d + ' IN A ' + redir_ip + '"\n')

for k in subdomain_groups.keys():
    h = open(os.path.join(os.path.realpath(sys.argv[2]), '{}.conf'.format(k)),'w')
    h.write('local-zone: "' + k + '" transparent\n')
    for d in subdoms[k]:
        h.write('local-data: "' + d + ' IN A ' + redir_ip + '"\n')
    h.close()
_ = [print(d) for d in unique_domains]
_ = [print(d) for d in ip_addrs]
for k,v in subdomain_groups:
    print("{} subdoms:")
    for e in v:
        print(e)

<|endoftext|>


<commit_before><commit_msg>Made 'echo_top_tracking._find_nearby_tracks' faster.<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import platform
import sys

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import envWithPath, captureStdout, isMac, isWin, isMozBuild64, normExpUserPath, \
    shellify, vdump

if os.name == 'nt':
    COMPILE_NSPR_LIB = 'libnspr4.lib' if isMozBuild64 else 'nspr4.lib'
    COMPILE_PLDS_LIB = 'libplds4.lib' if isMozBuild64 else 'plds4.lib'
    COMPILE_PLC_LIB = 'libplc4.lib' if isMozBuild64 else 'plc4.lib'

    RUN_NSPR_LIB = 'libnspr4.dll' if isMozBuild64 else 'nspr4.dll'
    RUN_PLDS_LIB = 'libplds4.dll' if isMozBuild64 else 'plds4.dll'
    RUN_PLC_LIB = 'libplc4.dll' if isMozBuild64 else 'plc4.dll'
else:
    COMPILE_NSPR_LIB = 'libnspr4.a'
    COMPILE_PLDS_LIB = 'libplds4.a'
    COMPILE_PLC_LIB = 'libplc4.a'

    if platform.system() == 'Darwin':
        RUN_NSPR_LIB = 'libnspr4.dylib'
        RUN_PLDS_LIB = 'libplds4.dylib'
        RUN_PLC_LIB = 'libplc4.dylib'
    elif (platform.system() == 'Linux'):
        RUN_NSPR_LIB = 'libnspr4.so'
        RUN_PLDS_LIB = 'libplds4.so'
        RUN_PLC_LIB = 'libplc4.so'

ALL_COMPILE_LIBS = (COMPILE_NSPR_LIB, COMPILE_PLDS_LIB, COMPILE_PLC_LIB)
ALL_RUN_LIBS = (RUN_NSPR_LIB, RUN_PLDS_LIB, RUN_PLC_LIB)

def archOfBinary(binary):
    '''This function tests if a binary is 32-bit or 64-bit.'''
    unsplitFiletype = captureStdout(['file', binary])[0]
    filetype = unsplitFiletype.split(':', 1)[1]
    if isWin:
        assert 'PE executable for MS Windows (console)' in filetype
        return '32' if 'Intel 80386 32-bit' in filetype else '64'
    else:
        if 'universal binary' in filetype:
            raise Exception("I don't know how to deal with multiple-architecture binaries")
        if '386' in filetype or '32-bit' in filetype:
            assert '64-bit' not in filetype
            return '32'
        if '64-bit' in filetype:
            assert '32-bit' not in filetype
            return '64'

def constructVgCmdList(errorCode=77):
    '''Constructs default parameters needed to run valgrind with.'''
    vgCmdList = []
    vgCmdList.append('valgrind')
    if isMac:
        vgCmdList.append('--dsymutil=yes')
    vgCmdList.append('--error-exitcode=' + str(errorCode))
    vgCmdList.append('--smc-check=all-non-file')
    vgCmdList.append('--gen-suppressions=all')
    vgCmdList.append('--leak-check=full')
    vgCmdList.append('--show-possibly-lost=no')
    vgCmdList.append('--num-callers=50')
    return vgCmdList

def shellSupports(shellPath, args):
    '''
    This function returns True if the shell likes the args.
    You can support for a function, e.g. ['-e', 'foo()'], or a flag, e.g. ['-j', '-e', '42'].
    '''
    output, retCode = testBinary(shellPath, args, False, False)
    if retCode == 0:
        return True
    elif 1 <= retCode <= 3:
        # Exit codes 1 through 3 are all plausible "non-support":
        #   * "Usage error" is 1 in new js shell, 2 in old js shell, 2 in xpcshell.
        #   * "Script threw an error" is 3 in most shells, but 1 in some versions (see bug 751425).
        # Since we want autoBisect to support all shell versions, allow all these exit codes.
        return False
    else:
        raise Exception('Unexpected exit code in shellSupports ' + str(retCode))

def testBinary(shellPath, args, useValgrind, threadsafeShell):
    '''Tests the given shell with the given args.'''
    testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
    vdump('The testing command is: ' + shellify(testCmd))

    newEnv = envWithPath(os.path.dirname(os.path.abspath(shellPath)))
    if threadsafeShell:
        # The NSPR libraries needed to run threadsafe js shell should have already been be copied to
        # the same destination as the shell.
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_NSPR_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLDS_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLC_LIB)))
    out, rCode = captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True,
                               env=newEnv)
    vdump('The exit code is: ' + str(rCode))
    return out, rCode

def testDbgOrOpt(s):
    '''This function tests if a binary is a debug or optimized shell.'''
    # Do not use disassemble(), old shells prior to cc4fdccc1135 did not have disassemble(), and
    # it landed fairly recently on March 31, 2011. See bug 396512 comment 36.
    # The changeset's patch date is not reflective of its actual landing date.
    return 'dbg' if shellSupports(s, ['-e', 'dis()']) else 'opt'

def testGetBuildConfiguration(s):
    '''This function tests if a binary supports getBuildConfiguration().'''
    return shellSupports(s, ['-e', 'getBuildConfiguration()'])

def testGetBuildConfigurationWithThreadsafe(s):
    '''
    This function tests if a binary supports getBuildConfiguration() with threadsafe.
    See bug 791146 - getBuildConfiguration() returns the wrong value for gczeal and threadsafe
    '''
    ans = testBinary(s,
            ['-e', 'print(getBuildConfiguration().hasOwnProperty("threadsafe"))'], False, False)[0]
    return ans.find('true') != -1

def testJsShellOrXpcshell(s):
    '''This function tests if a binary is a js shell or xpcshell.'''
    return 'xpcshell' if shellSupports(s, ['-e', 'Components']) else 'jsShell'

def queryBuildConfiguration(s, parameter):
    '''Tests if a binary is compiled with specified parameters, in getBuildConfiguration().'''
    ans = testBinary(s, ['-e', 'print(getBuildConfiguration()["' + parameter + '"])'],
                     False, False)[0]
    return ans.find('true') != -1

def verifyBinary(sh, options):
    '''Verifies that the binary is compiled as intended.'''
    assert archOfBinary(sh.getShellBaseTempDirWithName()) == sh.buildOptions.arch
    assert testDbgOrOpt(sh.getShellBaseTempDirWithName()) == sh.buildOptions.compileType
    if testGetBuildConfiguration(sh.getShellBaseTempDirWithName()):
        if testGetBuildConfigurationWithThreadsafe(sh.getShellBaseTempDirWithName()):
            assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'threadsafe') == \
                options.isThreadsafe
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'more-deterministic') == \
            options.enableMoreDeterministic
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'rooting-analysis') == \
            options.enableRootAnalysis
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'exact-rooting') == \
            options.enableExactRooting
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'generational-gc') == \
            options.enableGcGenerational
<commit_msg>Match 'i386' rather than just '386' in the output from the 'file' command, in case a hash of the file happens to contain '386'.<commit_after>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

import os
import platform
import sys

path0 = os.path.dirname(os.path.abspath(__file__))
path1 = os.path.abspath(os.path.join(path0, os.pardir, 'util'))
sys.path.append(path1)
from subprocesses import envWithPath, captureStdout, isMac, isWin, isMozBuild64, normExpUserPath, \
    shellify, vdump

if os.name == 'nt':
    COMPILE_NSPR_LIB = 'libnspr4.lib' if isMozBuild64 else 'nspr4.lib'
    COMPILE_PLDS_LIB = 'libplds4.lib' if isMozBuild64 else 'plds4.lib'
    COMPILE_PLC_LIB = 'libplc4.lib' if isMozBuild64 else 'plc4.lib'

    RUN_NSPR_LIB = 'libnspr4.dll' if isMozBuild64 else 'nspr4.dll'
    RUN_PLDS_LIB = 'libplds4.dll' if isMozBuild64 else 'plds4.dll'
    RUN_PLC_LIB = 'libplc4.dll' if isMozBuild64 else 'plc4.dll'
else:
    COMPILE_NSPR_LIB = 'libnspr4.a'
    COMPILE_PLDS_LIB = 'libplds4.a'
    COMPILE_PLC_LIB = 'libplc4.a'

    if platform.system() == 'Darwin':
        RUN_NSPR_LIB = 'libnspr4.dylib'
        RUN_PLDS_LIB = 'libplds4.dylib'
        RUN_PLC_LIB = 'libplc4.dylib'
    elif (platform.system() == 'Linux'):
        RUN_NSPR_LIB = 'libnspr4.so'
        RUN_PLDS_LIB = 'libplds4.so'
        RUN_PLC_LIB = 'libplc4.so'

ALL_COMPILE_LIBS = (COMPILE_NSPR_LIB, COMPILE_PLDS_LIB, COMPILE_PLC_LIB)
ALL_RUN_LIBS = (RUN_NSPR_LIB, RUN_PLDS_LIB, RUN_PLC_LIB)

def archOfBinary(binary):
    '''This function tests if a binary is 32-bit or 64-bit.'''
    unsplitFiletype = captureStdout(['file', binary])[0]
    filetype = unsplitFiletype.split(':', 1)[1]
    if isWin:
        assert 'PE executable for MS Windows (console)' in filetype
        return '32' if 'Intel 80386 32-bit' in filetype else '64'
    else:
        if 'universal binary' in filetype:
            raise Exception("I don't know how to deal with multiple-architecture binaries")
        if '32-bit' in filetype or 'i386' in filetype:
            assert '64-bit' not in filetype
            return '32'
        if '64-bit' in filetype:
            assert '32-bit' not in filetype
            return '64'

def constructVgCmdList(errorCode=77):
    '''Constructs default parameters needed to run valgrind with.'''
    vgCmdList = []
    vgCmdList.append('valgrind')
    if isMac:
        vgCmdList.append('--dsymutil=yes')
    vgCmdList.append('--error-exitcode=' + str(errorCode))
    vgCmdList.append('--smc-check=all-non-file')
    vgCmdList.append('--gen-suppressions=all')
    vgCmdList.append('--leak-check=full')
    vgCmdList.append('--show-possibly-lost=no')
    vgCmdList.append('--num-callers=50')
    return vgCmdList

def shellSupports(shellPath, args):
    '''
    This function returns True if the shell likes the args.
    You can support for a function, e.g. ['-e', 'foo()'], or a flag, e.g. ['-j', '-e', '42'].
    '''
    output, retCode = testBinary(shellPath, args, False, False)
    if retCode == 0:
        return True
    elif 1 <= retCode <= 3:
        # Exit codes 1 through 3 are all plausible "non-support":
        #   * "Usage error" is 1 in new js shell, 2 in old js shell, 2 in xpcshell.
        #   * "Script threw an error" is 3 in most shells, but 1 in some versions (see bug 751425).
        # Since we want autoBisect to support all shell versions, allow all these exit codes.
        return False
    else:
        raise Exception('Unexpected exit code in shellSupports ' + str(retCode))

def testBinary(shellPath, args, useValgrind, threadsafeShell):
    '''Tests the given shell with the given args.'''
    testCmd = (constructVgCmdList() if useValgrind else []) + [shellPath] + args
    vdump('The testing command is: ' + shellify(testCmd))

    newEnv = envWithPath(os.path.dirname(os.path.abspath(shellPath)))
    if threadsafeShell:
        # The NSPR libraries needed to run threadsafe js shell should have already been be copied to
        # the same destination as the shell.
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_NSPR_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLDS_LIB)))
        assert os.path.isfile(normExpUserPath(os.path.join(
            os.path.dirname(os.path.abspath(shellPath)), RUN_PLC_LIB)))
    out, rCode = captureStdout(testCmd, combineStderr=True, ignoreStderr=True, ignoreExitCode=True,
                               env=newEnv)
    vdump('The exit code is: ' + str(rCode))
    return out, rCode

def testDbgOrOpt(s):
    '''This function tests if a binary is a debug or optimized shell.'''
    # Do not use disassemble(), old shells prior to cc4fdccc1135 did not have disassemble(), and
    # it landed fairly recently on March 31, 2011. See bug 396512 comment 36.
    # The changeset's patch date is not reflective of its actual landing date.
    return 'dbg' if shellSupports(s, ['-e', 'dis()']) else 'opt'

def testGetBuildConfiguration(s):
    '''This function tests if a binary supports getBuildConfiguration().'''
    return shellSupports(s, ['-e', 'getBuildConfiguration()'])

def testGetBuildConfigurationWithThreadsafe(s):
    '''
    This function tests if a binary supports getBuildConfiguration() with threadsafe.
    See bug 791146 - getBuildConfiguration() returns the wrong value for gczeal and threadsafe
    '''
    ans = testBinary(s,
            ['-e', 'print(getBuildConfiguration().hasOwnProperty("threadsafe"))'], False, False)[0]
    return ans.find('true') != -1

def testJsShellOrXpcshell(s):
    '''This function tests if a binary is a js shell or xpcshell.'''
    return 'xpcshell' if shellSupports(s, ['-e', 'Components']) else 'jsShell'

def queryBuildConfiguration(s, parameter):
    '''Tests if a binary is compiled with specified parameters, in getBuildConfiguration().'''
    ans = testBinary(s, ['-e', 'print(getBuildConfiguration()["' + parameter + '"])'],
                     False, False)[0]
    return ans.find('true') != -1

def verifyBinary(sh, options):
    '''Verifies that the binary is compiled as intended.'''
    assert archOfBinary(sh.getShellBaseTempDirWithName()) == sh.buildOptions.arch
    assert testDbgOrOpt(sh.getShellBaseTempDirWithName()) == sh.buildOptions.compileType
    if testGetBuildConfiguration(sh.getShellBaseTempDirWithName()):
        if testGetBuildConfigurationWithThreadsafe(sh.getShellBaseTempDirWithName()):
            assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'threadsafe') == \
                options.isThreadsafe
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'more-deterministic') == \
            options.enableMoreDeterministic
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'rooting-analysis') == \
            options.enableRootAnalysis
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'exact-rooting') == \
            options.enableExactRooting
        assert queryBuildConfiguration(sh.getShellBaseTempDirWithName(), 'generational-gc') == \
            options.enableGcGenerational
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from oauthlib.common import extract_params
from oauthlib.oauth1 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)

CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'

import sys
if sys.version > "3":
    unicode = str

    def to_native_str(string):
        return string.decode('utf-8')
else:
    def to_native_str(string):
        return string

# OBS!: Correct signing of requests are conditional on invoking OAuth1
# as the last step of preparing a request, or at least having the
# content-type set properly.
class OAuth1(object):
    """Signs the request using OAuth 1 (RFC5849)"""
    def __init__(self, client_key,
            client_secret=None,
            resource_owner_key=None,
            resource_owner_secret=None,
            callback_uri=None,
            signature_method=SIGNATURE_HMAC,
            signature_type=SIGNATURE_TYPE_AUTH_HEADER,
            rsa_key=None, verifier=None,
            decoding='utf-8'):

        try:
            signature_type = signature_type.upper()
        except AttributeError:
            pass

        self.client = Client(client_key, client_secret, resource_owner_key,
            resource_owner_secret, callback_uri, signature_method,
            signature_type, rsa_key, verifier, decoding=decoding)

    def __call__(self, r):
        """Add OAuth parameters to the request.

        Parameters may be included from the body if the content-type is
        urlencoded, if no content type is set a guess is made.
        """
        # Overwriting url is safe here as request will not modify it past
        # this point.

        content_type = r.headers.get('Content-Type', '')
        if not content_type and extract_params(r.body):
            content_type = CONTENT_TYPE_FORM_URLENCODED
        if not isinstance(content_type, unicode):
            content_type = content_type.decode('utf-8')

        is_form_encoded = (CONTENT_TYPE_FORM_URLENCODED in content_type)

        if is_form_encoded:
            r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED
            r.url, headers, r.body = self.client.sign(
                unicode(r.url), unicode(r.method), r.body or '', r.headers)
        else:
            # Omit body data in the signing of non form-encoded requests
            r.url, headers, _ = self.client.sign(
                unicode(r.url), unicode(r.method), None, r.headers)

        r.prepare_headers(headers)
        r.url = to_native_str(r.url)
        return r
<commit_msg>Remove decode in wrong place.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from requests.compat import is_py3
from requests.utils import to_native_string
from oauthlib.common import extract_params
from oauthlib.oauth1 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)

CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'

if is_py3:
    unicode = str

# OBS!: Correct signing of requests are conditional on invoking OAuth1
# as the last step of preparing a request, or at least having the
# content-type set properly.
class OAuth1(object):
    """Signs the request using OAuth 1 (RFC5849)"""
    def __init__(self, client_key,
            client_secret=None,
            resource_owner_key=None,
            resource_owner_secret=None,
            callback_uri=None,
            signature_method=SIGNATURE_HMAC,
            signature_type=SIGNATURE_TYPE_AUTH_HEADER,
            rsa_key=None, verifier=None,
            decoding='utf-8'):

        try:
            signature_type = signature_type.upper()
        except AttributeError:
            pass

        self.client = Client(client_key, client_secret, resource_owner_key,
            resource_owner_secret, callback_uri, signature_method,
            signature_type, rsa_key, verifier, decoding=decoding)

    def __call__(self, r):
        """Add OAuth parameters to the request.

        Parameters may be included from the body if the content-type is
        urlencoded, if no content type is set a guess is made.
        """
        # Overwriting url is safe here as request will not modify it past
        # this point.

        content_type = r.headers.get('Content-Type', '')
        if not content_type and extract_params(r.body):
            content_type = CONTENT_TYPE_FORM_URLENCODED
        if not isinstance(content_type, unicode):
            content_type = content_type.decode('utf-8')

        is_form_encoded = (CONTENT_TYPE_FORM_URLENCODED in content_type)

        if is_form_encoded:
            r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED
            r.url, headers, r.body = self.client.sign(
                unicode(r.url), unicode(r.method), r.body or '', r.headers)
        else:
            # Omit body data in the signing of non form-encoded requests
            r.url, headers, _ = self.client.sign(
                unicode(r.url), unicode(r.method), None, r.headers)

        r.prepare_headers(headers)
        r.url = to_native_string(r.url)
        return r
<|endoftext|>


<commit_before># --------------------------------------------------------------------------------------
# Copyright 2016, Benedikt J. Daurer, Filipe R.N.C. Maia, Max F. Hantke, Carl Nettelblad
# Hummingbird is distributed under the terms of the Simplified BSD License.
# -------------------------------------------------------------------------
from __future__ import print_function, absolute_import # Compatibility with python 2 and 3
import numpy as np
from backend import ureg
from backend import add_record
import h5py

import analysis.cfel_geom

_nGains = 3
_nPanels = 16
_nCells = 30

_agipd_calibrator = None
def init_calib(filenames):
    global _agipd_calibrator
    _agipd_calibrator = AGIPD_Calibrator(filenames)

_agipd_yx         = None
_agipd_slap_shape = None
_agipd_img_shape  = None
_agipd_rot180     = False
def init_geom(filename, rot180=False):
    global _agipd_yx, _agipd_slap_shape, _agipd_img_shape, _agipd_rot180
    _agipd_yx, _agipd_slap_shape, _agipd_img_shape = analysis.cfel_geom.pixel_maps_for_image_view(geometry_filename=filename)
    _agipd_rot180 = rot180

def getAGIPD(evt, record, cellID=None, panelID=None, calibrate=True, assemble=False, copy=True):
    """
    Returns individual panels or the entire frame of the AGIPD.

    For obtaining calibrated data set calibrate=True and make sure the calibration data is initialised
    by calling init_agipd(filename=path_to_calibration_file) before.

    For obtaining assembled data set assemble=True and make sure the geometry data is initalised by
    calling init_geom(filename=location_of_calibration_file) before.

    """

    nPanels = record.data.shape[1]
    is_isolated_panel = panelID is not None
    if panelID is None and nPanels == 1:
        print("ERROR: Please provide a panelID to identify the panel that shall be processed.")
        return

    if is_isolated_panel:
        aduData  = record.data[0][0 if nPanels == 1 else panelID]
        gainData = record.data[1][0 if nPanels == 1 else panelID]
        calData = np.array(aduData, copy=copy, dtype=np.int32)
        if calibrate:
            _agipd_calibrator.calibrate_panel(aduData=calData, gainData=gainData,
                                              panelID=panelID, cellID=cellID)
            return add_record(evt['analysis'], 'analysis', 'AGIPD_panel_%d_cal' %(panelID), calData)
        else:
            return add_record(evt['analysis'], 'analysis', 'AGIPD_panel_%d_raw' %(panelID), aduData)
    else:
        aduData  = record.data[0]
        gainData = record.data[1]
        calData = np.array(aduData, copy=copy, dtype=np.int32)
        if calibrate:
            _agipd_calibrator.calibrate_all_panels(aduData=calData, gainData=gainData, cellID=cellID)
        if assemble:
            img = np.zeros(shape=_agipd_img_shape, dtype=calData.dtype)
            img[_agipd_yx[0], _agipd_yx[1]] = calData.ravel()
            if _agipd_rot180:
                img = img[::-1, ::-1]
            return add_record(evt['analysis'], 'analysis', 'AGIPD_assembled', img)
        else:
            return add_record(evt['analysis'], 'analysis', 'AGIPD', calData)

class AGIPD_Calibrator:

    def __init__(self, filenames):
        assert len(filenames) == _nPanels
        self._badpixData       = []
        self._darkOffsetData   = []
        self._relativeGainData = []
        self._gainLevelData    = []
        for filename in sorted(filenames):
            self._read_and_append_calibration_data(filename=filename)
        self._badpixData       = np.asarray(self._badpixData)
        self._darkOffsetData   = np.asarray(self._darkOffsetData)
        self._relativeGainData = np.asarray(self._relativeGainData)
        self._gainLevelData    = np.asarray(self._gainLevelData)

    def _read_and_append_calibration_data(self, filename):
        # Meaning of indices: (gainID, cellID, pixcol, pixrow)
        # Anton's AGIPD calibration format
        #> h5ls calib/agipd/Cheetah-AGIPD00-calib.h5
        #AnalogOffset             Dataset {3, 32, 512, 128} H5T_STD_I16LE
        #Badpixel                 Dataset {3, 32, 512, 128} H5T_STD_U8LE
        #DigitalGainLevel         Dataset {3, 32, 512, 128} H5T_STD_U16LE
        #RelativeGain             Dataset {3, 32, 512, 128} H5T_IEEE_F32LE
        with h5py.File(filename) as f:
            self._badpixData.append(np.asarray(f["/Badpixel"]))
            self._darkOffsetData.append(np.asarray(f["/AnalogOffset"]))
            self._relativeGainData.append(np.asarray(f["/RelativeGain"]))
            self._gainLevelData.append(np.asarray(f["/DigitalGainLevel"]))

    def calibrate_all_panels(self, aduData, gainData, cellID, apply_gain_switch=False):
        # WARNING: aduData is overwritten!
        badpixMask = np.empty(shape=(32, 512, 128), dtype='bool')

        for panelID in range(_nPanels):
            self.calibrate_panel(aduData=aduData[panelID],
                                 gainData=gainData[panelID],
                                 panelID=panelID,
                                 cellID=cellID,
                                 apply_gain_switch=apply_gain_switch, mask_write_to=badpixMask[panelID])

    def calibrate_panel(self, aduData, gainData, cellID, panelID, apply_gain_switch=False, mask_write_to=None):
        # WARNING: aduData is overwritten!
        apply_gain_switch = True
        assert aduData is not None
        assert gainData is not None
        assert (0 <= cellID < _nCells)
        assert (0 <= panelID < _nPanels)
        assert str(aduData.dtype) == 'int32'

        cellBadpix       = np.asarray([self._badpixData[panelID][g][cellID] for g in range(_nGains)])
        cellDarkOffset   = np.asarray([self._darkOffsetData[panelID][g][cellID] for g in range(_nGains)])
        cellRelativeGain = np.asarray([self._relativeGainData[panelID][g][cellID] for g in range(_nGains)])
        cellGainLevel    = np.asarray([self._gainLevelData[panelID][g][cellID] for g in range(_nGains)])

        if mask_write_to is not None:
            badpixMask = mask_write_to
        else:
            badpixMask = np.empty(shape=aduData.shape, dtype='bool')

        if apply_gain_switch:
            # Option: bypass multi-gain calibration
            # In this case use only the gain0 offset
            aduData -= cellDarkOffset[0]
        else:
	    # Determine which gain stage by thresholding
            # Thresholding for gain level 3 is dodgy - thresholds merge
            # Ignore gain level 3 for now and work on levels 0,1
            pixGainLevel0 = gainData < cellGainLevel[1]
            pixGainLevel1 = pixGainLevel0 == False
            # Do not use last gain level
            pixGainLevel2 = np.zeros(shape=aduData.shape, dtype='bool')

            pixGainLevels = [pixGainLevel0, pixGainLevel1, pixGainLevel2]

            # Subtract the appropriate offset
            for g, pixGain in enumerate(pixGainLevels):
                if not pixGain.any():
                    continue
                aduData[pixGain] = aduData[pixGain] - cellDarkOffset[g][pixGain]
                aduData[pixGain] = aduData[pixGain] * cellRelativeGain[g][pixGain]
                badpixMask[pixGain] = (cellBadpix[g][pixGain] != 0)

<commit_msg>Turn on gain switching<commit_after># --------------------------------------------------------------------------------------
# Copyright 2016, Benedikt J. Daurer, Filipe R.N.C. Maia, Max F. Hantke, Carl Nettelblad
# Hummingbird is distributed under the terms of the Simplified BSD License.
# -------------------------------------------------------------------------
from __future__ import print_function, absolute_import # Compatibility with python 2 and 3
import numpy as np
from backend import ureg
from backend import add_record
import h5py

import analysis.cfel_geom

_nGains = 3
_nPanels = 16
_nCells = 30

_agipd_calibrator = None
def init_calib(filenames):
    global _agipd_calibrator
    _agipd_calibrator = AGIPD_Calibrator(filenames)

_agipd_yx         = None
_agipd_slap_shape = None
_agipd_img_shape  = None
_agipd_rot180     = False
def init_geom(filename, rot180=False):
    global _agipd_yx, _agipd_slap_shape, _agipd_img_shape, _agipd_rot180
    _agipd_yx, _agipd_slap_shape, _agipd_img_shape = analysis.cfel_geom.pixel_maps_for_image_view(geometry_filename=filename)
    _agipd_rot180 = rot180

def getAGIPD(evt, record, cellID=None, panelID=None, calibrate=True, assemble=False, copy=True):
    """
    Returns individual panels or the entire frame of the AGIPD.

    For obtaining calibrated data set calibrate=True and make sure the calibration data is initialised
    by calling init_agipd(filename=path_to_calibration_file) before.

    For obtaining assembled data set assemble=True and make sure the geometry data is initalised by
    calling init_geom(filename=location_of_calibration_file) before.

    """

    nPanels = record.data.shape[1]
    is_isolated_panel = panelID is not None
    if panelID is None and nPanels == 1:
        print("ERROR: Please provide a panelID to identify the panel that shall be processed.")
        return

    if is_isolated_panel:
        aduData  = record.data[0][0 if nPanels == 1 else panelID]
        gainData = record.data[1][0 if nPanels == 1 else panelID]
        calData = np.array(aduData, copy=copy, dtype=np.int32)
        if calibrate:
            _agipd_calibrator.calibrate_panel(aduData=calData, gainData=gainData,
                                              panelID=panelID, cellID=cellID)
            return add_record(evt['analysis'], 'analysis', 'AGIPD_panel_%d_cal' %(panelID), calData)
        else:
            return add_record(evt['analysis'], 'analysis', 'AGIPD_panel_%d_raw' %(panelID), aduData)
    else:
        aduData  = record.data[0]
        gainData = record.data[1]
        calData = np.array(aduData, copy=copy, dtype=np.int32)
        if calibrate:
            _agipd_calibrator.calibrate_all_panels(aduData=calData, gainData=gainData, cellID=cellID)
        if assemble:
            img = np.zeros(shape=_agipd_img_shape, dtype=calData.dtype)
            img[_agipd_yx[0], _agipd_yx[1]] = calData.ravel()
            if _agipd_rot180:
                img = img[::-1, ::-1]
            return add_record(evt['analysis'], 'analysis', 'AGIPD_assembled', img)
        else:
            return add_record(evt['analysis'], 'analysis', 'AGIPD', calData)

class AGIPD_Calibrator:

    def __init__(self, filenames):
        assert len(filenames) == _nPanels
        self._badpixData       = []
        self._darkOffsetData   = []
        self._relativeGainData = []
        self._gainLevelData    = []
        for filename in sorted(filenames):
            self._read_and_append_calibration_data(filename=filename)
        self._badpixData       = np.asarray(self._badpixData)
        self._darkOffsetData   = np.asarray(self._darkOffsetData)
        self._relativeGainData = np.asarray(self._relativeGainData)
        self._gainLevelData    = np.asarray(self._gainLevelData)

    def _read_and_append_calibration_data(self, filename):
        # Meaning of indices: (gainID, cellID, pixcol, pixrow)
        # Anton's AGIPD calibration format
        #> h5ls calib/agipd/Cheetah-AGIPD00-calib.h5
        #AnalogOffset             Dataset {3, 32, 512, 128} H5T_STD_I16LE
        #Badpixel                 Dataset {3, 32, 512, 128} H5T_STD_U8LE
        #DigitalGainLevel         Dataset {3, 32, 512, 128} H5T_STD_U16LE
        #RelativeGain             Dataset {3, 32, 512, 128} H5T_IEEE_F32LE
        with h5py.File(filename) as f:
            self._badpixData.append(np.asarray(f["/Badpixel"]))
            self._darkOffsetData.append(np.asarray(f["/AnalogOffset"]))
            self._relativeGainData.append(np.asarray(f["/RelativeGain"]))
            self._gainLevelData.append(np.asarray(f["/DigitalGainLevel"]))

    def calibrate_all_panels(self, aduData, gainData, cellID, apply_gain_switch=False):
        # WARNING: aduData is overwritten!
        badpixMask = np.empty(shape=(32, 512, 128), dtype='bool')

        for panelID in range(_nPanels):
            self.calibrate_panel(aduData=aduData[panelID],
                                 gainData=gainData[panelID],
                                 panelID=panelID,
                                 cellID=cellID,
                                 apply_gain_switch=apply_gain_switch, mask_write_to=badpixMask[panelID])

    def calibrate_panel(self, aduData, gainData, cellID, panelID, apply_gain_switch=False, mask_write_to=None):
        # WARNING: aduData is overwritten!
        apply_gain_switch = False
        assert aduData is not None
        assert gainData is not None
        assert (0 <= cellID < _nCells)
        assert (0 <= panelID < _nPanels)
        assert str(aduData.dtype) == 'int32'

        cellBadpix       = np.asarray([self._badpixData[panelID][g][cellID] for g in range(_nGains)])
        cellDarkOffset   = np.asarray([self._darkOffsetData[panelID][g][cellID] for g in range(_nGains)])
        cellRelativeGain = np.asarray([self._relativeGainData[panelID][g][cellID] for g in range(_nGains)])
        cellGainLevel    = np.asarray([self._gainLevelData[panelID][g][cellID] for g in range(_nGains)])

        if mask_write_to is not None:
            badpixMask = mask_write_to
        else:
            badpixMask = np.empty(shape=aduData.shape, dtype='bool')

        if apply_gain_switch:
            # Option: bypass multi-gain calibration
            # In this case use only the gain0 offset
            aduData -= cellDarkOffset[0]
        else:
	    # Determine which gain stage by thresholding
            # Thresholding for gain level 3 is dodgy - thresholds merge
            # Ignore gain level 3 for now and work on levels 0,1
            pixGainLevel0 = gainData < cellGainLevel[1]
            pixGainLevel1 = pixGainLevel0 == False
            # Do not use last gain level
            pixGainLevel2 = np.zeros(shape=aduData.shape, dtype='bool')

            pixGainLevels = [pixGainLevel0, pixGainLevel1, pixGainLevel2]

            # Subtract the appropriate offset
            for g, pixGain in enumerate(pixGainLevels):
                if not pixGain.any():
                    continue
                aduData[pixGain] = aduData[pixGain] - cellDarkOffset[g][pixGain]
                aduData[pixGain] = aduData[pixGain] * cellRelativeGain[g][pixGain]
                badpixMask[pixGain] = (cellBadpix[g][pixGain] != 0)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
import requests
from bs4 import BeautifulSoup

from .common import *

def get_admin_announces():
    """Get announces from the administration office of University of Tokyo"""
    url="http://www.c.u-tokyo.ac.jp/zenki/news/kyoumu/index.html"
    host=url.split("/zenki", 2)[0]

    res=requests.get(url)
    soup=BeautifulSoup(res.text)
    # updated
    updated = soup.find('p', {'id': 'update'})
    updated = updated.text.strip().split(u'：')[1]
    # announces
    data=[]
    newslist=soup.find("div", id="newslist2").find('dl')
    for line in newslist.find_all(['dt', 'dd']):
        if line == "\n":
            continue
        if str(line).startswith("<dt>"):
            imgs = line.find_all('img')
            date = get_text_if_exists(line.contents[0])
            data.append(
                    {"date"       : date,
                     "kind_image" : host + imgs[0].attrs["src"],
                     "grade_image": host + imgs[1].attrs["src"],
                    },
            )
        elif str(line).startswith("<dd>"):
            href = line.contents[0].attrs['href']
            if not href.startswith('http'):
                href = host + line.contents[0].attrs["href"]
            data[len(data)-1]["href"] = href
            data[len(data)-1]["announce"] = line.contents[0].string

    return updated, data

<commit_msg>[kyomu.py] strip text<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
import requests
from bs4 import BeautifulSoup

from .common import get_text_if_exists

def get_admin_announces():
    """Get announces from the administration office of University of Tokyo"""
    url = "http://www.c.u-tokyo.ac.jp/zenki/news/kyoumu/index.html"
    host = url.split("/zenki", 2)[0]

    res = requests.get(url)
    soup = BeautifulSoup(res.text)
    # updated
    updated = soup.find('p', {'id': 'update'})
    updated = updated.text.strip().split(u'：')[1]
    # announces
    data=[]
    newslist=soup.find("div", id="newslist2").find('dl')
    for line in newslist.find_all(['dt', 'dd']):
        if line == "\n":
            continue
        if str(line).startswith("<dt>"):
            imgs = line.find_all('img')
            date = get_text_if_exists(line.contents[0]).strip()
            data.append(
                    {"date"       : date,
                     "kind_image" : host + imgs[0].attrs["src"],
                     "grade_image": host + imgs[1].attrs["src"],
                    },
            )
        elif str(line).startswith("<dd>"):
            href = line.contents[0].attrs['href']
            if not href.startswith('http'):
                href = host + line.contents[0].attrs["href"]
            data[len(data)-1]["href"] = href
            data[len(data)-1]["announce"] = line.contents[0].string

    return updated, data

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import argparse
import pypng.code.png as png
import numpy.random as nprnd

#defaults:
defwidth = 1920
defheight = 1080
defnumFiles = 600

def arguments():
	"""
	parsing command line arguments
	"""
	parser = argparse.ArgumentParser(description="Generate random PNG images. Each pixel will be a random combination of either 0% or 100% red, green and blue.")
	parser.add_argument("-d", "--dim", type=int, nargs=2, dest="dim", default=[defwidth, defheight], help="Dimension (width, heigth) of images")
	parser.add_argument("-n", "--numFiles", type=int, dest="numFiles", default=defnumFiles, help="Number of images")
	parser.add_argument("-p", "--no-png", action="store_false", dest="png", default=True, help="Don't create PNG-images (re-use)")
	parser.add_argument("-b", "--build", action="store_true", dest="build", default=False, help="Also build video")
	parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Print debug info")
	parser.add_argument("-s", "--stats", action="store_true", dest="stats", default=False, help="Compute statistics about color distribution (not yet implemented)")
	return parser.parse_args()

def main(args):
	"""
	create n random png-images wxh pixels
	"""
	numdigits = len(str(args.numFiles))
	strformat = "b{:0"+str(numdigits)+"d}.png"
	width = args.dim[0]
	height = args.dim[1]

	if args.png:
		if args.verbose:
			print("creating PNG-images")

		for n in range(args.numFiles):
			fname = strformat.format(n)
			if args.verbose:
				print(fname)
			f = open(fname, 'wb')
			w = png.Writer(width, height)

			dat = []

			for i in range(height):
				dat.append(list(nprnd.randint(2, size = 3 * width) * 255))

			w.write(f, dat)
			f.close()

	if args.build:
		if args.verbose:
			print("creating video")
		import subprocess
		cmd = "ffmpeg -framerate 60 -i b%0" + str(numdigits) + "d.png " \
			+ "-s " + str(width) + "x" + str(height) + " " \
			+ "-pix_fmt yuv420p -preset slow -crf 20 " \
			+ "video.mp4"
		p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE)
		while True:
			out = p.stderr.read(1)
			if out == '' and p.poll() != None:
				break
			if out != '':
				sys.stdout.write(out)
				sys.stdout.flush()

	if args.verbose:
		print("done")

if __name__ == '__main__':
	args = arguments()
	main(args)

<commit_msg>added h264 profile settings<commit_after>#!/usr/bin/env python

import sys
import argparse
import pypng.code.png as png
import numpy.random as nprnd

#defaults:
defwidth = 1920
defheight = 1080
defnumFiles = 600

def arguments():
	"""
	parsing command line arguments
	"""
	parser = argparse.ArgumentParser(description="Generate random PNG images. Each pixel will be a random combination of either 0% or 100% red, green and blue.")
	parser.add_argument("-d", "--dim", type=int, nargs=2, dest="dim", default=[defwidth, defheight], help="Dimension (width, heigth) of images")
	parser.add_argument("-n", "--numFiles", type=int, dest="numFiles", default=defnumFiles, help="Number of images")
	parser.add_argument("-p", "--no-png", action="store_false", dest="png", default=True, help="Don't create PNG-images (re-use)")
	parser.add_argument("-b", "--build", action="store_true", dest="build", default=False, help="Also build video")
	parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Print debug info")
	parser.add_argument("-s", "--stats", action="store_true", dest="stats", default=False, help="Compute statistics about color distribution (not yet implemented)")
	return parser.parse_args()

def main(args):
	"""
	create n random png-images wxh pixels
	"""
	numdigits = len(str(args.numFiles))
	strformat = "b{:0"+str(numdigits)+"d}.png"
	width = args.dim[0]
	height = args.dim[1]

	if args.png:
		if args.verbose:
			print("creating PNG-images")

		for n in range(args.numFiles):
			fname = strformat.format(n)
			if args.verbose:
				print(fname)
			f = open(fname, 'wb')
			w = png.Writer(width, height)

			dat = []

			for i in range(height):
				dat.append(list(nprnd.randint(2, size = 3 * width) * 255))

			w.write(f, dat)
			f.close()

	if args.build:
		if args.verbose:
			print("creating video")
		import subprocess
		cmd = "ffmpeg -framerate 60 -i b%0" + str(numdigits) + "d.png " \
			+ "-s " + str(width) + "x" + str(height) + " " \
			+ "-pix_fmt yuv420p -preset slow -crf 20 " \
			+ "-profile:v high -level 4.0 " \
			+ "video.mp4"
		p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE)
		while True:
			out = p.stderr.read(1)
			if out == '' and p.poll() != None:
				break
			if out != '':
				sys.stdout.write(out)
				sys.stdout.flush()

	if args.verbose:
		print("done")

if __name__ == '__main__':
	args = arguments()
	main(args)

<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import os
import sys

import six.moves.cPickle as pickle
import xia2.Handlers.Streams
from dials.array_family import flex

def munch_rogues(rogues):
    rogue_reflections = []
    for record in open(rogues):
        if not record.strip():
            continue
        tokens = record.split()
        if not tokens[-1] == "*":
            continue
        x, y, z = list(map(float, tokens[15:18]))
        b = int(tokens[6])
        h, k, l = list(
            map(int, tokens[3:6])
        )  # don't forget these are probably reindexed
        rogue_reflections.append((b, x, y, z, h, k, l))

    return rogue_reflections

def reconstruct_rogues(params):
    assert os.path.exists("xia2.json")
    from xia2.Schema.XProject import XProject

    xinfo = XProject.from_json(filename="xia2.json")

    from dxtbx.model.experiment_list import ExperimentListFactory

    crystals = xinfo.get_crystals()
    assert len(crystals) == 1

    for xname in crystals:
        crystal = crystals[xname]

    scaler = crystal._get_scaler()

    epochs = scaler._sweep_handler.get_epochs()

    rogues = os.path.join(scaler.get_working_directory(), xname, "scale", "ROGUES")

    rogue_reflections = munch_rogues(rogues)

    batched_reflections = {}

    for epoch in epochs:
        si = scaler._sweep_handler.get_sweep_information(epoch)
        intgr = si.get_integrater()
        experiments = ExperimentListFactory.from_json_file(
            intgr.get_integrated_experiments()
        )
        with open(intgr.get_integrated_reflections(), "rb") as fh:
            reflections = pickle.load(fh)
        batched_reflections[si.get_batch_range()] = (
            experiments,
            reflections,
            si.get_sweep_name(),
        )

    # - look up reflection in reflection list, get bounding box
    # - pull pixels given from image set, flatten these, write out

    from annlib_ext import AnnAdaptor as ann_adaptor

    reflections_run = {}
    for run in batched_reflections:
        reflections_run[run] = []

    for rogue in rogue_reflections:
        b = rogue[0]
        for run in batched_reflections:
            if b >= run[0] and b <= run[1]:
                reflections_run[run].append(rogue)
                break

    for run_no, run in enumerate(reflections_run):
        experiment = batched_reflections[run][0]
        reflections = batched_reflections[run][1]
        name = batched_reflections[run][2]
        rogues = reflections_run[run]
        reference = flex.double()
        scan = experiment.scans()[0]
        images = experiment.imagesets()[0]
        for xyz in reflections["xyzcal.px"]:
            reference.append(xyz[0])
            reference.append(xyz[1])
            reference.append(xyz[2])

        search = flex.double()
        for rogue in rogues:
            search.append(rogue[1])
            search.append(rogue[2])
            search.append(scan.get_array_index_from_angle(rogue[3]))

        ann = ann_adaptor(data=reference, dim=3, k=1)
        ann.query(search)

        keep = flex.bool(len(reflections), False)

        for j, rogue in enumerate(rogues):
            keep[ann.nn[j]] = True

        reflections = reflections.select(keep)

        if params.extract:
            reflections["shoebox"] = flex.shoebox(
                reflections["panel"], reflections["bbox"], allocate=True
            )
            reflections.extract_shoeboxes(images, verbose=False)

        if len(reflections_run) > 1:
            output = params.output.reflections.replace(".refl", "-%s.refl" % name)
            print(
                "Extracted %d rogue reflections for %s to %s"
                % (len(reflections), name, output)
            )
            reflections.as_file(output)
        else:
            output = params.output.reflections
            print("Extracted %d rogue reflections to %s" % (len(reflections), output))
            reflections.as_file(output)

if __name__ == "__main__":
    xia2.Handlers.Streams.setup_logging(
        logfile="xia2.rogues_gallery.txt", debugfile="xia2.rogues_gallery-debug.txt"
    )
    from libtbx.phil import parse

    phil_scope = parse(
        """
rogues {
  extract = False
    .type = bool
    .help = "Extract shoebox pixels"

  output {
    reflections = 'xia2-rogues.refl'
      .type = str
      .help = "The integrated output filename"
  }
}
  """
    )
    interp = phil_scope.command_line_argument_interpreter(home_scope="rogues")
    for arg in sys.argv[1:]:
        cl_phil = interp.process(arg)
        phil_scope = phil_scope.fetch(cl_phil)
    params = phil_scope.extract()
    reconstruct_rogues(params.rogues)
<commit_msg>Retire xia2.rogues_gallery, #352<commit_after><|endoftext|>


<commit_before><commit_msg>add get_job in CytomineJob<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

from os import chdir
from os.path import isfile, isdir
import sys

from util import status

from sh import Command, make, git

import os

from multiprocessing import cpu_count
max_load = cpu_count() * 0.9

minimal = False
if '--minimal' in sys.argv:
    minimal = True

def check_llvm():
    try:
        llvm_path = './llvm/Release+Asserts'
        assert isfile(llvm_path+'/lib/ocaml/llvm.cma') # is llvm.cma there?
        llvm_config = Command(llvm_path+'/bin/llvm-config')
        ver = llvm_config('--version') # try calling llvm-config
        print ver
        assert '3.2' in ver
        return True
    except:
        return False

class OCamlVersionError(Exception):
    url = 'http://caml.inria.fr/'
    def __str__(self):
        return 'OCaml >= 3.12 is required'

class OCamlFindlibError(Exception):
    url = 'http://projects.camlcity.org/projects/findlib.html'
    def __str__(self):
        return 'OCaml findlib is required'

class OCamlLibMissingError(Exception):
    def __init__(self, name):
        self.name = name
    def __str__(self):
        return 'Missing required OCaml library: %s' % self.name

def check_ocaml():
    try:
        # Test for ocaml 3.12.*
        status('Testing for OCaml 3.12.* or greater')
        from sh import ocaml, ocamlbuild
        ver = ocaml('-version')
        print ver
        assert '3.12' in ver or '4.' in ver
    except:
        raise OCamlVersionError()

    try:
        status('Testing for ocamlfind')
        from sh import ocamlfind
        print '..OK!'
    except:
        raise OCamlFindlibError()

    def assert_lib_version(lib, ver_req):
        def lib_version(lib):
            status('Testing for %s' % lib)
            info = ocamlfind('query', '-l', lib)
            return [l.split()[-1] for l in info.splitlines() if l.startswith('version')][0]
        try:
            ver = lib_version(lib)
            ver = float( '.'.join( ver.split('.')[0:2] ) )
            assert ver >= ver_req
        except:
            raise OCamlLibMissingError('%s >= %.1f' % (lib, ver_req))

    assert_lib_version('sexplib', 7.0)
    assert_lib_version('batteries', 1.4)
    return ocamlbuild

ocamlbuild = None
platform = sys.platform.lower()
if 'linux' in platform:
    try:
        status('Testing for g++')
        from sh import which
        assert which('g++')
        print '...OK!'

        status('Testing for package libc6-dev-i386')
        assert isfile('/usr/include/x86_64-linux-gnu/gnu/stubs-32.h')
        print '...OK!'

        try:
            ocamlbuild = check_ocaml()
            print '...OK!'
        except OCamlVersionError as e:
            print 'e'

        status('Testing for package libsexplib-camlp4-dev')
        assert isdir('/usr/lib/ocaml/sexplib')
        print '...OK!'
    except:
        print 'You appear to be missing some required packages. Try:'
        print 'sudo apt-get install g++ libc6-i386-dev ocaml libsexplib-camlp4-dev'
        sys.exit(1)

if 'darwin' in platform:
    status('Testing for a sufficiently new g++')
    gxx = Command('g++')
    ver = gxx('--version')
    try:
        assert 'g++-4.' in ver
        print '...OK!'
    except:
        print 'Your g++ compiler is missing or too old.'
        print 'Trying installing the command line tools from xcode.'
        print 'They can be found in preferences -> downloads -> command line tools.'
        print 'If that doesn\'t work, update xcode and try again.'
        sys.exit(1)
    try:
        ocamlbuild = check_ocaml()
        print '...OK!'
    except OCamlVersionError as e:
        print e
        sys.exit(1)
    except OCamlFindlibError as e:
        print e
        sys.exit(1)
    except OCamlLibMissingError as e:
        print e
        sys.exit(1)

# Submodule update/init
# TODO: make --recursive optional
status('Checking out submodules')
git('submodule', 'sync')
git('submodule', 'update', '--init', '--recursive')

# TODO: always run make -C llvm, just to make sure it's up to date. Does configure cache its settings when a reconfigure is forced?
# TODO: make install in subdir, with docs
#        requires graphviz, doxygen; target ocamlbuild to alt dir?; make clean?
# Build llvm
if check_llvm():
    status('llvm appears to be present -- skipping')
else:
    chdir('llvm')
    configure = Command('./configure')
    llvm_cfg = ['--enable-assertions', '--enable-optimized']
    if minimal:
        llvm_cfg = llvm_cfg + ['--enable-targets=host,ptx,x86_64,arm']
    else:
        llvm_cfg = llvm_cfg + ['--enable-targets=all', '--enable-docs', '--enable-doxygen']
    status('''Configuring llvm:
    %s''' % llvm_cfg)
    print configure(llvm_cfg)

    status('Building llvm')
    # print make('-j', '--load-average=%f' % max_load)
    print make('-j12')

    chdir('docs')
    if not minimal:
        make('ocamldoc', '-j', '--load-average=%f' % max_load)
    # optional: check_call('make doxygen'.split(' '))
    chdir('../..')
    assert check_llvm()

# Test building
chdir('src')
status('Test: building halide.cmxa')
print ocamlbuild('-use-ocamlfind', 'halide.cmxa')
chdir('..')

status('Building C++ bindings')
print make('-C', 'cpp_bindings', '-j1') # can be flakey with first parallel build on SSD
<commit_msg>Fix Ubuntu package recommendations in bootstrap<commit_after>#!/usr/bin/env python

from os import chdir
from os.path import isfile, isdir
import sys

from util import status

from sh import Command, make, git

import os

from multiprocessing import cpu_count
max_load = cpu_count() * 0.9

minimal = False
if '--minimal' in sys.argv:
    minimal = True

def check_llvm():
    try:
        llvm_path = './llvm/Release+Asserts'
        assert isfile(llvm_path+'/lib/ocaml/llvm.cma') # is llvm.cma there?
        llvm_config = Command(llvm_path+'/bin/llvm-config')
        ver = llvm_config('--version') # try calling llvm-config
        print ver
        assert '3.2' in ver
        return True
    except:
        return False

class OCamlVersionError(Exception):
    url = 'http://caml.inria.fr/'
    def __str__(self):
        return 'OCaml >= 3.12 is required'

class OCamlFindlibError(Exception):
    url = 'http://projects.camlcity.org/projects/findlib.html'
    def __str__(self):
        return 'OCaml findlib is required'

class OCamlLibMissingError(Exception):
    def __init__(self, name):
        self.name = name
    def __str__(self):
        return 'Missing required OCaml library: %s' % self.name

def check_ocaml():
    try:
        # Test for ocaml 3.12.*
        status('Testing for OCaml 3.12.* or greater')
        from sh import ocaml, ocamlbuild
        ver = ocaml('-version')
        print ver
        assert '3.12' in ver or '4.' in ver
    except:
        raise OCamlVersionError()

    try:
        status('Testing for ocamlfind')
        from sh import ocamlfind
        print '..OK!'
    except:
        raise OCamlFindlibError()

    def assert_lib_version(lib, ver_req):
        def lib_version(lib):
            status('Testing for %s' % lib)
            info = ocamlfind('query', '-l', lib)
            return [l.split()[-1] for l in info.splitlines() if l.startswith('version')][0]
        try:
            ver = lib_version(lib)
            ver = float( '.'.join( ver.split('.')[0:2] ) )
            assert ver >= ver_req
        except:
            raise OCamlLibMissingError('%s >= %.1f' % (lib, ver_req))

    assert_lib_version('sexplib', 7.0)
    assert_lib_version('batteries', 1.4)
    return ocamlbuild

ocamlbuild = None
platform = sys.platform.lower()
if 'linux' in platform:
    try:
        status('Testing for g++')
        from sh import which
        assert which('g++')
        print '...OK!'

        status('Testing for package libc6-dev-i386')
        assert isfile('/usr/include/x86_64-linux-gnu/gnu/stubs-32.h')
        print '...OK!'

        try:
            ocamlbuild = check_ocaml()
            print '...OK!'
        except OCamlVersionError as e:
            print 'e'

        status('Testing for package libsexplib-camlp4-dev')
        assert isdir('/usr/lib/ocaml/sexplib')
        print '...OK!'
    except:
        print 'You appear to be missing some required packages. Try:'
        print 'sudo apt-get install g++ libc6-dev-i386 ocaml libsexplib-camlp4-dev'
        sys.exit(1)

if 'darwin' in platform:
    status('Testing for a sufficiently new g++')
    gxx = Command('g++')
    ver = gxx('--version')
    try:
        assert 'g++-4.' in ver
        print '...OK!'
    except:
        print 'Your g++ compiler is missing or too old.'
        print 'Trying installing the command line tools from xcode.'
        print 'They can be found in preferences -> downloads -> command line tools.'
        print 'If that doesn\'t work, update xcode and try again.'
        sys.exit(1)
    try:
        ocamlbuild = check_ocaml()
        print '...OK!'
    except OCamlVersionError as e:
        print e
        sys.exit(1)
    except OCamlFindlibError as e:
        print e
        sys.exit(1)
    except OCamlLibMissingError as e:
        print e
        sys.exit(1)

# Submodule update/init
# TODO: make --recursive optional
status('Checking out submodules')
git('submodule', 'sync')
git('submodule', 'update', '--init', '--recursive')

# TODO: always run make -C llvm, just to make sure it's up to date. Does configure cache its settings when a reconfigure is forced?
# TODO: make install in subdir, with docs
#        requires graphviz, doxygen; target ocamlbuild to alt dir?; make clean?
# Build llvm
if check_llvm():
    status('llvm appears to be present -- skipping')
else:
    chdir('llvm')
    configure = Command('./configure')
    llvm_cfg = ['--enable-assertions', '--enable-optimized']
    if minimal:
        llvm_cfg = llvm_cfg + ['--enable-targets=host,ptx,x86_64,arm']
    else:
        llvm_cfg = llvm_cfg + ['--enable-targets=all', '--enable-docs', '--enable-doxygen']
    status('''Configuring llvm:
    %s''' % llvm_cfg)
    print configure(llvm_cfg)

    status('Building llvm')
    # print make('-j', '--load-average=%f' % max_load)
    print make('-j12')

    chdir('docs')
    if not minimal:
        make('ocamldoc', '-j', '--load-average=%f' % max_load)
    # optional: check_call('make doxygen'.split(' '))
    chdir('../..')
    assert check_llvm()

# Test building
chdir('src')
status('Test: building halide.cmxa')
print ocamlbuild('-use-ocamlfind', 'halide.cmxa')
chdir('..')

status('Building C++ bindings')
print make('-C', 'cpp_bindings', '-j1') # can be flakey with first parallel build on SSD
<|endoftext|>