<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

# implements a db-based repository

import changeset
import copy
import filecontents
import files
import log
import os
import package
import patch
import repository
import util
import versions

from repository import DataStoreRepository
from repository import AbstractRepository
from localrep import trovestore

class FilesystemRepository(DataStoreRepository, AbstractRepository):

    createBranches = 1

    ### Package access functions

    def iterAllTroveNames(self):
	return self.troveStore.iterTroveNames()

    def getAllTroveLeafs(self, troveNameList):
	d = {}
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
				self.troveStore.iterAllTroveLeafs(troveName) ]
	return d

    def getTroveVersionList(self, troveNameList):
	d = {}
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
				self.troveStore.iterTroveVersions(troveName) ]

	return d

    def getTroveLeavesByLabel(self, troveNameList, label):
	d = {}
	labelStr = label.asString()
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
			     self.troveStore.iterTroveLeafsByLabel(troveName,
								   labelStr) ]

	return d

    def getTroveFlavorsLatestVersion(self, troveName, branch):
	return [ (versions.VersionFromString(x[0]),
		  deps.ThawDependencySet([1])) for x in
		    self.troveStore.iterTrovePerFlavorLeafs(troveName,
							    branch.asString()) ]

    def getTroveVersionFlavors(self, troveDict):
	newD = {}
	for (troveName, versionList) in troveDict.iteritems():
	    innerD = {}
	    for version in versionList:
		innerD[version] = [ x for x in
		    self.troveStore.iterTroveFlavors(troveName, version) ]
	    newD[troveName] = innerD

	return newD

    def hasPackage(self, pkgName):
	return self.troveStore.hasTrove(pkgName)

    def hasTrove(self, pkgName, version, flavor):
	return self.troveStore.hasTrove(pkgName, troveVersion = version,
					troveFlavor = flavor)

    def getTroveLatestVersion(self, pkgName, branch):
        try:
            version = self.troveStore.troveLatestVersion(pkgName, branch)
        except KeyError:
            raise repository.PackageMissing(pkgName, branch)

    def getTrove(self, pkgName, version, flavor, pristine = True):
	try:
	    return self.troveStore.getTrove(pkgName, version, flavor)
	except KeyError:
	    raise repository.PackageMissing(pkgName, version)

    def eraseTrove(self, pkgName, version, flavor):
	self.troveStore.eraseTrove(pkgName, version, flavor)

    def addPackage(self, pkg):
	self.troveStore.addTrove(pkg)

    def commit(self):
	self.troveStore.commit()

    def branchesOfTroveLabel(self, troveName, label):
	return self.troveStore.branchesOfTroveLabel(troveName, label)

    def createTroveBranch(self, pkgName, branch):
	log.debug("creating branch %s for %s", branch.asString(), pkgName)
	if not self.hasPackage(pkgName):
	    raise repository.PackageMissing, pkgName
        return self.troveStore.createTroveBranch(pkgName, branch)

    def iterFilesInTrove(self, troveName, version, flavor,
                         sortByPath = False, withFiles = False):
	return self.troveStore.iterFilesInTrove(troveName, version, flavor,
                                                sortByPath, withFiles)

    ### File functions

    def getFileVersion(self, fileId, version, withContents = 0):
	file = self.troveStore.getFile(fileId, version)
	if withContents:
	    if file.hasContents:
		cont = filecontents.FromRepository(self, file.contents.sha1(),
						   file.contents.size())
	    else:
		cont = None

	    return (file, cont)

	return file

    def addFileVersion(self, fileId, version, file):
	# don't add duplicated to this repository
	if not self.troveStore.hasFile(fileId, version):
	    self.troveStore.addFile(file, version)

    def eraseFileVersion(self, fileId, version):
	self.troveStore.eraseFile(fileId, version)

    ###

    def __del__(self):
	self.close()

    def createBranch(self, newBranch, where, troveList = []):
	if not troveList:
	    troveList = self.iterAllTroveNames()

	troveList = [ (x, where) for x in troveList ]

	branchedTroves = {}
	branchedFiles = {}

	while troveList:
	    troveName = troveList[0][0]
	    location = troveList[0][1]
	    del troveList[0]

	    if branchedTroves.has_key(troveName): continue
	    branchedTroves[troveName] = 1
	    if not self.hasPackage(troveName):
		log.warning("package %s does not exist" % troveName)
		continue

	    list = []
	    if isinstance(location, versions.Version):
		if min(location.timeStamps()) == 0:
		    self.troveStore.getFullVersion(troveName, location)
		list.append(location)
	    else:
		branchList = self.branchesOfTroveLabel(troveName, location)
		if not branchList:
		    log.error("%s does not have branch %s", troveName, location)
		for branch in branchList:
		    v = self.getTroveLatestVersion(troveName, branch)
		    list.append(v)

	    # XXX this probably doesn't get flavors right

	    d = self.getTroveVersionFlavors({troveName: list})
	    for (version, flavors) in d[troveName].iteritems():
		for flavor in flavors:
		    pkg = self.getTrove(troveName, version, flavor)
		    branchedVersion = version.fork(newBranch, sameVerRel = 0)
		    self.createTroveBranch(troveName, branchedVersion)

		    for (name, version, flavor) in pkg.iterTroveList():
			troveList.append((name, version))

        # commit branch to the repository
        self.commit()

    def open(self):
	if self.troveStore is not None:
	    self.close()

	self.troveStore = trovestore.TroveStore(self.sqlDB)

    def commitChangeSet(self, cs):
        job = ChangeSetJob(self, cs)
        self.commit()

    def close(self):
	if self.troveStore is not None:
	    self.troveStore.db.close()
	    self.troveStore = None

    def __init__(self, path):
	self.top = path
	self.troveStore = None

	self.sqlDB = self.top + "/sqldb"

	try:
	    util.mkdirChain(self.top)
	except OSError, e:
	    raise repository.OpenError(str(e))

        self.open()

	DataStoreRepository.__init__(self, path)
	AbstractRepository.__init__(self)

class ChangeSetJobFile(object):

    __slots__ = [ "theVersion" , "theFile", "theRestoreContents",
		  "fileContents", "changeSet", "thePath", "theFileId" ]

    def version(self):
	return self.theVersion

    def changeVersion(self, ver):
	self.theVersion = ver

    def restoreContents(self):
	return self.theRestoreContents

    def file(self):
	return self.theFile

    def changeFile(self, fileObj):
	self.theFile = fileObj

    def path(self):
	return self.thePath

    def fileId(self):
	return self.theFileId

    def copy(self):
	return copy.deepcopy(self)

    def getContents(self):
	if self.fileContents == "":
	    return None
	elif self.fileContents:
	    return self.fileContents

	return self.changeSet.getFileContents(self.theFileId)[1]

    # overrideContents = None means use contents from changeset
    # overrideContents = "" means there are no contents
    def __init__(self, changeSet, fileId, file, version, path,
		 overrideContents, restoreContents):
	self.theVersion = version
	self.theFile = file
	self.theRestoreContents = restoreContents
	self.fileContents = overrideContents
	self.changeSet = changeSet
	self.thePath = path
	self.theFileId = fileId

class ChangeSetJob:
    """
    ChangeSetJob provides a to-do list for applying a change set; file
    remappings should have been applied to the change set before it gets
    this far. Derivative classes can override these methods to change the
    behavior; for example, if addPackage is overridden no pacakges will
    make it to the database. The same holds for oldFile.
    """

    def addPackage(self, pkg):
	self.repos.addPackage(pkg)

    def oldPackage(self, pkg):
	pass

    def oldFile(self, fileId, fileVersion, fileObj):
	pass

    def addFile(self, newFile, storeContents = True):
	file = newFile.file()
	fileId = newFile.fileId()

	# duplicates are filtered out (as necessary) by addFileVersion
	self.repos.addFileVersion(fileId, newFile.version(), file)

	# Note that the order doesn't matter, we're just copying
	# files into the repository. Restore the file pointer to
	# the beginning of the file as we may want to commit this
	# file to multiple locations.
	if storeContents:
	    self.repos.storeFileFromContents(newFile.getContents(), file,
					     newFile.restoreContents())

    def __init__(self, repos, cs):
	self.repos = repos
	self.cs = cs

	self.packagesToCommit = []

	fileMap = {}

	# create the package objects which need to be installed; the
	# file objects which map up with them are created later, but
	# we do need a map from fileId to the path and version of the
	# file we need, so build up a dictionary with that information
	for csPkg in cs.iterNewPackageList():
	    newVersion = csPkg.getNewVersion()
	    old = csPkg.getOldVersion()
	    pkgName = csPkg.getName()

	    if repos.hasTrove(pkgName, newVersion, csPkg.getFlavor()):
		raise repository.CommitError, \
		       "version %s for %s is already installed" % \
			(newVersion.asString(), csPkg.getName())

	    if old:
		newPkg = repos.getTrove(pkgName, old, csPkg.getFlavor(),
					pristine = True)
		newPkg.changeVersion(newVersion)
	    else:
		newPkg = package.Trove(csPkg.getName(), newVersion,
					 csPkg.getFlavor())

	    newFileMap = newPkg.applyChangeSet(csPkg)

	    self.packagesToCommit.append(newPkg)
	    fileMap.update(newFileMap)

	# Create the file objects we'll need for the commit. This handles
	# files which were added and files which have changed
	list = cs.getFileList()
	# sort this by fileid to ensure we pull files from the change
	# set in the right order
	list.sort()
	for (fileId, (oldVer, newVer, diff)) in list:
	    restoreContents = 1
	    if oldVer:
		oldfile = repos.getFileVersion(fileId, oldVer)
		file = oldfile.copy()
		file.twm(diff, oldfile)

		if file.hasContents and oldfile.hasContents and	    \
		   file.contents.sha1() == oldfile.contents.sha1():
		    restoreContents = 0
	    else:
		# this is for new files
		file = files.ThawFile(diff, fileId)

	    # we should have had a package which requires this (new) version
	    # of the file
	    assert(newVer == fileMap[fileId][1])

	    if file.hasContents and restoreContents:
		fileContents = None

		if repos.hasFileContents(file.contents.sha1()):
		    # if we already have the file in the data store we can
		    # get the contents from there
		    fileContents = filecontents.FromRepository(repos,
				    file.contents.sha1(), file.contents.size())
		    contType = changeset.ChangedFileTypes.file
		else:
		    contType = cs.getFileContentsType(fileId)
		    if contType == changeset.ChangedFileTypes.diff:
			# the content for this file is in the form of a diff,
			# which we need to apply against the file in the
			# repository
			assert(oldVer)
			(contType, fileContents) = cs.getFileContents(fileId)
			sha1 = oldfile.contents.sha1()
			f = repos.getFileContents((sha1,))[sha1]
			oldLines = f.readlines()
			del f
			diff = fileContents.get().readlines()
			(newLines, failedHunks) = patch.patch(oldLines, diff)
			fileContents = filecontents.FromString("".join(newLines))

			if failedHunks:
			    fileContents = filecontents.WithFailedHunks(
						fileContents, failedHunks)
	    else:
		# this means there are no contents to restore (None
		# means get the contents from the change set)
		fileContents = ""

	    path = fileMap[fileId][0]
	    self.addFile(ChangeSetJobFile(cs, fileId, file, newVer, path,
					  fileContents, restoreContents))

	for (pkgName, version, flavor) in cs.getOldPackageList():
	    pkg = self.repos.getTrove(pkgName, version, flavor)
	    self.oldPackage(pkg)

	    for (fileId, path, version) in pkg.iterFileList():
		file = self.repos.getFileVersion(fileId, version)
		self.oldFile(fileId, version, file)

	for newPkg in self.packagesToCommit:
	    self.addPackage(newPkg)

<commit_msg>return the version<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

# implements a db-based repository

import changeset
import copy
import filecontents
import files
import log
import os
import package
import patch
import repository
import util
import versions

from repository import DataStoreRepository
from repository import AbstractRepository
from localrep import trovestore

class FilesystemRepository(DataStoreRepository, AbstractRepository):

    createBranches = 1

    ### Package access functions

    def iterAllTroveNames(self):
	return self.troveStore.iterTroveNames()

    def getAllTroveLeafs(self, troveNameList):
	d = {}
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
				self.troveStore.iterAllTroveLeafs(troveName) ]
	return d

    def getTroveVersionList(self, troveNameList):
	d = {}
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
				self.troveStore.iterTroveVersions(troveName) ]

	return d

    def getTroveLeavesByLabel(self, troveNameList, label):
	d = {}
	labelStr = label.asString()
	for troveName in troveNameList:
	    d[troveName] = [ versions.VersionFromString(x) for x in
			     self.troveStore.iterTroveLeafsByLabel(troveName,
								   labelStr) ]

	return d

    def getTroveFlavorsLatestVersion(self, troveName, branch):
	return [ (versions.VersionFromString(x[0]),
		  deps.ThawDependencySet([1])) for x in
		    self.troveStore.iterTrovePerFlavorLeafs(troveName,
							    branch.asString()) ]

    def getTroveVersionFlavors(self, troveDict):
	newD = {}
	for (troveName, versionList) in troveDict.iteritems():
	    innerD = {}
	    for version in versionList:
		innerD[version] = [ x for x in
		    self.troveStore.iterTroveFlavors(troveName, version) ]
	    newD[troveName] = innerD

	return newD

    def hasPackage(self, pkgName):
	return self.troveStore.hasTrove(pkgName)

    def hasTrove(self, pkgName, version, flavor):
	return self.troveStore.hasTrove(pkgName, troveVersion = version,
					troveFlavor = flavor)

    def getTroveLatestVersion(self, pkgName, branch):
        try:
            return self.troveStore.troveLatestVersion(pkgName, branch)
        except KeyError:
            raise repository.PackageMissing(pkgName, branch)

    def getTrove(self, pkgName, version, flavor, pristine = True):
	try:
	    return self.troveStore.getTrove(pkgName, version, flavor)
	except KeyError:
	    raise repository.PackageMissing(pkgName, version)

    def eraseTrove(self, pkgName, version, flavor):
	self.troveStore.eraseTrove(pkgName, version, flavor)

    def addPackage(self, pkg):
	self.troveStore.addTrove(pkg)

    def commit(self):
	self.troveStore.commit()

    def branchesOfTroveLabel(self, troveName, label):
	return self.troveStore.branchesOfTroveLabel(troveName, label)

    def createTroveBranch(self, pkgName, branch):
	log.debug("creating branch %s for %s", branch.asString(), pkgName)
	if not self.hasPackage(pkgName):
	    raise repository.PackageMissing, pkgName
        return self.troveStore.createTroveBranch(pkgName, branch)

    def iterFilesInTrove(self, troveName, version, flavor,
                         sortByPath = False, withFiles = False):
	return self.troveStore.iterFilesInTrove(troveName, version, flavor,
                                                sortByPath, withFiles)

    ### File functions

    def getFileVersion(self, fileId, version, withContents = 0):
	file = self.troveStore.getFile(fileId, version)
	if withContents:
	    if file.hasContents:
		cont = filecontents.FromRepository(self, file.contents.sha1(),
						   file.contents.size())
	    else:
		cont = None

	    return (file, cont)

	return file

    def addFileVersion(self, fileId, version, file):
	# don't add duplicated to this repository
	if not self.troveStore.hasFile(fileId, version):
	    self.troveStore.addFile(file, version)

    def eraseFileVersion(self, fileId, version):
	self.troveStore.eraseFile(fileId, version)

    ###

    def __del__(self):
	self.close()

    def createBranch(self, newBranch, where, troveList = []):
	if not troveList:
	    troveList = self.iterAllTroveNames()

	troveList = [ (x, where) for x in troveList ]

	branchedTroves = {}
	branchedFiles = {}

	while troveList:
	    troveName = troveList[0][0]
	    location = troveList[0][1]
	    del troveList[0]

	    if branchedTroves.has_key(troveName): continue
	    branchedTroves[troveName] = 1
	    if not self.hasPackage(troveName):
		log.warning("package %s does not exist" % troveName)
		continue

	    list = []
	    if isinstance(location, versions.Version):
		if min(location.timeStamps()) == 0:
		    self.troveStore.getFullVersion(troveName, location)
		list.append(location)
	    else:
		branchList = self.branchesOfTroveLabel(troveName, location)
		if not branchList:
		    log.error("%s does not have branch %s", troveName, location)
		for branch in branchList:
		    v = self.getTroveLatestVersion(troveName, branch)
		    list.append(v)

	    # XXX this probably doesn't get flavors right

	    d = self.getTroveVersionFlavors({troveName: list})
	    for (version, flavors) in d[troveName].iteritems():
		for flavor in flavors:
		    pkg = self.getTrove(troveName, version, flavor)
		    branchedVersion = version.fork(newBranch, sameVerRel = 0)
		    self.createTroveBranch(troveName, branchedVersion)

		    for (name, version, flavor) in pkg.iterTroveList():
			troveList.append((name, version))

        # commit branch to the repository
        self.commit()

    def open(self):
	if self.troveStore is not None:
	    self.close()

	self.troveStore = trovestore.TroveStore(self.sqlDB)

    def commitChangeSet(self, cs):
        job = ChangeSetJob(self, cs)
        self.commit()

    def close(self):
	if self.troveStore is not None:
	    self.troveStore.db.close()
	    self.troveStore = None

    def __init__(self, path):
	self.top = path
	self.troveStore = None

	self.sqlDB = self.top + "/sqldb"

	try:
	    util.mkdirChain(self.top)
	except OSError, e:
	    raise repository.OpenError(str(e))

        self.open()

	DataStoreRepository.__init__(self, path)
	AbstractRepository.__init__(self)

class ChangeSetJobFile(object):

    __slots__ = [ "theVersion" , "theFile", "theRestoreContents",
		  "fileContents", "changeSet", "thePath", "theFileId" ]

    def version(self):
	return self.theVersion

    def changeVersion(self, ver):
	self.theVersion = ver

    def restoreContents(self):
	return self.theRestoreContents

    def file(self):
	return self.theFile

    def changeFile(self, fileObj):
	self.theFile = fileObj

    def path(self):
	return self.thePath

    def fileId(self):
	return self.theFileId

    def copy(self):
	return copy.deepcopy(self)

    def getContents(self):
	if self.fileContents == "":
	    return None
	elif self.fileContents:
	    return self.fileContents

	return self.changeSet.getFileContents(self.theFileId)[1]

    # overrideContents = None means use contents from changeset
    # overrideContents = "" means there are no contents
    def __init__(self, changeSet, fileId, file, version, path,
		 overrideContents, restoreContents):
	self.theVersion = version
	self.theFile = file
	self.theRestoreContents = restoreContents
	self.fileContents = overrideContents
	self.changeSet = changeSet
	self.thePath = path
	self.theFileId = fileId

class ChangeSetJob:
    """
    ChangeSetJob provides a to-do list for applying a change set; file
    remappings should have been applied to the change set before it gets
    this far. Derivative classes can override these methods to change the
    behavior; for example, if addPackage is overridden no pacakges will
    make it to the database. The same holds for oldFile.
    """

    def addPackage(self, pkg):
	self.repos.addPackage(pkg)

    def oldPackage(self, pkg):
	pass

    def oldFile(self, fileId, fileVersion, fileObj):
	pass

    def addFile(self, newFile, storeContents = True):
	file = newFile.file()
	fileId = newFile.fileId()

	# duplicates are filtered out (as necessary) by addFileVersion
	self.repos.addFileVersion(fileId, newFile.version(), file)

	# Note that the order doesn't matter, we're just copying
	# files into the repository. Restore the file pointer to
	# the beginning of the file as we may want to commit this
	# file to multiple locations.
	if storeContents:
	    self.repos.storeFileFromContents(newFile.getContents(), file,
					     newFile.restoreContents())

    def __init__(self, repos, cs):
	self.repos = repos
	self.cs = cs

	self.packagesToCommit = []

	fileMap = {}

	# create the package objects which need to be installed; the
	# file objects which map up with them are created later, but
	# we do need a map from fileId to the path and version of the
	# file we need, so build up a dictionary with that information
	for csPkg in cs.iterNewPackageList():
	    newVersion = csPkg.getNewVersion()
	    old = csPkg.getOldVersion()
	    pkgName = csPkg.getName()

	    if repos.hasTrove(pkgName, newVersion, csPkg.getFlavor()):
		raise repository.CommitError, \
		       "version %s for %s is already installed" % \
			(newVersion.asString(), csPkg.getName())

	    if old:
		newPkg = repos.getTrove(pkgName, old, csPkg.getFlavor(),
					pristine = True)
		newPkg.changeVersion(newVersion)
	    else:
		newPkg = package.Trove(csPkg.getName(), newVersion,
					 csPkg.getFlavor())

	    newFileMap = newPkg.applyChangeSet(csPkg)

	    self.packagesToCommit.append(newPkg)
	    fileMap.update(newFileMap)

	# Create the file objects we'll need for the commit. This handles
	# files which were added and files which have changed
	list = cs.getFileList()
	# sort this by fileid to ensure we pull files from the change
	# set in the right order
	list.sort()
	for (fileId, (oldVer, newVer, diff)) in list:
	    restoreContents = 1
	    if oldVer:
		oldfile = repos.getFileVersion(fileId, oldVer)
		file = oldfile.copy()
		file.twm(diff, oldfile)

		if file.hasContents and oldfile.hasContents and	    \
		   file.contents.sha1() == oldfile.contents.sha1():
		    restoreContents = 0
	    else:
		# this is for new files
		file = files.ThawFile(diff, fileId)

	    # we should have had a package which requires this (new) version
	    # of the file
	    assert(newVer == fileMap[fileId][1])

	    if file.hasContents and restoreContents:
		fileContents = None

		if repos.hasFileContents(file.contents.sha1()):
		    # if we already have the file in the data store we can
		    # get the contents from there
		    fileContents = filecontents.FromRepository(repos,
				    file.contents.sha1(), file.contents.size())
		    contType = changeset.ChangedFileTypes.file
		else:
		    contType = cs.getFileContentsType(fileId)
		    if contType == changeset.ChangedFileTypes.diff:
			# the content for this file is in the form of a diff,
			# which we need to apply against the file in the
			# repository
			assert(oldVer)
			(contType, fileContents) = cs.getFileContents(fileId)
			sha1 = oldfile.contents.sha1()
			f = repos.getFileContents((sha1,))[sha1]
			oldLines = f.readlines()
			del f
			diff = fileContents.get().readlines()
			(newLines, failedHunks) = patch.patch(oldLines, diff)
			fileContents = filecontents.FromString("".join(newLines))

			if failedHunks:
			    fileContents = filecontents.WithFailedHunks(
						fileContents, failedHunks)
	    else:
		# this means there are no contents to restore (None
		# means get the contents from the change set)
		fileContents = ""

	    path = fileMap[fileId][0]
	    self.addFile(ChangeSetJobFile(cs, fileId, file, newVer, path,
					  fileContents, restoreContents))

	for (pkgName, version, flavor) in cs.getOldPackageList():
	    pkg = self.repos.getTrove(pkgName, version, flavor)
	    self.oldPackage(pkg)

	    for (fileId, path, version) in pkg.iterFileList():
		file = self.repos.getFileVersion(fileId, version)
		self.oldFile(fileId, version, file)

	for newPkg in self.packagesToCommit:
	    self.addPackage(newPkg)

<|endoftext|>


<commit_before><commit_msg>FIX: wait half a second for the state file to be updated<commit_after><|endoftext|>


<commit_before>
"""Change conclusion dropdowns options in control assessment

Revision ID: 29dca3ce0556
Revises: 2d8a46b1e4a4
Create Date: 2015-09-11 13:18:18.269109

"""

# revision identifiers, used by Alembic.
revision = '29dca3ce0556'
down_revision = '2d8a46b1e4a4'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    sql = """
    UPDATE control_assessments
    SET design = 'Ineffective'
    WHERE
        design = 'Material weakness' OR
        design = 'Significant deficiency'
    """
    op.execute(sql)

    sql = """
    UPDATE control_assessments
    SET operationally = 'Ineffective'
    WHERE
        operationally = 'Material weakness' OR
        operationally = 'Significant deficiency'
    """
    op.execute(sql)

def downgrade():
    sql = """
    UPDATE control_assessments
    SET design = 'Significant deficiency'
    WHERE design = 'Ineffective'
    """
    op.execute(sql)

    sql = """
    UPDATE control_assessments
    SET operationally = 'Significant deficiency'
    WHERE operationally = 'Ineffective'
    """
    op.execute(sql)
<commit_msg>Add copyright header to migration<commit_after># Copyright (C) 2015 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: jost@reciprocitylabs.com
# Maintained By: jost@reciprocitylabs.com

"""Change conclusion dropdowns options in control assessment

Revision ID: 29dca3ce0556
Revises: 2d8a46b1e4a4
Create Date: 2015-09-11 13:18:18.269109

"""

# revision identifiers, used by Alembic.
revision = '29dca3ce0556'
down_revision = '2d8a46b1e4a4'

from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

def upgrade():
    sql = """
    UPDATE control_assessments
    SET design = 'Ineffective'
    WHERE
        design = 'Material weakness' OR
        design = 'Significant deficiency'
    """
    op.execute(sql)

    sql = """
    UPDATE control_assessments
    SET operationally = 'Ineffective'
    WHERE
        operationally = 'Material weakness' OR
        operationally = 'Significant deficiency'
    """
    op.execute(sql)

def downgrade():
    sql = """
    UPDATE control_assessments
    SET design = 'Significant deficiency'
    WHERE design = 'Ineffective'
    """
    op.execute(sql)

    sql = """
    UPDATE control_assessments
    SET operationally = 'Significant deficiency'
    WHERE operationally = 'Ineffective'
    """
    op.execute(sql)
<|endoftext|>


<commit_before><commit_msg>updated the version to 2.3.0.<commit_after><|endoftext|>


<commit_before>from django.contrib import admin
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from symposion.sponsorship.models import SponsorLevel, Sponsor, Benefit, \
    BenefitLevel, SponsorBenefit, BENEFITS

class BenefitLevelInline(admin.TabularInline):
    model = BenefitLevel
    extra = 0

class SponsorBenefitInline(admin.StackedInline):
    model = SponsorBenefit
    extra = 0
    fieldsets = [
        (None, {
            "fields": [
                ("benefit", "active"),
                ("max_words", "other_limits"),
                "text",
                "upload",
            ]
        })
    ]

class SponsorAdmin(admin.ModelAdmin):

    save_on_top = True
    fieldsets = [
        (None, {
            "fields": [
                ("name", "applicant"),
                ("level", "active"),
                "external_url",
                "annotation",
                ("contact_name", "contact_email")
            ]
        }),
        ("Metadata", {
            "fields": ["added"],
            "classes": ["collapse"]
        })
    ]
    inlines = [SponsorBenefitInline]
<<<<<<< HEAD
    list_display = ["name", "external_url", "level", "active", "contact", "applicant_field"]

    def contact(self, sponsor):
        return mark_safe('%s' % (escape(sponsor.contact_email), escape(sponsor.contact_name)))

    def applicant_field(self, sponsor):
        name = sponsor.applicant.get_full_name()
        email = sponsor.applicant.email
        return mark_safe('%s' % (escape(email), escape(name)))
    applicant_field.short_description = _(u"Applicant")
=======
    list_display = ["name", "external_url", "level", "active"]
    list_filter = ["level", "active"]
>>>>>>> admin_sponsor_benefit

    def get_form(self, *args, **kwargs):
        # @@@ kinda ugly but using choices= on NullBooleanField is broken
        form = super(SponsorAdmin, self).get_form(*args, **kwargs)
        form.base_fields["active"].widget.choices = [
            (u"1", _("unreviewed")),
            (u"2", _("approved")),
            (u"3", _("rejected"))
        ]
        return form

    # Define accessor functions for our benefit fields and add them to
    # list_display, so we can sort on them and give them sensible names.
    # Add the fields to list_filters while we're at it.
    for benefit in BENEFITS:
        benefit_name = benefit['name']
        field_name = benefit['field_name']

        def func_generator(ben):
            def column_func(obj):
                return getattr(obj, ben['field_name'])
            column_func.short_description = ben['column_title']
            column_func.boolean = True
            column_func.admin_order_field = ben['field_name']
            return column_func
        list_display.append(func_generator(benefit))
        list_filter.append(field_name)

    def save_related(self, request, form, formsets, change):
        super(SponsorAdmin, self).save_related(request, form, formsets, change)
        obj = form.instance
        obj.save()

class BenefitAdmin(admin.ModelAdmin):

    list_display = ["name", "type", "description", "levels"]
    inlines = [BenefitLevelInline]

    def levels(self, benefit):
        return u", ".join(l.level.name for l in benefit.benefit_levels.all())

class SponsorLevelAdmin(admin.ModelAdmin):

    inlines = [BenefitLevelInline]

class SponsorBenefitAdmin(admin.ModelAdmin):
    list_display = ('benefit', 'sponsor', 'active', '_is_complete', 'show_text')

    def show_text(self, sponsor_benefit):
        if sponsor_benefit.text:
            return sponsor_benefit.text[:100]
        else:
            return _("None")

admin.site.register(SponsorLevel, SponsorLevelAdmin)
admin.site.register(Sponsor, SponsorAdmin)
admin.site.register(Benefit, BenefitAdmin)
admin.site.register(SponsorBenefit, SponsorBenefitAdmin)
<commit_msg>fix merge conflict resolve mistake<commit_after>from django.contrib import admin
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _

from symposion.sponsorship.models import SponsorLevel, Sponsor, Benefit, \
    BenefitLevel, SponsorBenefit, BENEFITS

class BenefitLevelInline(admin.TabularInline):
    model = BenefitLevel
    extra = 0

class SponsorBenefitInline(admin.StackedInline):
    model = SponsorBenefit
    extra = 0
    fieldsets = [
        (None, {
            "fields": [
                ("benefit", "active"),
                ("max_words", "other_limits"),
                "text",
                "upload",
            ]
        })
    ]

class SponsorAdmin(admin.ModelAdmin):

    save_on_top = True
    fieldsets = [
        (None, {
            "fields": [
                ("name", "applicant"),
                ("level", "active"),
                "external_url",
                "annotation",
                ("contact_name", "contact_email")
            ]
        }),
        ("Metadata", {
            "fields": ["added"],
            "classes": ["collapse"]
        })
    ]
    inlines = [SponsorBenefitInline]
    list_display = ["name", "external_url", "level", "active", "contact", "applicant_field"]
    list_filter = ["level", "active"]

    def contact(self, sponsor):
        return mark_safe('%s' % (escape(sponsor.contact_email), escape(sponsor.contact_name)))

    def applicant_field(self, sponsor):
        name = sponsor.applicant.get_full_name()
        email = sponsor.applicant.email
        return mark_safe('%s' % (escape(email), escape(name)))
    applicant_field.short_description = _(u"Applicant")

    def get_form(self, *args, **kwargs):
        # @@@ kinda ugly but using choices= on NullBooleanField is broken
        form = super(SponsorAdmin, self).get_form(*args, **kwargs)
        form.base_fields["active"].widget.choices = [
            (u"1", _("unreviewed")),
            (u"2", _("approved")),
            (u"3", _("rejected"))
        ]
        return form

    # Define accessor functions for our benefit fields and add them to
    # list_display, so we can sort on them and give them sensible names.
    # Add the fields to list_filters while we're at it.
    for benefit in BENEFITS:
        benefit_name = benefit['name']
        field_name = benefit['field_name']

        def func_generator(ben):
            def column_func(obj):
                return getattr(obj, ben['field_name'])
            column_func.short_description = ben['column_title']
            column_func.boolean = True
            column_func.admin_order_field = ben['field_name']
            return column_func
        list_display.append(func_generator(benefit))
        list_filter.append(field_name)

    def save_related(self, request, form, formsets, change):
        super(SponsorAdmin, self).save_related(request, form, formsets, change)
        obj = form.instance
        obj.save()

class BenefitAdmin(admin.ModelAdmin):

    list_display = ["name", "type", "description", "levels"]
    inlines = [BenefitLevelInline]

    def levels(self, benefit):
        return u", ".join(l.level.name for l in benefit.benefit_levels.all())

class SponsorLevelAdmin(admin.ModelAdmin):

    inlines = [BenefitLevelInline]

class SponsorBenefitAdmin(admin.ModelAdmin):
    list_display = ('benefit', 'sponsor', 'active', '_is_complete', 'show_text')

    def show_text(self, sponsor_benefit):
        if sponsor_benefit.text:
            return sponsor_benefit.text[:100]
        else:
            return _("None")

admin.site.register(SponsorLevel, SponsorLevelAdmin)
admin.site.register(Sponsor, SponsorAdmin)
admin.site.register(Benefit, BenefitAdmin)
admin.site.register(SponsorBenefit, SponsorBenefitAdmin)
<|endoftext|>


<commit_before><commit_msg>properly handle duplicate files in ignition<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import luigi

from .genome_mapping_tasks import GetFasta
from .genome_mapping_tasks import CleanSplitFasta
from .genome_mapping_tasks import GetChromosomes
from .genome_mapping_tasks import BlatJob
from .genome_mapping_tasks import ParsePslOutput
from .genome_mapping_tasks import SpeciesBlatJob

from .pgload_exact_matches import GenomeMappingPGLoadExactMatches
from .pgload_inexact_matches import GenomeMappingPGLoadInexactMatches

GENOMES = {
    9606: 'homo_sapiens',
    10090: 'mus_musculus',
    10116: 'rattus_norvegicus',
    7227: 'drosophila_melanogaster',
    7165: 'anopheles_gambiae',
    # 352472: 'dictyostelium_discoideum',
    36329: 'plasmodium_falciparum',
    3702: 'arabidopsis_thaliana',
    7091: 'bombyx_mori',
    # 284812: 'schizosaccharomyces_pombe',
    559292: 'saccharomyces_cerevisiae',
    6239: 'caenorhabditis_elegans',
}

def get_taxids_for_genome_mapping():
    """
    Get taxids for genomes that are used for mapping.
    """
    return GENOMES.keys()

class SpeciesFastaExportWrapper(luigi.WrapperTask):
    """
    A wrapper task to export fasta files for all species that will be mapped
    to the reference genomes using blat.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield GetFasta(taxid=taxid)

class SpeciesFastaCleanSplitWrapper(luigi.WrapperTask):
    """
    A wrapper task to keep only sequences of certain length and split fasta
    files in chunks.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield CleanSplitFasta(taxid=taxid)

class GetChromosomeFastaWrapper(luigi.WrapperTask):
    """
    A wrapper task for getting a list of all chromosome fasta files
    that are used in parallel blat searches.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield GetChromosomes(taxid=taxid)

class BlatJobsWrapper(luigi.WrapperTask):
    """
    A wrapper task for running blat searches of all split RNAcentral fasta files
    against all chromosomes within the same species.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            SpeciesBlatJob(taxid=taxid)

class ParsePslOutputWrapper(luigi.WrapperTask):
    """
    A wrapper task for parsing all blat output into tsv files that can be
    loaded into the database.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield ParsePslOutput(taxid=taxid)

class PGLoadGenomeMappingWrapper(luigi.WrapperTask):
    """
    A wrapper task for loading parsed blat output into the database.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield [
                GenomeMappingPGLoadExactMatches(taxid=taxid),
                GenomeMappingPGLoadInexactMatches(taxid=taxid),
            ]

class GenomeMappingPipelineWrapper(luigi.WrapperTask):
    """
    A wrapper task for the entire genome mapping pipeline.
    """
    def requires(self):
        yield GetChromosomeFastaWrapper()
        yield SpeciesFastaCleanSplitWrapper()
        yield BlatJobsWrapper()
        yield ParsePslOutputWrapper()
        yield PGLoadGenomeMappingWrapper()
<commit_msg>Update the list of genomes for which the mapping runs<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2018] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import luigi

from .genome_mapping_tasks import GetFasta
from .genome_mapping_tasks import CleanSplitFasta
from .genome_mapping_tasks import GetChromosomes
from .genome_mapping_tasks import BlatJob
from .genome_mapping_tasks import ParsePslOutput
from .genome_mapping_tasks import SpeciesBlatJob

from .pgload_exact_matches import GenomeMappingPGLoadExactMatches
from .pgload_inexact_matches import GenomeMappingPGLoadInexactMatches

GENOMES = {
    9606: 'homo_sapiens',
    10090: 'mus_musculus',
    10116: 'rattus_norvegicus',
    7227: 'drosophila_melanogaster',
    #7165: 'anopheles_gambiae',
    # 352472: 'dictyostelium_discoideum',
    # 36329: 'plasmodium_falciparum',
    3702: 'arabidopsis_thaliana',
    7091: 'bombyx_mori',
    # 284812: 'schizosaccharomyces_pombe', # wrong taxid = need to use 4896
    # 559292: 'saccharomyces_cerevisiae', # correct taxid but ensembl uses something else
    6239: 'caenorhabditis_elegans',
}

def get_taxids_for_genome_mapping():
    """
    Get taxids for genomes that are used for mapping.
    """
    return GENOMES.keys()

class SpeciesFastaExportWrapper(luigi.WrapperTask):
    """
    A wrapper task to export fasta files for all species that will be mapped
    to the reference genomes using blat.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield GetFasta(taxid=taxid)

class SpeciesFastaCleanSplitWrapper(luigi.WrapperTask):
    """
    A wrapper task to keep only sequences of certain length and split fasta
    files in chunks.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield CleanSplitFasta(taxid=taxid)

class GetChromosomeFastaWrapper(luigi.WrapperTask):
    """
    A wrapper task for getting a list of all chromosome fasta files
    that are used in parallel blat searches.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield GetChromosomes(taxid=taxid)

class BlatJobsWrapper(luigi.WrapperTask):
    """
    A wrapper task for running blat searches of all split RNAcentral fasta files
    against all chromosomes within the same species.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            SpeciesBlatJob(taxid=taxid)

class ParsePslOutputWrapper(luigi.WrapperTask):
    """
    A wrapper task for parsing all blat output into tsv files that can be
    loaded into the database.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield ParsePslOutput(taxid=taxid)

class PGLoadGenomeMappingWrapper(luigi.WrapperTask):
    """
    A wrapper task for loading parsed blat output into the database.
    """
    def requires(self):
        for taxid in get_taxids_for_genome_mapping():
            yield [
                GenomeMappingPGLoadExactMatches(taxid=taxid),
                GenomeMappingPGLoadInexactMatches(taxid=taxid),
            ]

class GenomeMappingPipelineWrapper(luigi.WrapperTask):
    """
    A wrapper task for the entire genome mapping pipeline.
    """
    def requires(self):
        yield GetChromosomeFastaWrapper()
        yield SpeciesFastaCleanSplitWrapper()
        yield BlatJobsWrapper()
        yield ParsePslOutputWrapper()
        yield PGLoadGenomeMappingWrapper()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

from datetime import datetime
import time
import functools
from collections import defaultdict

from turbo.util import import_object

from bson.objectid import ObjectId
from pymongo import ASCENDING, DESCENDING

from turbo.log import model_log
from turbo.util import escape as _es

class Record(dict):
    """a dict object to replace default mongodb record

    """
    def __init__(self, record, *args, **kwargs):
        super(Record, self).__init__(*args, **kwargs)
        self.update(record)

    def __getitem__(self, key, default=None):
        return super(Record, self).get(key, default)

def convert_to_record(func):
    """wrap mongodb record to a dict record

    """

    @functools.wraps(func)
    def wrapper(self, *args, **kwargs):
        result = func(self, *args, **kwargs)
        if result is not None:
            if isinstance(result, dict):
                return Record(result)

            return (Record(i) for i in result)

        return result

    return wrapper

class MixinModel(object):

    @staticmethod
    def utctimestamp(seconds=None):
        if seconds:
            return long(time.mktime(time.gmtime(seconds)))
        else:
            return long(time.mktime(time.gmtime()))

    @staticmethod
    def timestamp():
        return long(time.time())

    @staticmethod
    def datetime(dt=None):
        if dt:
            return datetime.strptime(dt, '%Y-%m-%d %H:%M')
        else:
            return datetime.now()

    @staticmethod
    def utcdatetime(dt=None):
        if dt:
            return datetime.strptime(dt, '%Y-%m-%d %H:%M')
        else:
            return datetime.utcnow()

    @staticmethod
    def to_one_str(value, *args, **kwargs):
        """string化的单个record对象
        """
        if kwargs.get('wrapper'):
            return MixinModel._wrapper_to_one_str(value)

        return _es.to_dict_str(value)

    @staticmethod
    def to_str(values, callback=None):
        """string化的多个record组成的list对象
        """
        if callback and callable(callback):
            return [callback(MixinModel.to_one_str(i)) for i in values]

        return [MixinModel.to_one_str(i) for i in values]

    @staticmethod
    @convert_to_record
    def _wrapper_to_one_str(value):
        return _es.to_dict_str(value)

    @staticmethod
    def default_encode(v):
        return _es.default_encode(v)

    @staticmethod
    def json_encode(v):
        return _es.json_encode(v)

    @staticmethod
    def json_decode(v):
        return _es.json_decode(v)

    @staticmethod
    def to_objectid(objid):
        return _es.to_objectid(objid)

    @staticmethod
    def create_objectid():
        """return ObjectId
        """
        return ObjectId()

    _instance = {}

    @classmethod
    def instance(cls, name):
        """
        instance application model
        """
        if not cls._instance.get(name):
            model_name = name.split('.')
            ins_name = '.'.join(['models', model_name[0], 'model', model_name[1]])
            cls._instance[name] = cls.import_model(ins_name)()

        return cls._instance[name]

    @classmethod
    def import_model(cls, ins_name):
        try:
            package_space = getattr(cls,  'package_space')
        except AttributeError:
            raise NotImplementedError
        else:
            return import_object(ins_name, package_space)

    @staticmethod
    @property
    def default_record():
        """
        generate one default record which return '' when key is empty
        """
        return defaultdict(lambda: '')

class BaseBaseModel(MixinModel):
    """mongodb 基础api 访问

    """

    name = None                            # mongodb collection name
    field = None                           # collection key
    column = None                          # need to query field
    _operators = {
        '$set': '',
        '$unset': '',
        '$rename': '',
        '$currentDate': '',
        '$inc': '',
        '$max': '',
        '$min': '',
        '$mul': '',
        '$setOnInsert': '',

        '$addToSet': '',
        '$pop': '',
        '$pushAll': '',
        '$push': '',
    }

    def __init__(self, db_name='test', _MONGO_DB_MAPPING=None):
        if _MONGO_DB_MAPPING is None:
            raise Exception("db mapping is invalid")

        # databases
        self.__db = _MONGO_DB_MAPPING['db']
        # databases file
        self.__db_file = _MONGO_DB_MAPPING['db_file']

        #databse name
        if db_name not in self.__db or self.__db.get(db_name, None) is None:
            raise Exception("%s is invalid databse" % db_name)

        # collection name
        if not self.name:
            raise Exception("%s is invalid collection name" % self.name)

        # collection field
        if not self.field or not isinstance(self.field, dict):
            raise Exception("%s is invalid collection field" % self.field)

        # collect as private variable
        self.__collect = getattr(self.__db.get(db_name, object), self.name, None)
        if self.__collect is None:
            raise Exception("%s is invalid collection" % self.name)

        # gridfs as private variable
        self.__gridfs = self.__db_file.get(db_name, None)
        if self.__gridfs is None:
            model_log.warning("%s is invalid gridfs" % self.__gridfs)

    def __setitem__(self, k, v):
        setattr(self, k, v)

    def __getitem__(self, k):
        getattr(self, k)

    def __str__(self):
        if isinstance(self.field, dict):
            return unicode(self.field)

        return None

    # pymongo collection method
    def insert(self, doc_or_docs, **kwargs):
        """collection insert method
        args:
            w(optional):(integer or string) If this is a replica set,
            write operations will block until they have been replicated to the specified number or tagged set of servers.
            w=<int> always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to two secondaries).
            Passing w=0 disables write acknowledgement and all other write concern options.

            wtimeout(optional): (integer) Used in conjunction with w.
            Specify a value in milliseconds to control how long to wait for write propagation to complete.
            If replication does not complete in the given timeframe, a timeout exception is raised.

        """
        if not kwargs.get('check') == False:
            return self.create(doc_or_docs, **kwargs)

        return self.__collect.insert(doc_or_docs, **kwargs)

    def save(self, to_save, **kwargs):
        """collection save method

        """
        return self.__collect.save(to_save, **kwargs)

    def find_one(self, spec_or_id=None, *args, **kwargs):
        """collection find_one method

        """
        if kwargs.get('wrapper', False):
            return self._wrapper_find_one(spec_or_id, *args, **kwargs)

        return self.__collect.find_one(spec_or_id, *args, **kwargs)

    def find(self, *args, **kwargs):
        """collection find method

        """
        if kwargs.get('wrapper', False):
            return self._wrapper_find(*args, **kwargs)

        return self.__collect.find(*args, **kwargs)

    @convert_to_record
    def _wrapper_find_one(self, spec_or_id=None, *args, **kwargs):
        """convert record to a dict that has no key error

        """
        return self.__collect.find_one(spec_or_id, *args, **kwargs)

    @convert_to_record
    def _wrapper_find(self, *args, **kwargs):
        """convert record to a dict that has no key error

        """
        return self.__collect.find(*args, **kwargs)

    def update(self, spec, document, multi=False, **kwargs):
        """collection update method

        """
        for opk in document.keys():
            if not opk.startswith('$') or opk not in self._operators:
                raise Exception("invalid document update operator")

        return self.__collect.update(spec, document, multi=multi, **kwargs)

    def remove(self, spec_or_id=None, **kwargs):
        """collection remove method
        warning:
            if you want to remove all documents,
            you must override _remove_all method to make sure
            you understand the result what you do
        """
        if isinstance(spec_or_id, dict) and spec_or_id == {}:
            raise Exception("not allowed remove all documents")

        if spec_or_id is None:
            raise Exception("not allowed remove all documents")

        return self.__collect.remove(spec_or_id, **kwargs)

    # gridfs method
    def put(self, value, **kwargs):
        """gridfs put method
        """
        if value:
            return self.__gridfs.put(value, **kwargs)

        return None

    def delete(self, _id):
        """gridfs delete method
        """
        return self.__gridfs.delete(self.to_objectid(_id))

    def get(self, _id):
        """gridfs get method
        """
        return self.__gridfs.get(self.to_objectid(_id))

    def read(self, _id):
        """gridfs read method
        """
        return self.__gridfs.get(self.to_objectid(_id)).read()

    def find_by_id(self, _id, column=None):
        """find record by _id
        """
        if isinstance(_id, list) or isinstance(_id, tuple):
            return (self.find_by_id(i, column) for i in _id if i)

        document_id = self.to_objectid(_id)

        if document_id is None:
            return None

        return self.__collect.find_one({'_id': document_id}, column)

    def remove_by_id(self, _id):
        if isinstance(_id, list) or isinstance(_id, tuple):
            return (self.remove_by_id(i) for i in _id)

        return self.__collect.remove({'_id': self.to_objectid(_id)})

    def find_and_modify(self, query=None, update=None, upsert=False, sort=None, full_response=False, **kwargs):
        return  self.__collect.find_and_modify(query=query, update=update, upsert=upsert, sort=sort, full_response=full_response, **kwargs)

    def ensure_index(self, key_or_list, cache_for=300, **kwargs):
        return self.__collect.ensure_index(key_or_list, cache_for=cache_for, **kwargs)

    def find_new_one(self, *args, **kwargs):
        cur = list(self.__collect.find(*args, **kwargs).limit(1).sort('_id', DESCENDING))
        if cur:
            return cur[0]

        return None

    def get_as_column(self, condition=None, column=None, skip=0, limit=0, sort=None):
        if column is None:
            column = self.column

        return self.find(condition, column, skip=skip, limit=limit, sort=sort)

    def get_as_dict(self, condition=None, column=None, skip=0, limit=0, sort=None):
        if column is None:
            column = self.column

        l = self.__collect.find(condition, column, skip=skip, limit=limit, sort=sort)

        as_dict, as_list = {}, []
        for i in l:
            as_dict[unicode(i['_id'])] = i
            as_list.append(i)

        return as_dict, as_list

    def create(self, record=None, **args):
        """init the new record
        function:
            创建一个新的record，并为数据一致性做初始化
        """
        if isinstance(record, list) or isinstance(record, tuple):
            for i in record:
                i = self._valid_record(i)

        if isinstance(record, dict):
            record = self._valid_record(record)

        return self.__collect.insert(record, **args)

    def _valid_record(self, record):
        if not isinstance(record, dict):
            raise Exception("%s record is not dict" % record)

        rset = set(record.keys())
        fset = set(self.field.keys())

        try:
            rset.remove('_id')
        except Exception as e:
            pass

        try:
            fset.remove('_id')
        except Exception as e:
            pass

        if not (fset ^ rset) <= fset:
            raise Exception("record keys is not equal to fields keys %s" % (list((fset ^ rset)-fset)))

        for k, v in self.field.items():
            if k not in record:
                if v[0] is datetime and not v[1]:
                    record[k] = self.datetime()
                    continue

                if v[0] is time and not v[1]:
                    record[k] = self.timestamp()
                    continue

                record[k] = v[1]

        return record

    def inc(self, spec_or_id, key, num=1):
        self.__collect.update(spec_or_id, {'$inc': {key: num}})

class BaseModel(BaseBaseModel):
    """全局公共业务方法

    """

    def __init__(self, db_name='test', _MONGO_DB_MAPPING=None):
        super(BaseModel, self).__init__(db_name, _MONGO_DB_MAPPING)
<commit_msg>fix model to_str<commit_after># -*- coding:utf-8 -*-

from datetime import datetime
import time
import functools
from collections import defaultdict

from turbo.util import import_object

from bson.objectid import ObjectId
from pymongo import ASCENDING, DESCENDING

from turbo.log import model_log
from turbo.util import escape as _es

class Record(dict):
    """a dict object to replace default mongodb record

    """
    def __init__(self, record, *args, **kwargs):
        super(Record, self).__init__(*args, **kwargs)
        self.update(record)

    def __getitem__(self, key, default=None):
        return super(Record, self).get(key, default)

def convert_to_record(func):
    """wrap mongodb record to a dict record

    """

    @functools.wraps(func)
    def wrapper(self, *args, **kwargs):
        result = func(self, *args, **kwargs)
        if result is not None:
            if isinstance(result, dict):
                return Record(result)

            return (Record(i) for i in result)

        return result

    return wrapper

class MixinModel(object):

    @staticmethod
    def utctimestamp(seconds=None):
        if seconds:
            return long(time.mktime(time.gmtime(seconds)))
        else:
            return long(time.mktime(time.gmtime()))

    @staticmethod
    def timestamp():
        return long(time.time())

    @staticmethod
    def datetime(dt=None):
        if dt:
            return datetime.strptime(dt, '%Y-%m-%d %H:%M')
        else:
            return datetime.now()

    @staticmethod
    def utcdatetime(dt=None):
        if dt:
            return datetime.strptime(dt, '%Y-%m-%d %H:%M')
        else:
            return datetime.utcnow()

    @classmethod
    def to_one_str(cls, value, *args, **kwargs):
        """string化的单个record对象
        """
        if kwargs.get('wrapper'):
            return cls._wrapper_to_one_str(value)

        return _es.to_dict_str(value)

    @classmethod
    def to_str(cls, values, callback=None):
        """string化的多个record组成的list对象
        """
        if callback and callable(callback):
            return [callback(cls.to_one_str(i)) for i in values]

        return [cls.to_one_str(i) for i in values]

    @staticmethod
    @convert_to_record
    def _wrapper_to_one_str(value):
        return _es.to_dict_str(value)

    @staticmethod
    def default_encode(v):
        return _es.default_encode(v)

    @staticmethod
    def json_encode(v):
        return _es.json_encode(v)

    @staticmethod
    def json_decode(v):
        return _es.json_decode(v)

    @staticmethod
    def to_objectid(objid):
        return _es.to_objectid(objid)

    @staticmethod
    def create_objectid():
        """return ObjectId
        """
        return ObjectId()

    _instance = {}

    @classmethod
    def instance(cls, name):
        """
        instance application model
        """
        if not cls._instance.get(name):
            model_name = name.split('.')
            ins_name = '.'.join(['models', model_name[0], 'model', model_name[1]])
            cls._instance[name] = cls.import_model(ins_name)()

        return cls._instance[name]

    @classmethod
    def import_model(cls, ins_name):
        try:
            package_space = getattr(cls,  'package_space')
        except AttributeError:
            raise NotImplementedError
        else:
            return import_object(ins_name, package_space)

    @staticmethod
    @property
    def default_record():
        """
        generate one default record which return '' when key is empty
        """
        return defaultdict(lambda: '')

class BaseBaseModel(MixinModel):
    """mongodb 基础api 访问

    """

    name = None                            # mongodb collection name
    field = None                           # collection key
    column = None                          # need to query field
    _operators = {
        '$set': '',
        '$unset': '',
        '$rename': '',
        '$currentDate': '',
        '$inc': '',
        '$max': '',
        '$min': '',
        '$mul': '',
        '$setOnInsert': '',

        '$addToSet': '',
        '$pop': '',
        '$pushAll': '',
        '$push': '',
    }

    def __init__(self, db_name='test', _MONGO_DB_MAPPING=None):
        if _MONGO_DB_MAPPING is None:
            raise Exception("db mapping is invalid")

        # databases
        self.__db = _MONGO_DB_MAPPING['db']
        # databases file
        self.__db_file = _MONGO_DB_MAPPING['db_file']

        #databse name
        if db_name not in self.__db or self.__db.get(db_name, None) is None:
            raise Exception("%s is invalid databse" % db_name)

        # collection name
        if not self.name:
            raise Exception("%s is invalid collection name" % self.name)

        # collection field
        if not self.field or not isinstance(self.field, dict):
            raise Exception("%s is invalid collection field" % self.field)

        # collect as private variable
        self.__collect = getattr(self.__db.get(db_name, object), self.name, None)
        if self.__collect is None:
            raise Exception("%s is invalid collection" % self.name)

        # gridfs as private variable
        self.__gridfs = self.__db_file.get(db_name, None)
        if self.__gridfs is None:
            model_log.warning("%s is invalid gridfs" % self.__gridfs)

    def __setitem__(self, k, v):
        setattr(self, k, v)

    def __getitem__(self, k):
        getattr(self, k)

    def __str__(self):
        if isinstance(self.field, dict):
            return unicode(self.field)

        return None

    # pymongo collection method
    def insert(self, doc_or_docs, **kwargs):
        """collection insert method
        args:
            w(optional):(integer or string) If this is a replica set,
            write operations will block until they have been replicated to the specified number or tagged set of servers.
            w=<int> always includes the replica set primary (e.g. w=3 means write to the primary and wait until replicated to two secondaries).
            Passing w=0 disables write acknowledgement and all other write concern options.

            wtimeout(optional): (integer) Used in conjunction with w.
            Specify a value in milliseconds to control how long to wait for write propagation to complete.
            If replication does not complete in the given timeframe, a timeout exception is raised.

        """
        if not kwargs.get('check') == False:
            return self.create(doc_or_docs, **kwargs)

        return self.__collect.insert(doc_or_docs, **kwargs)

    def save(self, to_save, **kwargs):
        """collection save method

        """
        return self.__collect.save(to_save, **kwargs)

    def find_one(self, spec_or_id=None, *args, **kwargs):
        """collection find_one method

        """
        if kwargs.get('wrapper', False):
            return self._wrapper_find_one(spec_or_id, *args, **kwargs)

        return self.__collect.find_one(spec_or_id, *args, **kwargs)

    def find(self, *args, **kwargs):
        """collection find method

        """
        if kwargs.get('wrapper', False):
            return self._wrapper_find(*args, **kwargs)

        return self.__collect.find(*args, **kwargs)

    @convert_to_record
    def _wrapper_find_one(self, spec_or_id=None, *args, **kwargs):
        """convert record to a dict that has no key error

        """
        return self.__collect.find_one(spec_or_id, *args, **kwargs)

    @convert_to_record
    def _wrapper_find(self, *args, **kwargs):
        """convert record to a dict that has no key error

        """
        return self.__collect.find(*args, **kwargs)

    def update(self, spec, document, multi=False, **kwargs):
        """collection update method

        """
        for opk in document.keys():
            if not opk.startswith('$') or opk not in self._operators:
                raise Exception("invalid document update operator")

        return self.__collect.update(spec, document, multi=multi, **kwargs)

    def remove(self, spec_or_id=None, **kwargs):
        """collection remove method
        warning:
            if you want to remove all documents,
            you must override _remove_all method to make sure
            you understand the result what you do
        """
        if isinstance(spec_or_id, dict) and spec_or_id == {}:
            raise Exception("not allowed remove all documents")

        if spec_or_id is None:
            raise Exception("not allowed remove all documents")

        return self.__collect.remove(spec_or_id, **kwargs)

    # gridfs method
    def put(self, value, **kwargs):
        """gridfs put method
        """
        if value:
            return self.__gridfs.put(value, **kwargs)

        return None

    def delete(self, _id):
        """gridfs delete method
        """
        return self.__gridfs.delete(self.to_objectid(_id))

    def get(self, _id):
        """gridfs get method
        """
        return self.__gridfs.get(self.to_objectid(_id))

    def read(self, _id):
        """gridfs read method
        """
        return self.__gridfs.get(self.to_objectid(_id)).read()

    def find_by_id(self, _id, column=None):
        """find record by _id
        """
        if isinstance(_id, list) or isinstance(_id, tuple):
            return (self.find_by_id(i, column) for i in _id if i)

        document_id = self.to_objectid(_id)

        if document_id is None:
            return None

        return self.__collect.find_one({'_id': document_id}, column)

    def remove_by_id(self, _id):
        if isinstance(_id, list) or isinstance(_id, tuple):
            return (self.remove_by_id(i) for i in _id)

        return self.__collect.remove({'_id': self.to_objectid(_id)})

    def find_and_modify(self, query=None, update=None, upsert=False, sort=None, full_response=False, **kwargs):
        return  self.__collect.find_and_modify(query=query, update=update, upsert=upsert, sort=sort, full_response=full_response, **kwargs)

    def ensure_index(self, key_or_list, cache_for=300, **kwargs):
        return self.__collect.ensure_index(key_or_list, cache_for=cache_for, **kwargs)

    def find_new_one(self, *args, **kwargs):
        cur = list(self.__collect.find(*args, **kwargs).limit(1).sort('_id', DESCENDING))
        if cur:
            return cur[0]

        return None

    def get_as_column(self, condition=None, column=None, skip=0, limit=0, sort=None):
        if column is None:
            column = self.column

        return self.find(condition, column, skip=skip, limit=limit, sort=sort)

    def get_as_dict(self, condition=None, column=None, skip=0, limit=0, sort=None):
        if column is None:
            column = self.column

        l = self.__collect.find(condition, column, skip=skip, limit=limit, sort=sort)

        as_dict, as_list = {}, []
        for i in l:
            as_dict[unicode(i['_id'])] = i
            as_list.append(i)

        return as_dict, as_list

    def create(self, record=None, **args):
        """init the new record
        function:
            创建一个新的record，并为数据一致性做初始化
        """
        if isinstance(record, list) or isinstance(record, tuple):
            for i in record:
                i = self._valid_record(i)

        if isinstance(record, dict):
            record = self._valid_record(record)

        return self.__collect.insert(record, **args)

    def _valid_record(self, record):
        if not isinstance(record, dict):
            raise Exception("%s record is not dict" % record)

        rset = set(record.keys())
        fset = set(self.field.keys())

        try:
            rset.remove('_id')
        except Exception as e:
            pass

        try:
            fset.remove('_id')
        except Exception as e:
            pass

        if not (fset ^ rset) <= fset:
            raise Exception("record keys is not equal to fields keys %s" % (list((fset ^ rset)-fset)))

        for k, v in self.field.items():
            if k not in record:
                if v[0] is datetime and not v[1]:
                    record[k] = self.datetime()
                    continue

                if v[0] is time and not v[1]:
                    record[k] = self.timestamp()
                    continue

                record[k] = v[1]

        return record

    def inc(self, spec_or_id, key, num=1):
        self.__collect.update(spec_or_id, {'$inc': {key: num}})

class BaseModel(BaseBaseModel):
    """全局公共业务方法

    """

    def __init__(self, db_name='test', _MONGO_DB_MAPPING=None):
        super(BaseModel, self).__init__(db_name, _MONGO_DB_MAPPING)
<|endoftext|>


<commit_before>from masterapp.lib.base import request
from masterapp.model import Session, User
from pylons import session, g
from decimal import Decimal

def build_json(results):
    json = { "data": []}
    for row in results:
        lrow = {}
        for key in row.keys():
            value = getattr(row, key)
            if isinstance(value, Decimal):
                value = int(value)
            lrow[key] = value
        json['data'].append(lrow)
        json['data'][len(json['data'])-1]['type'] = request.params.get('type')
    json['success']=True
    return json

def get_user():
    friendid = request.params.get('friend')
    if not friendid:
        user = Session.query(User).get(session['userid'])
    else:
        # TODO: Make sure they're friends
        user = Session.query(User).get(friendid)
        if not user:
            abort(400)
    return user

def get_session_user():
    return Session.query(User).get(session['userid'])

def is_ie6(user_agent):
	if user_agent.find('MSIE 6.0') >= 0:
		for not_6 in ('MSIE 7.0', 'Opera', 'Netscape'):
			if user_agent.find(not_6) >= 0:
				return False
		return True
	else:
		return False
<commit_msg>put in a forgetten import<commit_after>from masterapp.lib.base import request
from masterapp.model import Session, User
from pylons import session, g
from pylons.controllers.util import abort
from decimal import Decimal

def build_json(results):
    json = { "data": []}
    for row in results:
        lrow = {}
        for key in row.keys():
            value = getattr(row, key)
            if isinstance(value, Decimal):
                value = int(value)
            lrow[key] = value
        json['data'].append(lrow)
        json['data'][len(json['data'])-1]['type'] = request.params.get('type')
    json['success']=True
    return json

def get_user():
    friendid = request.params.get('friend')
    if not friendid:
        user = Session.query(User).get(session['userid'])
    else:
        # TODO: Make sure they're friends
        user = Session.query(User).get(friendid)
        if not user:
            abort(400)
    return user

def get_session_user():
    return Session.query(User).get(session['userid'])

def is_ie6(user_agent):
	if user_agent.find('MSIE 6.0') >= 0:
		for not_6 in ('MSIE 7.0', 'Opera', 'Netscape'):
			if user_agent.find(not_6) >= 0:
				return False
		return True
	else:
		return False
<|endoftext|>


<commit_before>"""
Input and Output ports for PortMidi.

There is no need to use this module directly. All you need is
available in the toplevel module.
"""
import time

from .parser import Parser
from .messages import Message

class PortMidiInitializer:
    """
    This class is responsible for loading PortMidi on demand. It
    camouflages in the global name space as a module, and forwards all
    attribute access to the real module.

    For testing, this mock module can be replaced with another mock
    module.
    """
    def __init__(self):
        self.pm = None

    def __getattr__(self, attr):
        # print('initializer was asked for', attr)
        if self.pm is None:
            # print('Initializing PortMidi...')
            from . import portmidi_init as pm
            self.pm = pm
            self.pm.lib.Pm_Initialize()

        return getattr(self.pm, attr)

    def __del__(self):
        if self.pm is not None:
            # print('Terminating PortMidi...')
            self.pm.lib.Pm_Terminate()
            self.pm = None

pm = PortMidiInitializer()

def _check_error(return_value):
    """Raise IOError if return_value < 0.

    The exception will be raised with the error message from PortMidi.
    """
    if return_value < 0:
        raise IOError(pm.lib.Pm_GetErrorText(return_value))

class DeviceInfo(object):
    """
    Info about a PortMidi device.

        device_id   an integer
        interface   interface name (for example 'ALSA')
        name        device name (the same as port name)
        is_input    boolean, True if this is an input device
        is_output   boolean, True if this is an output device
    """

    def __init__(self, device_id):
        """Create a new DeviceInfo object."""
        info_pointer = pm.lib.Pm_GetDeviceInfo(device_id)
        if not info_pointer:
            raise IOError('PortMidi device with id={} not found'.format(
                          device_id))
        info = info_pointer.contents

        self.device_id = device_id
        self.interface = info.interface.decode('utf-8')
        self.name = info.name.decode('utf-8')
        self.is_input = info.is_input
        self.is_output = info.is_output
        self.opened = bool(info.opened)

    def __repr__(self):
        if self.opened:
            state = 'open'
        else:
            state = 'closed'

        if self.is_input:
            device_type = 'input'
        else:
            device_type = 'output'

        return "<{state} {device_type} device '{self.name}'" \
            " '{self.interface}'>" \
            "".format(**locals())

def get_devices():
    """Return a list of DeviceInfo objects, one for each PortMidi device."""
    devices = []
    for device_id in range(pm.lib.Pm_CountDevices()):
        devices.append(DeviceInfo(device_id))

    return devices

def input_names():
    """Return a sorted list of all input port names.

    These names can be passed to Input().
    """
    names = [device.name for device in get_devices() if device.is_input]
    return list(sorted(names))

def output_names():
    """Return a sorted list of all input port names.

    These names can be passed to Output().
    """
    names = [device.name for device in get_devices() if device.is_output]
    return list(sorted(names))

class Port(object):
    """
    Abstract base class for PortMidi Input and Output ports.
    """

    def __init__(self, name=None):
        self.closed = True
        self._stream = pm.PortMidiStreamPtr()
        self.device = None

        opening_input = (self.__class__ is Input)

        if name is None:
            self.device = self._get_default_device(opening_input)
        else:
            self.device = self._get_named_device(name, opening_input)
        self.name = self.device.name

        # Make a shortcut, since this is so long
        device_id = self.device.device_id

        if opening_input:
            _check_error(pm.lib.Pm_OpenInput(
                         pm.byref(self._stream),
                         device_id,  # Input device
                         pm.null,    # Input driver info
                         1000,       # Buffer size
                         pm.NullTimeProcPtr,  # Time callback
                         pm.null))    # Time info
        else:
            _check_error(pm.lib.Pm_OpenOutput(
                         pm.byref(self._stream),
                         device_id,  # Output device
                         pm.null,    # Output diver info
                         0,          # Buffer size
                                     # (ignored when latency == 0?)
                         pm.NullTimeProcPtr,  # Default to internal clock
                         pm.null,    # Time info
                         0))         # Latency

        self.closed = False
        self.device.opened = True

    def _get_default_device(self, get_input):
        if get_input:
            device_id = pm.lib.Pm_GetDefaultInputDeviceID()
        else:
            device_id = pm.lib.Pm_GetDefaultOutputDeviceID()

        if device_id < 0:
            raise IOError('no default port found')

        return DeviceInfo(device_id)

    def _get_named_device(self, name, get_input):
        # Look for the device by name and type (input / output)
        for device in get_devices():
            if device.name != name:
                continue

            # Skip if device is the wrong type
            if get_input:
                if device.is_output:
                    continue
            else:
                if device.is_input:
                    continue

            if device.opened:
                raise IOError('port already opened: {!r}'.format(name))

            return device
        else:
            raise IOError('unknown port: {!r}'.format(name))

    def close(self):
        """Close the port.

        If the port is already closed, nothing will happen.
        The port is automatically closed when the object goes
        out of scope or is garbage collected.
        """

        if not self.closed:
            # Todo: Abort is not implemented for ALSA,
            # so we get a warning here.
            # But is it really needed?
            # _check_error(pm.lib.Pm_Abort(self._stream))

            _check_error(pm.lib.Pm_Close(self._stream))

            self.closed = True
            self.device.opened = False

    def __del__(self):
        self.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

    def __repr__(self):
        if self.closed:
            state = 'closed'
        else:
            state = 'open'

        if self.__class__ is Input:
            port_type = 'input'
        else:
            port_type = 'output'

        return "<{state} {port_type} port '{self.name}'" \
               " ({self.device.interface})>".format(**locals())

class Input(Port):
    """
    PortMidi Input port
    """

    def __init__(self, name=None):
        """Create an input port.

        name is the port name, as returned by input_names(). If
        name is not passed, the default input is used instead.
        """
        Port.__init__(self, name)
        self._parser = Parser()

    def pending(self):
        """Return how many messages are ready to be received.

        This can be used for non-blocking receive(), for example:

             for _ in range(port.pending()):
                 message = port.receive()

        Todo: return 0 or raise exception if the port is closed?
        """
        if self.closed:
            return self._parser.pending()

        # I get hanging notes if MAX_EVENTS > 1, so I'll have to
        # resort to calling Pm_Read() in a loop until there are no
        # more pending events.

        max_events = 1
        # Todo: this should be allocated once
        BufferType = pm.PmEvent * max_events
        read_buffer = BufferType()

        while pm.lib.Pm_Poll(self._stream):

            # Read one message. Should return 1.
            # If num_events < 0, an error occured.
            length = 1  # Buffer length
            num_events = pm.lib.Pm_Read(self._stream, read_buffer, length)
            _check_error(num_events)

            # Get the event
            event = read_buffer[0]
            # print('Received: {:x}'.format(event.message))

            # The bytes of the message are stored like this:
            #    0x00201090 -> (0x90, 0x10, 0x10)
            # (Todo: not sure if this is correct.)
            packed_message = event.message & 0xffffffff

            for i in range(4):
                byte = packed_message & 0xff
                self._parser.feed_byte(byte)
                packed_message >>= 8

        return self._parser.pending()

    def iter_pending(self):
        """Iterate through pending messages."""
        for _ in range(self.pending()):
            yield self.receive()

    def receive(self):
        """Return the next message.

        This will block until a message arrives. For non-blocking
        behavior, you can use pending() to see how many messages can
        safely be received without blocking:

            for _ in range(port.pending()):
                message = port.receive()

        NOTE: Blocking is currently implemented with polling and
        time.sleep(). This is inefficient, but the proper way doesn't
        work yet, so it's better than nothing.

        Todo: What should happen when the port is closed?
        - raise exception?
        - return pending messages until we run out, then
          raise exception?
        """

        # If there is a message pending, return it right away.
        message = self._parser.get_message()
        if message:
            return message

        if self.closed:
            raise ValueError('receive() called on closed port')

        # Wait for a message to arrive.
        while 1:
            time.sleep(0.001)
            if self.pending():
                # pending() has read at least one message from the
                # device. Return the first message.
                return self._parser.get_message()

    def __iter__(self):
        """Iterate through messages as they arrive on the port."""
        while 1:
            yield self.receive()

class Output(Port):
    """
    PortMidi output port
    """

    def __init__(self, name=None):
        """Create an output port

        name is the port name, as returned by output_names(). If
        name is not passed, the default output is used instead.
        """
        Port.__init__(self, name)

    def send(self, message):
        """Send a message."""
        if not isinstance(message, Message):
            raise TypeError('argument to send() must be a Message')

        if self.closed:
            raise ValueError('send() called on closed port')

        if message.type == 'sysex':
            # Sysex messages are written as a string.
            string = pm.c_char_p(bytes(message.bin()))
            timestamp = 0  # Ignored when latency = 0
            _check_error(pm.lib.Pm_WriteSysEx(self._stream, timestamp, string))
        else:
            # The bytes of a message as packed into a 32 bit integer.
            packed_message = 0
            for byte in reversed(message.bytes()):
                packed_message <<= 8
                packed_message |= byte

            timestamp = 0  # Ignored when latency = 0
            _check_error(pm.lib.Pm_WriteShort(self._stream,
                                              timestamp,
                                              packed_message))
<commit_msg>PortMidi ports now raise IOError if they are already open.<commit_after>"""
Input and Output ports for PortMidi.

There is no need to use this module directly. All you need is
available in the toplevel module.
"""
import time

from .parser import Parser
from .messages import Message

class PortMidiInitializer:
    """
    This class is responsible for loading PortMidi on demand. It
    camouflages in the global name space as a module, and forwards all
    attribute access to the real module.

    For testing, this mock module can be replaced with another mock
    module.
    """
    def __init__(self):
        self.pm = None

    def __getattr__(self, attr):
        # print('initializer was asked for', attr)
        if self.pm is None:
            # print('Initializing PortMidi...')
            from . import portmidi_init as pm
            self.pm = pm
            self.pm.lib.Pm_Initialize()

        return getattr(self.pm, attr)

    def __del__(self):
        if self.pm is not None:
            # print('Terminating PortMidi...')
            self.pm.lib.Pm_Terminate()
            self.pm = None

pm = PortMidiInitializer()

def _check_error(return_value):
    """Raise IOError if return_value < 0.

    The exception will be raised with the error message from PortMidi.
    """
    print(return_value)
    if return_value < 0:
        raise IOError(pm.lib.Pm_GetErrorText(return_value))

class DeviceInfo(object):
    """
    Info about a PortMidi device.

        device_id   an integer
        interface   interface name (for example 'ALSA')
        name        device name (the same as port name)
        is_input    boolean, True if this is an input device
        is_output   boolean, True if this is an output device
    """

    def __init__(self, device_id):
        """Create a new DeviceInfo object."""
        info_pointer = pm.lib.Pm_GetDeviceInfo(device_id)
        if not info_pointer:
            raise IOError('PortMidi device with id={} not found'.format(
                          device_id))
        info = info_pointer.contents

        self.device_id = device_id
        self.interface = info.interface.decode('utf-8')
        self.name = info.name.decode('utf-8')
        self.is_input = info.is_input
        self.is_output = info.is_output
        self.opened = bool(info.opened)

    def __repr__(self):
        if self.opened:
            state = 'open'
        else:
            state = 'closed'

        if self.is_input:
            device_type = 'input'
        else:
            device_type = 'output'

        return "<{state} {device_type} device '{self.name}'" \
            " '{self.interface}'>" \
            "".format(**locals())

def get_devices():
    """Return a list of DeviceInfo objects, one for each PortMidi device."""
    devices = []
    for device_id in range(pm.lib.Pm_CountDevices()):
        devices.append(DeviceInfo(device_id))

    return devices

def input_names():
    """Return a sorted list of all input port names.

    These names can be passed to Input().
    """
    names = [device.name for device in get_devices() if device.is_input]
    return list(sorted(names))

def output_names():
    """Return a sorted list of all input port names.

    These names can be passed to Output().
    """
    names = [device.name for device in get_devices() if device.is_output]
    return list(sorted(names))

class Port(object):
    """
    Abstract base class for PortMidi Input and Output ports.
    """

    def __init__(self, name=None):
        self.closed = True
        self._stream = pm.PortMidiStreamPtr()
        self.device = None

        opening_input = (self.__class__ is Input)

        if name is None:
            self.device = self._get_default_device(opening_input)
        else:
            self.device = self._get_named_device(name, opening_input)
        self.name = self.device.name

        if self.device.opened:
            if opening_input:
                devtype = 'input'
            else:
                devtype = 'output'
            raise IOError('{} port {!r} is already open'.format(devtype,
                                                                self.name))

        # Make a shortcut, since this is so long
        device_id = self.device.device_id

        if opening_input:
            _check_error(pm.lib.Pm_OpenInput(
                         pm.byref(self._stream),
                         device_id,  # Input device
                         pm.null,    # Input driver info
                         1000,       # Buffer size
                         pm.NullTimeProcPtr,  # Time callback
                         pm.null))    # Time info
        else:
            _check_error(pm.lib.Pm_OpenOutput(
                         pm.byref(self._stream),
                         device_id,  # Output device
                         pm.null,    # Output diver info
                         0,          # Buffer size
                                     # (ignored when latency == 0?)
                         pm.NullTimeProcPtr,  # Default to internal clock
                         pm.null,    # Time info
                         0))         # Latency

        self.closed = False
        self.device.opened = True

    def _get_default_device(self, get_input):
        if get_input:
            device_id = pm.lib.Pm_GetDefaultInputDeviceID()
        else:
            device_id = pm.lib.Pm_GetDefaultOutputDeviceID()

        if device_id < 0:
            raise IOError('no default port found')

        return DeviceInfo(device_id)

    def _get_named_device(self, name, get_input):
        # Look for the device by name and type (input / output)
        for device in get_devices():
            if device.name != name:
                continue

            # Skip if device is the wrong type
            if get_input:
                if device.is_output:
                    continue
            else:
                if device.is_input:
                    continue

            if device.opened:
                raise IOError('port already opened: {!r}'.format(name))

            return device
        else:
            raise IOError('unknown port: {!r}'.format(name))

    def close(self):
        """Close the port.

        If the port is already closed, nothing will happen.
        The port is automatically closed when the object goes
        out of scope or is garbage collected.
        """

        if not self.closed:
            # Todo: Abort is not implemented for ALSA,
            # so we get a warning here.
            # But is it really needed?
            # _check_error(pm.lib.Pm_Abort(self._stream))

            _check_error(pm.lib.Pm_Close(self._stream))

            self.closed = True
            self.device.opened = False

    def __del__(self):
        self.close()

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False

    def __repr__(self):
        if self.closed:
            state = 'closed'
        else:
            state = 'open'

        if self.__class__ is Input:
            port_type = 'input'
        else:
            port_type = 'output'

        return "<{state} {port_type} port '{self.name}'" \
               " ({self.device.interface})>".format(**locals())

class Input(Port):
    """
    PortMidi Input port
    """

    def __init__(self, name=None):
        """Create an input port.

        name is the port name, as returned by input_names(). If
        name is not passed, the default input is used instead.
        """
        Port.__init__(self, name)
        self._parser = Parser()

    def pending(self):
        """Return how many messages are ready to be received.

        This can be used for non-blocking receive(), for example:

             for _ in range(port.pending()):
                 message = port.receive()

        Todo: return 0 or raise exception if the port is closed?
        """
        if self.closed:
            return self._parser.pending()

        # I get hanging notes if MAX_EVENTS > 1, so I'll have to
        # resort to calling Pm_Read() in a loop until there are no
        # more pending events.

        max_events = 1
        # Todo: this should be allocated once
        BufferType = pm.PmEvent * max_events
        read_buffer = BufferType()

        while pm.lib.Pm_Poll(self._stream):

            # Read one message. Should return 1.
            # If num_events < 0, an error occured.
            length = 1  # Buffer length
            num_events = pm.lib.Pm_Read(self._stream, read_buffer, length)
            _check_error(num_events)

            # Get the event
            event = read_buffer[0]
            # print('Received: {:x}'.format(event.message))

            # The bytes of the message are stored like this:
            #    0x00201090 -> (0x90, 0x10, 0x10)
            # (Todo: not sure if this is correct.)
            packed_message = event.message & 0xffffffff

            for i in range(4):
                byte = packed_message & 0xff
                self._parser.feed_byte(byte)
                packed_message >>= 8

        return self._parser.pending()

    def iter_pending(self):
        """Iterate through pending messages."""
        for _ in range(self.pending()):
            yield self.receive()

    def receive(self):
        """Return the next message.

        This will block until a message arrives. For non-blocking
        behavior, you can use pending() to see how many messages can
        safely be received without blocking:

            for _ in range(port.pending()):
                message = port.receive()

        NOTE: Blocking is currently implemented with polling and
        time.sleep(). This is inefficient, but the proper way doesn't
        work yet, so it's better than nothing.

        Todo: What should happen when the port is closed?
        - raise exception?
        - return pending messages until we run out, then
          raise exception?
        """

        # If there is a message pending, return it right away.
        message = self._parser.get_message()
        if message:
            return message

        if self.closed:
            raise ValueError('receive() called on closed port')

        # Wait for a message to arrive.
        while 1:
            time.sleep(0.001)
            if self.pending():
                # pending() has read at least one message from the
                # device. Return the first message.
                return self._parser.get_message()

    def __iter__(self):
        """Iterate through messages as they arrive on the port."""
        while 1:
            yield self.receive()

class Output(Port):
    """
    PortMidi output port
    """

    def __init__(self, name=None):
        """Create an output port

        name is the port name, as returned by output_names(). If
        name is not passed, the default output is used instead.
        """
        Port.__init__(self, name)

    def send(self, message):
        """Send a message."""
        if not isinstance(message, Message):
            raise TypeError('argument to send() must be a Message')

        if self.closed:
            raise ValueError('send() called on closed port')

        if message.type == 'sysex':
            # Sysex messages are written as a string.
            string = pm.c_char_p(bytes(message.bin()))
            timestamp = 0  # Ignored when latency = 0
            _check_error(pm.lib.Pm_WriteSysEx(self._stream, timestamp, string))
        else:
            # The bytes of a message as packed into a 32 bit integer.
            packed_message = 0
            for byte in reversed(message.bytes()):
                packed_message <<= 8
                packed_message |= byte

            timestamp = 0  # Ignored when latency = 0
            _check_error(pm.lib.Pm_WriteShort(self._stream,
                                              timestamp,
                                              packed_message))
<|endoftext|>