<commit_before>##########################################################################
#
#  Copyright (c) 2016, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import gc
import sys
import time
import resource
import collections

import IECore

import Gaffer

class stats( Gaffer.Application ) :

	def __init__( self ) :

		Gaffer.Application.__init__(
			self,
			"""
			Outputs statistics about a Gaffer script, including the version of
			Gaffer that created it, the values of all setting and variables and
			the types of node in use. May also be used to perform performance
			analysis of image and scene processing nodes within the script, using
			a performance monitor to generate advanced statistics.

			To output basic information about a script :

			```
			gaffer stats fileName.gfr
			```

			To run a scene processing node using the performance monitor :

			```
			gaffer stats fileName.gfr -scene NameOfNode -performanceMonitor
			```

			To run an image processing node using the performance monitor :

			```
			gaffer stats fileName.gfr -image NameOfNode -performanceMonitor
			```
			"""
		)

		self.parameters().addParameters(

			[
				IECore.FileNameParameter(
					name = "script",
					description = "The script to examine.",
					defaultValue = "",
					allowEmptyString = False,
					extensions = "gfr",
					check = IECore.FileNameParameter.CheckType.MustExist,
				),

				IECore.FileNameParameter(
					name = "outputFile",
					description = "Output the results to this file on disk rather than stdout",
					defaultValue = "",
					allowEmptyString = True,
					extensions = "",
				),

				IECore.FloatParameter(
					name = "frame",
					description = "The frame to evaluate statistics at.",
					defaultValue = 1,
				),

				IECore.StringParameter(
					name = "scene",
					description = "The name of a SceneNode or ScenePlug to examine.",
					defaultValue = "",
				),

				IECore.StringParameter(
					name = "image",
					description = "The name of an ImageNode or ImagePlug to examine.",
					defaultValue = "",
				),

				IECore.BoolParameter(
					name = "performanceMonitor",
					description = "Turns on a performance monitor to provide additional"
						"statistics about the operation of the node graph.",
					defaultValue = False,
				),

				IECore.IntParameter(
					name = "maxLinesPerMetric",
					description = "The maximum number of plugs to list for each metric "
						"captured by the performance monitor.",
					defaultValue = 50,
				),

				IECore.BoolParameter(
					name = "contextMonitor",
					description = "Turns on a context monitor to provide additional "
						"statistics about the operation of the node graph.",
					defaultValue = False,
				),

				IECore.StringParameter(
					name = "contextMonitorRoot",
					description = "The name of a node or plug to provide a root for the "
						"context monitor. Statistics will only be captured for this root "
						"downwards.",
					defaultValue = "",
				)

			]

		)

		self.parameters().userData()["parser"] = IECore.CompoundObject(
			{
				"flagless" : IECore.StringVectorData( [ "script" ] )
			}
		)

	def _run( self, args ) :

		self.__timers = collections.OrderedDict()
		self.__memory = collections.OrderedDict()

		self.__memory["Application"] = _Memory.maxRSS()

		script = Gaffer.ScriptNode()
		script["fileName"].setValue( os.path.abspath( args["script"].value ) )

		with _Timer() as loadingTimer :
			script.load( continueOnError = True )
		self.__timers["Loading"] = loadingTimer

		self.__memory["Script"] = _Memory.maxRSS() - self.__memory["Application"]

		if args["performanceMonitor"].value :
			self.__performanceMonitor = Gaffer.PerformanceMonitor()
		else :
			self.__performanceMonitor = None

		if args["contextMonitor"].value :
			contextMonitorRoot = None
			if args["contextMonitorRoot"].value :
				contextMonitorRoot = script.descendant( args["contextMonitorRoot"].value )
				if contextMonitorRoot is None :
					IECore.msg( IECore.Msg.Level.Error, "stats", "Context monitor root \"%s\" does not exist" % args["contextMonitorRoot"].value )
					return 1
			self.__contextMonitor = Gaffer.ContextMonitor( contextMonitorRoot )
		else :
			self.__contextMonitor = None

		self.__output = file( args["outputFile"].value, "w" ) if args["outputFile"].value else sys.stdout

		with Gaffer.Context( script.context() ) as context :

			context.setFrame( args["frame"].value )

			self.__writeVersion( script )

			self.__output.write( "\n" )

			self.__writeArgs( args )

			self.__output.write( "\n" )

			self.__writeSettings( script )

			self.__output.write( "\n" )

			self.__writeVariables( script )

			self.__output.write( "\n" )

			self.__writeNodes( script )

			if args["scene"].value :

				self.__writeScene( script, args )

			if args["image"].value :

				self.__writeImage( script, args )

		self.__output.write( "\n" )

		self.__writeMemory()

		self.__output.write( "\n" )

		self.__writePerformance( script, args )

		self.__output.write( "\n" )

		self.__writeContext( script, args )

		self.__output.write( "\n" )

		self.__output.close()

		return 0

	def __writeVersion( self, script ) :

		numbers = [ Gaffer.Metadata.nodeValue( script, "serialiser:" + x + "Version" ) for x in ( "milestone", "major", "minor", "patch" ) ]
		if None not in numbers :
			version = ".".join( str( x ) for x in numbers )
		else :
			version = "unknown"

		versions = (
			( "Script", version ),
			( "Current", Gaffer.About.versionString() ),
		)

		self.__output.write( "Gaffer Version :\n\n" )
		self.__writeItems( versions )

	def __writeItems( self, items ) :

		if not len( items ) :
			return

		width = max( [ len( x[0] ) for x in items ] ) + 4
		for name, value in items :
			self.__output.write( "  {name:<{width}}{value}\n".format( name = name, width = width, value = value ) )

	def __writeArgs( self, args ) :

		self.__output.write( "Args :\n\n" )
		self.__writeItems( args.items() )

	def __writeSettings( self, script ) :

		plugsToIgnore = {
			script["fileName"],
			script["unsavedChanges"],
			script["variables"],
		}

		items = []
		def itemsWalk( p ) :

			if p in plugsToIgnore :
				return

			if hasattr( p, "getValue" ) :
				items.append( ( p.relativeName( script ), p.getValue() ) )
			else :
				for c in p.children( Gaffer.Plug ) :
					itemsWalk( c )

		itemsWalk( script )

		self.__output.write( "Settings :\n\n" )
		self.__writeItems( items )

	def __writeVariables( self, script ) :

		items = []
		for p in script["variables"] :
			data, name = script["variables"].memberDataAndName( p )
			if data is not None :
				items.append( ( name, data ) )

		self.__output.write( "Variables :\n\n" )
		self.__writeItems( items )

	def __writeNodes( self, script ) :

		def countWalk( node, counter ) :

			if not isinstance( node, Gaffer.ScriptNode ) :
				counter[node.typeName()] += 1

			for c in node.children( Gaffer.Node ) :
				countWalk( c, counter )

		counter = collections.Counter()
		countWalk( script, counter )

		items = [ ( nodeType.rpartition( ":" )[2], count ) for nodeType, count in counter.most_common() ]
		items.extend( [
			( "", "" ),
			( "Total", sum( counter.values() ) ),
		] )

		self.__output.write( "Nodes :\n\n" )
		self.__writeItems( items )

	def __writeScene( self, script, args ) :

		import GafferScene
		import GafferSceneTest

		scene = script.descendant( args["scene"].value )
		if isinstance( scene, Gaffer.Node ) :
			scene = next( ( x for x in scene.children( GafferScene.ScenePlug ) ), None )

		if scene is None :
			IECore.msg( IECore.Msg.Level.Error, "stats", "Scene \"%s\" does not exist" % args["scene"].value )
			return

		memory = _Memory.maxRSS()
		with _Timer() as sceneTimer :
			with self.__performanceMonitor or _NullContextManager(), self.__contextMonitor or _NullContextManager() :
				GafferSceneTest.traverseScene( scene )
		self.__timers["Scene generation"] = sceneTimer
		self.__memory["Scene generation"] = _Memory.maxRSS() - memory

		## \todo Calculate and write scene stats
		#  - Locations
		#  - Unique objects, attributes etc

	def __writeImage( self, script, args ) :

		import GafferImage
		import GafferImageTest

		image = script.descendant( args["image"].value )
		if isinstance( image, Gaffer.Node ) :
			image = next( ( x for x in image.children( GafferImage.ImagePlug ) ), None )

		if image is None :
			IECore.msg( IECore.Msg.Level.Error, "stats", "Image \"%s\" does not exist" % args["image"].value )
			return

		memory = _Memory.maxRSS()
		with _Timer() as imageTimer :
			with self.__performanceMonitor or _NullContextManager(), self.__contextMonitor or _NullContextManager() :
				GafferImageTest.processTiles( image )
		self.__timers["Image generation"] = imageTimer
		self.__memory["Image generation"] = _Memory.maxRSS() - memory
		self.__memory["OIIO cache limit"] = _Memory( GafferImage.OpenImageIOReader.getCacheMemoryLimit() )
		self.__memory["OIIO cache usage"] = _Memory( GafferImage.OpenImageIOReader.cacheMemoryUsage() )

		items = [
			( "Format", image["format"].getValue() ),
			( "Data window", image["dataWindow"].getValue() ),
			( "Channel names", image["channelNames"].getValue() ),
		]

		self.__output.write( "\nImage :\n\n" )
		self.__writeItems( items )

	def __writeMemory( self ) :

		objectPool = IECore.ObjectPool.defaultObjectPool()

		items = self.__memory.items()

		items.extend( [
			( "", "" ),
			( "Cache limit", _Memory( Gaffer.ValuePlug.getCacheMemoryLimit() ) ),
			( "Cache usage", _Memory( Gaffer.ValuePlug.cacheMemoryUsage() ) ),
			( "", "" ),
			( "Object pool limit", _Memory( objectPool.getMaxMemoryUsage() ) ),
			( "Object pool usage", _Memory( objectPool.memoryUsage() ) ),
			( "", "" ),
			( "Max resident size", _Memory.maxRSS() ),
		] )

		self.__output.write( "Memory :\n\n" )
		self.__writeItems( items )

	def __writeStatisticsItems( self, script, stats, key, n ) :

		stats.sort( key = key, reverse = True )
		items = [ ( x[0].relativeName( script ), key( x ) ) for x in stats[:n] ]
		self.__writeItems( items )

	def __writePerformance( self, script, args ) :

			self.__output.write( "Performance :\n\n" )
			self.__writeItems( self.__timers.items() )

			if self.__performanceMonitor is not None :
				self.__output.write(
					"\n" + Gaffer.MonitorAlgo.formatStatistics(
						self.__performanceMonitor,
						maxLinesPerMetric = args["maxLinesPerMetric"].value
					)
				)

	def __writeContext( self, script, args ) :

			if self.__contextMonitor is None :
				return

			self.__output.write( "Contexts :\n\n" )

			stats = self.__contextMonitor.combinedStatistics()

			items = [ ( n, stats.numUniqueValues( n ) ) for n in stats.variableNames() ]
			items += [
				( "", "" ),
				( "Unique contexts", stats.numUniqueContexts() ),
			]

			self.__writeItems( items )

class _Timer( object ) :

	def __enter__( self ) :

		self.__time = time.time()
		self.__clock = time.clock()

		return self

	def __exit__( self, type, value, traceBack ) :

		self.__time = time.time() - self.__time
		self.__clock = time.clock() - self.__clock

	def __str__( self ) :

		return "%.3fs (wall), %.3fs (CPU)" % ( self.__time, self.__clock )

class _Memory( object ) :

	def __init__( self, bytes ) :

		self.__bytes = bytes

	@classmethod
	def maxRSS( cls ) :

		if sys.platform == "darwin" :
			return cls( resource.getrusage( resource.RUSAGE_SELF ).ru_maxrss )
		else :
			return cls( resource.getrusage( resource.RUSAGE_SELF ).ru_maxrss * 1024 )

	def __str__( self ) :

		return "%.3fM" % ( self.__bytes / ( 1024 * 1024. ) )

	def __sub__( self, other ) :

		return _Memory( self.__bytes - other.__bytes )

class _NullContextManager( object ) :

	def __enter__( self ) :

		pass

	def __exit__( self, type, value, traceBack ) :

		pass

IECore.registerRunTimeTyped( stats )
<commit_msg>Stats App : Add parameter to optionally supress the node summary.<commit_after>##########################################################################
#
#  Copyright (c) 2016, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import gc
import sys
import time
import resource
import collections

import IECore

import Gaffer

class stats( Gaffer.Application ) :

	def __init__( self ) :

		Gaffer.Application.__init__(
			self,
			"""
			Outputs statistics about a Gaffer script, including the version of
			Gaffer that created it, the values of all setting and variables and
			the types of node in use. May also be used to perform performance
			analysis of image and scene processing nodes within the script, using
			a performance monitor to generate advanced statistics.

			To output basic information about a script :

			```
			gaffer stats fileName.gfr
			```

			To run a scene processing node using the performance monitor :

			```
			gaffer stats fileName.gfr -scene NameOfNode -performanceMonitor
			```

			To run an image processing node using the performance monitor :

			```
			gaffer stats fileName.gfr -image NameOfNode -performanceMonitor
			```
			"""
		)

		self.parameters().addParameters(

			[
				IECore.FileNameParameter(
					name = "script",
					description = "The script to examine.",
					defaultValue = "",
					allowEmptyString = False,
					extensions = "gfr",
					check = IECore.FileNameParameter.CheckType.MustExist,
				),

				IECore.FileNameParameter(
					name = "outputFile",
					description = "Output the results to this file on disk rather than stdout",
					defaultValue = "",
					allowEmptyString = True,
					extensions = "",
				),

				IECore.FloatParameter(
					name = "frame",
					description = "The frame to evaluate statistics at.",
					defaultValue = 1,
				),

				IECore.BoolParameter(
					name = "nodeSummary",
					description = "Turns on a summary of nodes in the script.",
					defaultValue = True,
				),

				IECore.StringParameter(
					name = "scene",
					description = "The name of a SceneNode or ScenePlug to examine.",
					defaultValue = "",
				),

				IECore.StringParameter(
					name = "image",
					description = "The name of an ImageNode or ImagePlug to examine.",
					defaultValue = "",
				),

				IECore.BoolParameter(
					name = "performanceMonitor",
					description = "Turns on a performance monitor to provide additional"
						"statistics about the operation of the node graph.",
					defaultValue = False,
				),

				IECore.IntParameter(
					name = "maxLinesPerMetric",
					description = "The maximum number of plugs to list for each metric "
						"captured by the performance monitor.",
					defaultValue = 50,
				),

				IECore.BoolParameter(
					name = "contextMonitor",
					description = "Turns on a context monitor to provide additional "
						"statistics about the operation of the node graph.",
					defaultValue = False,
				),

				IECore.StringParameter(
					name = "contextMonitorRoot",
					description = "The name of a node or plug to provide a root for the "
						"context monitor. Statistics will only be captured for this root "
						"downwards.",
					defaultValue = "",
				)

			]

		)

		self.parameters().userData()["parser"] = IECore.CompoundObject(
			{
				"flagless" : IECore.StringVectorData( [ "script" ] )
			}
		)

	def _run( self, args ) :

		self.__timers = collections.OrderedDict()
		self.__memory = collections.OrderedDict()

		self.__memory["Application"] = _Memory.maxRSS()

		script = Gaffer.ScriptNode()
		script["fileName"].setValue( os.path.abspath( args["script"].value ) )

		with _Timer() as loadingTimer :
			script.load( continueOnError = True )
		self.__timers["Loading"] = loadingTimer

		self.__memory["Script"] = _Memory.maxRSS() - self.__memory["Application"]

		if args["performanceMonitor"].value :
			self.__performanceMonitor = Gaffer.PerformanceMonitor()
		else :
			self.__performanceMonitor = None

		if args["contextMonitor"].value :
			contextMonitorRoot = None
			if args["contextMonitorRoot"].value :
				contextMonitorRoot = script.descendant( args["contextMonitorRoot"].value )
				if contextMonitorRoot is None :
					IECore.msg( IECore.Msg.Level.Error, "stats", "Context monitor root \"%s\" does not exist" % args["contextMonitorRoot"].value )
					return 1
			self.__contextMonitor = Gaffer.ContextMonitor( contextMonitorRoot )
		else :
			self.__contextMonitor = None

		self.__output = file( args["outputFile"].value, "w" ) if args["outputFile"].value else sys.stdout

		with Gaffer.Context( script.context() ) as context :

			context.setFrame( args["frame"].value )

			self.__writeVersion( script )

			self.__output.write( "\n" )

			self.__writeArgs( args )

			self.__output.write( "\n" )

			self.__writeSettings( script )

			self.__output.write( "\n" )

			self.__writeVariables( script )

			self.__output.write( "\n" )

			if args["nodeSummary"].value :

				self.__writeNodes( script )

			if args["scene"].value :

				self.__writeScene( script, args )

			if args["image"].value :

				self.__writeImage( script, args )

		self.__output.write( "\n" )

		self.__writeMemory()

		self.__output.write( "\n" )

		self.__writePerformance( script, args )

		self.__output.write( "\n" )

		self.__writeContext( script, args )

		self.__output.write( "\n" )

		self.__output.close()

		return 0

	def __writeVersion( self, script ) :

		numbers = [ Gaffer.Metadata.nodeValue( script, "serialiser:" + x + "Version" ) for x in ( "milestone", "major", "minor", "patch" ) ]
		if None not in numbers :
			version = ".".join( str( x ) for x in numbers )
		else :
			version = "unknown"

		versions = (
			( "Script", version ),
			( "Current", Gaffer.About.versionString() ),
		)

		self.__output.write( "Gaffer Version :\n\n" )
		self.__writeItems( versions )

	def __writeItems( self, items ) :

		if not len( items ) :
			return

		width = max( [ len( x[0] ) for x in items ] ) + 4
		for name, value in items :
			self.__output.write( "  {name:<{width}}{value}\n".format( name = name, width = width, value = value ) )

	def __writeArgs( self, args ) :

		self.__output.write( "Args :\n\n" )
		self.__writeItems( args.items() )

	def __writeSettings( self, script ) :

		plugsToIgnore = {
			script["fileName"],
			script["unsavedChanges"],
			script["variables"],
		}

		items = []
		def itemsWalk( p ) :

			if p in plugsToIgnore :
				return

			if hasattr( p, "getValue" ) :
				items.append( ( p.relativeName( script ), p.getValue() ) )
			else :
				for c in p.children( Gaffer.Plug ) :
					itemsWalk( c )

		itemsWalk( script )

		self.__output.write( "Settings :\n\n" )
		self.__writeItems( items )

	def __writeVariables( self, script ) :

		items = []
		for p in script["variables"] :
			data, name = script["variables"].memberDataAndName( p )
			if data is not None :
				items.append( ( name, data ) )

		self.__output.write( "Variables :\n\n" )
		self.__writeItems( items )

	def __writeNodes( self, script ) :

		def countWalk( node, counter ) :

			if not isinstance( node, Gaffer.ScriptNode ) :
				counter[node.typeName()] += 1

			for c in node.children( Gaffer.Node ) :
				countWalk( c, counter )

		counter = collections.Counter()
		countWalk( script, counter )

		items = [ ( nodeType.rpartition( ":" )[2], count ) for nodeType, count in counter.most_common() ]
		items.extend( [
			( "", "" ),
			( "Total", sum( counter.values() ) ),
		] )

		self.__output.write( "Nodes :\n\n" )
		self.__writeItems( items )

	def __writeScene( self, script, args ) :

		import GafferScene
		import GafferSceneTest

		scene = script.descendant( args["scene"].value )
		if isinstance( scene, Gaffer.Node ) :
			scene = next( ( x for x in scene.children( GafferScene.ScenePlug ) ), None )

		if scene is None :
			IECore.msg( IECore.Msg.Level.Error, "stats", "Scene \"%s\" does not exist" % args["scene"].value )
			return

		memory = _Memory.maxRSS()
		with _Timer() as sceneTimer :
			with self.__performanceMonitor or _NullContextManager(), self.__contextMonitor or _NullContextManager() :
				GafferSceneTest.traverseScene( scene )
		self.__timers["Scene generation"] = sceneTimer
		self.__memory["Scene generation"] = _Memory.maxRSS() - memory

		## \todo Calculate and write scene stats
		#  - Locations
		#  - Unique objects, attributes etc

	def __writeImage( self, script, args ) :

		import GafferImage
		import GafferImageTest

		image = script.descendant( args["image"].value )
		if isinstance( image, Gaffer.Node ) :
			image = next( ( x for x in image.children( GafferImage.ImagePlug ) ), None )

		if image is None :
			IECore.msg( IECore.Msg.Level.Error, "stats", "Image \"%s\" does not exist" % args["image"].value )
			return

		memory = _Memory.maxRSS()
		with _Timer() as imageTimer :
			with self.__performanceMonitor or _NullContextManager(), self.__contextMonitor or _NullContextManager() :
				GafferImageTest.processTiles( image )
		self.__timers["Image generation"] = imageTimer
		self.__memory["Image generation"] = _Memory.maxRSS() - memory
		self.__memory["OIIO cache limit"] = _Memory( GafferImage.OpenImageIOReader.getCacheMemoryLimit() )
		self.__memory["OIIO cache usage"] = _Memory( GafferImage.OpenImageIOReader.cacheMemoryUsage() )

		items = [
			( "Format", image["format"].getValue() ),
			( "Data window", image["dataWindow"].getValue() ),
			( "Channel names", image["channelNames"].getValue() ),
		]

		self.__output.write( "\nImage :\n\n" )
		self.__writeItems( items )

	def __writeMemory( self ) :

		objectPool = IECore.ObjectPool.defaultObjectPool()

		items = self.__memory.items()

		items.extend( [
			( "", "" ),
			( "Cache limit", _Memory( Gaffer.ValuePlug.getCacheMemoryLimit() ) ),
			( "Cache usage", _Memory( Gaffer.ValuePlug.cacheMemoryUsage() ) ),
			( "", "" ),
			( "Object pool limit", _Memory( objectPool.getMaxMemoryUsage() ) ),
			( "Object pool usage", _Memory( objectPool.memoryUsage() ) ),
			( "", "" ),
			( "Max resident size", _Memory.maxRSS() ),
		] )

		self.__output.write( "Memory :\n\n" )
		self.__writeItems( items )

	def __writeStatisticsItems( self, script, stats, key, n ) :

		stats.sort( key = key, reverse = True )
		items = [ ( x[0].relativeName( script ), key( x ) ) for x in stats[:n] ]
		self.__writeItems( items )

	def __writePerformance( self, script, args ) :

			self.__output.write( "Performance :\n\n" )
			self.__writeItems( self.__timers.items() )

			if self.__performanceMonitor is not None :
				self.__output.write(
					"\n" + Gaffer.MonitorAlgo.formatStatistics(
						self.__performanceMonitor,
						maxLinesPerMetric = args["maxLinesPerMetric"].value
					)
				)

	def __writeContext( self, script, args ) :

			if self.__contextMonitor is None :
				return

			self.__output.write( "Contexts :\n\n" )

			stats = self.__contextMonitor.combinedStatistics()

			items = [ ( n, stats.numUniqueValues( n ) ) for n in stats.variableNames() ]
			items += [
				( "", "" ),
				( "Unique contexts", stats.numUniqueContexts() ),
			]

			self.__writeItems( items )

class _Timer( object ) :

	def __enter__( self ) :

		self.__time = time.time()
		self.__clock = time.clock()

		return self

	def __exit__( self, type, value, traceBack ) :

		self.__time = time.time() - self.__time
		self.__clock = time.clock() - self.__clock

	def __str__( self ) :

		return "%.3fs (wall), %.3fs (CPU)" % ( self.__time, self.__clock )

class _Memory( object ) :

	def __init__( self, bytes ) :

		self.__bytes = bytes

	@classmethod
	def maxRSS( cls ) :

		if sys.platform == "darwin" :
			return cls( resource.getrusage( resource.RUSAGE_SELF ).ru_maxrss )
		else :
			return cls( resource.getrusage( resource.RUSAGE_SELF ).ru_maxrss * 1024 )

	def __str__( self ) :

		return "%.3fM" % ( self.__bytes / ( 1024 * 1024. ) )

	def __sub__( self, other ) :

		return _Memory( self.__bytes - other.__bytes )

class _NullContextManager( object ) :

	def __enter__( self ) :

		pass

	def __exit__( self, type, value, traceBack ) :

		pass

IECore.registerRunTimeTyped( stats )
<|endoftext|>


<commit_before>from django.test import TestCase
from django.test import Client
from django.conf import settings
import unittest
import json
import csv

class StoriesApiTestCase(TestCase):

    stories_base_url = "/api/v1/stories/"

    def setUp(self):
        print "Hello world"
        self.client = Client()
        self.schools_lib_id = settings.TESTS_STORIES_INPUT['SCHOOLS_TEST_ID1']

    def testGetStoryForSchoolId(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id
        print "Testing get story for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        print results
        if results['count'] > 0:
            print "Stories exist for this school"
            sample_school = results['features'][0]
            self.assertTrue('id' in sample_school,
                            "has no a property called id")
            self.assertTrue('name' in sample_school,
                            "has no a property called name")
            self.assertTrue('date' in sample_school, "has no property called date")
            self.assertTrue('date_of_visit' in sample_school, "has no property called date_of_visit")
            self.assertTrue('school' in sample_school, "has no school id")
            self.assertTrue('school_name' in sample_school, "has no school name")
            self.assertTrue('school_url' in sample_school, "has no school URL")
            self.assertTrue('is_verified' in sample_school, "has no verified field")
        else:
            print "No stories exist for this school"

    # This test exercises getting only verified stories for a schoo. Should NOT return is_verified=false stories.
    def testGetVerifiedStories(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=yes"
        print "Testing verified stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        #print results
        for x in range(0,results['count']):
            sample_school = results['features'][x]
            print sample_school['is_verified']
            self.assertEqual(sample_school['is_verified'],True)

    # This test exercises getting only unverified stories for a school.
    # Results should not contain verified stories.
    def testGetUnverifiedStories(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=no"
        print "Testing unverified stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        print "Number of unverified stories:" + str(results['count'])
        for x in range(0,results['count']):
            sample_school = results['features'][x]
            print sample_school['is_verified']
            self.assertEqual(sample_school['is_verified'],False)

    def testValidateUrlParamVerified(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=somethingbluw"
        print "Testing invalid param for stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertNotEqual(response.status_code, 200,
                         "Shoud return a 400 application error in case of invalid param")

    def tearDown(self):
        print "teardown"

<commit_msg>Added more tests to stories<commit_after>from django.test import TestCase
from django.test import Client
from django.conf import settings
import unittest
import json
import csv

class StoriesApiTestCase(TestCase):

    stories_base_url = "/api/v1/stories/"

    def setUp(self):
        self.client = Client()
        self.schools_lib_id = settings.TESTS_STORIES_INPUT['SCHOOLS_TEST_ID1']

    def testGetStoryForSchoolId(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id
        print "Testing get story for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        print results
        if results['count'] > 0:
            print "Stories exist for this school"
            sample_school = results['features'][0]
            self.assertTrue('id' in sample_school,
                            "has no a property called id")
            self.assertTrue('name' in sample_school,
                            "has no a property called name")
            self.assertTrue('date' in sample_school, "has no property called date")
            self.assertTrue('date_of_visit' in sample_school, "has no property called date_of_visit")
            self.assertTrue('school' in sample_school, "has no school id")
            self.assertTrue('school_name' in sample_school, "has no school name")
            self.assertTrue('school_url' in sample_school, "has no school URL")
            self.assertTrue('is_verified' in sample_school, "has no verified field")
        else:
            print "No stories exist for this school"

    # This test exercises getting only verified stories for a schoo. Should NOT return is_verified=false stories.
    def testGetVerifiedStories(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=yes"
        print "Testing verified stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        #print results
        for x in range(0,results['count']):
            sample_school = results['features'][x]
            print sample_school['is_verified']
            self.assertEqual(sample_school['is_verified'],True)

    # This test exercises getting only unverified stories for a school.
    # Results should not contain verified stories.
    def testGetUnverifiedStories(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=no"
        print "Testing unverified stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools stories status code is 200")
        results = json.loads(response.content)
        print "Number of unverified stories:" + str(results['count'])
        for x in range(0,results['count']):
            sample_school = results['features'][x]
            print sample_school['is_verified']
            self.assertEqual(sample_school['is_verified'],False)

    def testValidateUrlParamVerified(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&verified=somethingbluw"
        print "Testing invalid param for stories for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertNotEqual(response.status_code, 200,
                         "Shoud return a 400 application error in case of invalid param")

    def testGetAnswersForSchool(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&answers=yes"
        print "Testing getting answers for school ID -- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools answers status code is 200")
        results = json.loads(response.content)
        sample_school = results['features'][0]
        self.assertTrue('answers' in sample_school, "has no property called answers")

    # Would be better if error is returned in case invalid params are entered for answers.
    def testGetAnswersForSchool2(self):
        query_url = self.stories_base_url + "?school_id=" + self.schools_lib_id + "&answers=something"
        print "Checking parameter validation for answers param-- " + query_url
        response = self.client.get(query_url)
        print response.status_code
        #print response
        self.assertEqual(response.status_code, 200,
                         "schools answers status code is 200")
        results = json.loads(response.content)
        sample_school = results['features'][0]
        self.assertFalse('answers' in sample_school, "has no property called answers")

    def testStoriesInformation(self):
        query_url=self.stories_base_url + "info/"
        print "Testing stories meta info..."
        response=self.client.get(query_url)
        print response.status_code
        self.assertEqual(response.status_code, 200, "stories meta info is 200")
        results=json.loads(response.content)
        self.assertTrue('total_verified_stories' in results, "Has no property called total_verified_stories")
        self.assertTrue('total_stories' in results, "has no property called total_stories")
        self.assertTrue('total_images' in results, "has no property called total_images")

    def testStoriesMetaDataFail1(self):

        query_url=self.stories_base_url + "meta/?source=web"
        print "Testing stories meta data..." + query_url
        response=self.client.get(query_url)
        print response.status_code
        self.assertEqual(response.status_code, 500, "Need school type")

    def testStoriesMetaDataPrimary(self):
        schoolType="Primary School"
        query_url=self.stories_base_url + "meta/?source=web&school_type="+schoolType
        print "Testing stories meta data..." + query_url
        response=self.client.get(query_url)
        print response.status_code
        self.assertEqual(response.status_code, 200, "Need school type")
        results=json.loads(response.content)
        self.assertTrue('school_type' in results, "Has no property called school_type")
        self.assertTrue('total_responses' in results, "Has no property called total_responses")
        self.assertTrue('responses_yesterday' in results, "has no property called responses_yesterday")
        self.assertTrue('per_month_responses' in results, "has no property called per_month_responses")
        self.assertTrue('questions' in results, "has no property questions")
        self.assertEqual(results['school_type'],schoolType)

    def testStoriesMetaDataPreschool(self):
        schoolType="Preschool"
        query_url=self.stories_base_url + "meta/?source=web&school_type="+schoolType
        print "Testing stories meta data..." + query_url
        response=self.client.get(query_url)
        print response.status_code
        self.assertEqual(response.status_code, 200, "Need school type")
        results=json.loads(response.content)
        self.assertTrue('school_type' in results, "Has no property called school_type")
        self.assertTrue('total_responses' in results, "Has no property called total_responses")
        self.assertTrue('responses_yesterday' in results, "has no property called responses_yesterday")
        self.assertTrue('per_month_responses' in results, "has no property called per_month_responses")
        self.assertTrue('questions' in results, "has no property questions")
        self.assertEqual(results['school_type'],schoolType)

    def tearDown(self):
        pass

<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk import get_resource_service
import os
import glob
import json
import superdesk
from bson.objectid import ObjectId
import eve.io.base
from superdesk.errors import SuperdeskApiError

class ThemesResource(Resource):

    schema = {
        'name': {
            'type': 'string',
            'unique': True
        },
        'label': {
            'type': 'string'
        },
        'extends': {
            'type': 'string'
        },
        'version': {
            'type': 'string'
        },
        'styles': {
            'type': 'list',
            'schema': {
                'type': 'string'
            }
        },
        'scripts': {
            'type': 'list',
            'schema': {
                'type': 'string'
            }
        },
        'options': {
            'type': 'list',
            'schema': {
                'type': 'dict'
            }
        }
    }
    datasource = {
        'source': 'themes',
        'default_sort': [('_updated', -1)]
    }
    ITEM_METHODS = ['GET', 'POST', 'DELETE']
    privileges = {'GET': 'global_preferences', 'POST': 'global_preferences',
                  'PATCH': 'global_preferences', 'DELETE': 'global_preferences'}

class ThemesService(BaseService):

    def get_local_themes_packages(self):
            embed_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                        '../', 'embed', 'embed_assets', 'themes')
            for file in glob.glob(embed_folder + '/**/theme.json'):
                yield json.loads(open(file).read())

    def update_registered_theme_with_local_files(self):
        created = []
        updated = []
        for theme in self.get_local_themes_packages():
            previous_theme = self.find_one(req=None, name=theme.get('name'))
            if previous_theme:
                self.replace(previous_theme['_id'], theme, previous_theme)
                updated.append(theme)
            else:
                self.create([theme])
                created.append(theme)
        return (created, updated)

    def on_delete(self, deleted_theme):
        global_prefs = get_resource_service('global_preferences').get_global_prefs()
        if deleted_theme['name'] == global_prefs['theme']:
            get_resource_service('themes').find_one(req=None, name=deleted_theme['name'])
            raise SuperdeskApiError.forbiddenError("This is a default theme and can not be deleted")
        blogs_service = get_resource_service('blogs')
        blogs = blogs_service.get(req=None, lookup={'theme._id': deleted_theme['_id']})
        for blog in blogs:
            # will assign the default theme to this blog
            global_prefs = get_resource_service('global_preferences').get_global_prefs()
            theme = blogs_service.get_theme_snapshot(global_prefs['theme'])

            try:
                blogs_service.system_update(ObjectId(blog['_id']), {'theme': theme}, blog)
            except eve.io.base.DataLayer.OriginalChangedError:
                print(u'! an error occured during saving blog "%s".' % (blog['title']),
                      'Can be a broken relationship (with user for instance)')
            else:
                print('- Blog "%s"\'s theme was updated to %s %s' % (
                    blog['title'], theme['name'], theme['version']))

class ThemesCommand(superdesk.Command):

    def run(self):
        theme_service = get_resource_service('themes')
        created, updated = theme_service.update_registered_theme_with_local_files()
        print('%d themes registered' % (len(created) + len(updated)))
        if created:
            print('added:')
            for theme in created:
                print('\t+ %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))
        if updated:
            print('updated:')
            for theme in updated:
                print('\t* %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))

superdesk.command('register_local_themes', ThemesCommand())
<commit_msg>Clean and comment theme deletion<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk import get_resource_service
import os
import glob
import json
import superdesk
from bson.objectid import ObjectId
from superdesk.errors import SuperdeskApiError

class ThemesResource(Resource):

    schema = {
        'name': {
            'type': 'string',
            'unique': True
        },
        'label': {
            'type': 'string'
        },
        'extends': {
            'type': 'string'
        },
        'version': {
            'type': 'string'
        },
        'styles': {
            'type': 'list',
            'schema': {
                'type': 'string'
            }
        },
        'scripts': {
            'type': 'list',
            'schema': {
                'type': 'string'
            }
        },
        'options': {
            'type': 'list',
            'schema': {
                'type': 'dict'
            }
        }
    }
    datasource = {
        'source': 'themes',
        'default_sort': [('_updated', -1)]
    }
    ITEM_METHODS = ['GET', 'POST', 'DELETE']
    privileges = {'GET': 'global_preferences', 'POST': 'global_preferences',
                  'PATCH': 'global_preferences', 'DELETE': 'global_preferences'}

class ThemesService(BaseService):

    def get_local_themes_packages(self):
            embed_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                        '../', 'embed', 'embed_assets', 'themes')
            for file in glob.glob(embed_folder + '/**/theme.json'):
                yield json.loads(open(file).read())

    def update_registered_theme_with_local_files(self):
        created = []
        updated = []
        for theme in self.get_local_themes_packages():
            previous_theme = self.find_one(req=None, name=theme.get('name'))
            if previous_theme:
                self.replace(previous_theme['_id'], theme, previous_theme)
                updated.append(theme)
            else:
                self.create([theme])
                created.append(theme)
        return (created, updated)

    def on_delete(self, deleted_theme):
        global_default_theme = get_resource_service('global_preferences').get_global_prefs()['theme']
        # raise an exception if the removed theme is the default one
        if deleted_theme['name'] == global_default_theme:
            raise SuperdeskApiError.forbiddenError("This is a default theme and can not be deleted")
        # update all the blogs using the removed theme and assign the default theme
        blogs_service = get_resource_service('blogs')
        blogs = blogs_service.get(req=None, lookup={'theme._id': deleted_theme['_id']})
        for blog in blogs:
            # will assign the default theme to this blog
            default_theme = blogs_service.get_theme_snapshot(global_default_theme)
            blogs_service.system_update(ObjectId(blog['_id']), {'theme': default_theme}, blog)

class ThemesCommand(superdesk.Command):

    def run(self):
        theme_service = get_resource_service('themes')
        created, updated = theme_service.update_registered_theme_with_local_files()
        print('%d themes registered' % (len(created) + len(updated)))
        if created:
            print('added:')
            for theme in created:
                print('\t+ %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))
        if updated:
            print('updated:')
            for theme in updated:
                print('\t* %s %s (%s)' % (theme.get('label', theme['name']), theme['version'], theme['name']))

superdesk.command('register_local_themes', ThemesCommand())
<|endoftext|>


<commit_before>'''
Created on Oct 17, 2012

@author: dstrauss

implementation of contrast source optimization with soft update - i.e. requires one factorization.

'''

import numpy as np
import scipy.sparse as sparse
import scipy.sparse.linalg as lin
from mpi4py import MPI
import sparseTools as spt
import scipy.io as spio
from optimize import optimizer

class problem(optimizer):
    ''' an instance of an optimizer that will solve using the soft contrast X algorithm
    In this algorithm, I'm going to discard the "background field" its just annoying.'''
    def initOpt(self, uHat, D):
        self.rho = D['rho']
        self.xi = D['xi']
        self.uHat = uHat
        self.upperBound = D['uBound']
        self.lmb = D['lmb']
        self.obj = np.zeros(D['maxIter'])

        # add some local vars for ease
        self.s = self.fwd.getS() #  1j*self.muo*self.w
        self.A = self.fwd.nabla2+self.fwd.getk(0)

        # oh. this is going to get strange.
        # integrate TE concepts first.
        # contrast variable
#        self.X = np.zeros(self.fwd.nRx*self.fwd.nRy,dtype='complex128')
        # dual variable
#        self.Z = np.zeros(self.fwd.nRx*self.fwd.nRy,dtype='complex128')
        # scattered fields
        self.us = np.zeros(self.fwd.N,dtype='complex128')
        # just to make life easier:
        self.ub = self.fwd.sol[0] # shouldn't need --> .flatten()

        # create some new operators for doing what is necessary for the
        # contrast X work
        self.X = np.zeros(self.fwd.getXSize(),dtype='complex128')
        self.Z = np.zeros(self.fwd.getXSize(),dtype='complex128')

        # this operator creates a matrix that maps to the X variables which are field specific
        self.fwd.setCTRX()
        # create an operator that makes a projector used for updating u,x jointly
        self.prepProjector()

    def runOpt(self,P):
        ''' to run at each layer at each iteration '''

        # update dual variable
        self.Z = self.Z + (self.X - (self.s*self.fwd.x2u.T*self.us)*(self.fwd.p2x*P))

        uHatLocal =  self.uHat  #remove background field

        self.us,self.X = self.contrastProjector(P)

        obj = np.linalg.norm(uHatLocal-self.fwd.Ms*self.us)
        return obj

    def writeOut(self, rank, ix=0):
        import os
        assert os.path.exists(self.outDir + 'Data')

        us = self.fwd.parseFields(self.us)
        ub = self.fwd.parseFields(self.fwd.sol[0])
        sgmm = self.fwd.parseFields(self.fwd.sigmap[0])
        uTrue = self.fwd.parseFields(self.fwd.sol[1])

        D = {'f':self.fwd.f, 'angle':self.fwd.incAng, 'sigMat':sgmm[0], 'ub':ub[0], \
             'us':us[0], 'uTrue':uTrue[0], \
             'X':self.X, 'obj':self.obj, 'flavor':self.fwd.flavor}

        spio.savemat(self.outDir + 'Data/contrastX' + repr(rank) + '_' + repr(ix), D)

    def prepProjector(self):
        ''' create a projection function, stored inside, that does the u,x projection step'''
        n = self.fwd.N
        m = self.fwd.getXSize()

        uu = self.fwd.Ms.T*self.fwd.Ms+self.xi*sparse.eye(n,n)
        ux = sparse.coo_matrix((n,m))
        ul = self.A.T.conj()

        xu = sparse.coo_matrix((m,n))
        xx = self.xi*sparse.eye(m,m)
        xl = self.fwd.x2u.T.conj()

        lu = self.A
        lx = self.fwd.x2u
        ll = sparse.coo_matrix((n,n))

        M = spt.vCat([spt.hCat([uu,ux,ul]),\
                      spt.hCat([xu,xx,xl]),\
                      spt.hCat([lu,lx,ll])])

        self.projector = lin.factorized(M.tocsc())

    def contrastProjector(self,P):
        '''subroutine to actually do the projection '''
        n = self.fwd.N
        m = self.fwd.getXSize()

        Aa = sparse.eye(n+m,n+m);
        Bb = -sparse.eye(n+m,n+m);
        eAbs = 1e-5;
        eRel = 1e-5;

        u = np.zeros(n);
        ut = self.us;
        ud = np.zeros(n);

        x = np.zeros(m);
        xt = self.X;
        xd = np.zeros(m);

        z = np.zeros(n+m);

        TT = self.s*sparse.spdiags(self.fwd.p2x*P,0,m,m)*self.fwd.x2u.T

        uu = self.rho*TT.T.conj()*TT + self.xi*sparse.eye(n,n);
        ux = -self.rho*TT.T.conj()
        xu = -self.rho*TT
        xx = self.rho*sparse.eye(m,m) + self.xi*sparse.eye(m,m)

        R = spt.vCat([spt.hCat([uu, ux]), spt.hCat([xu, xx])])
        g = lin.factorized(R.tocsc())

        ePri = 0.0
        eDua = 0.0
        rErr = np.ones(n+m)
        sErr = np.ones(n+m)

        iter = 1
        while (iter<20): # & (np.linalg.norm(rErr)>ePri) & (np.linalg.norm(sErr)>eDua):
            ''' inner loop to solve the projection '''
            iter += 1
            rhs = np.concatenate((self.fwd.Ms.T*self.uHat + self.xi*(ut-ud),\
                                  self.xi*(xt-xd),\
                                  self.fwd.rhs))
            updt = self.projector(rhs)
            u = updt[:n]
            x = updt[n:(n+m)]

            rhs = np.concatenate((self.xi*(u+ud) + self.rho*TT.T.conj()*self.Z,\
                                  self.xi*(x+xd)-self.rho*self.Z))

            zold = z;
            z = g(rhs);
            ut = z[:n]
            xt = z[n:]

            ud = ud + u-ut;
            xd = xd + x-xt;

            sErr = -self.rho*(z-zold);
            rErr = np.concatenate((u,x)) - z;

            ePri = np.sqrt(2*n)*eAbs + eRel*max(np.linalg.norm(Aa*np.concatenate((ut,xt))),\
                                                np.linalg.norm(Bb*z))

            eDua = np.sqrt(2*n)*eAbs + eRel*np.linalg.norm(np.concatenate((ud,xd))*self.rho);

        return u,x

    def aggregatorSemiParallel(self,S, comm):
        ''' Do the aggregation step in parallel whoop! '''
        N = np.size(S)
        n = S[0].fwd.nRx*S[0].fwd.nRy

        uL = sparse.lil_matrix((n,n),dtype='complex128')
        bL = np.zeros(n,dtype='complex128')

#        U = np.zeros((n,N),dtype='complex128')
#        Q = np.zeros((n,N),dtype='complex128')
        nX = self.fwd.getXSize()
        for L in S:
            # for ix in range(N):
#            s = S[ix].s
            # print s
#            U[:,ix] = s*S[ix].fwd.Md*(S[ix].ub + S[ix].us)
#            Q[:,ix] = S[ix].X + S[ix].Z
            # print L.fwd.x2u.shape

            M = L.s*(sparse.spdiags(L.fwd.x2u.T*(L.ub+L.us),0,nX,nX))*self.fwd.p2x
            uL += M.T.conj()*M
            bL += M.T.conj()*(L.X + L.Z)

        U = sparse.lil_matrix((n,n),dtype='complex128')
        B = np.zeros(n,dtype='complex128')

        U = comm.allreduce(uL,U,op=MPI.SUM)
        B = comm.allreduce(bL,B,op=MPI.SUM)

        P = lin.spsolve(U,B)
        # print num
        # print den

        P = P.real

        # print P[1]
        P = np.maximum(P,0)
        # print self.upperBound
        P = np.minimum(P,self.upperBound)
        # print P[1]

        return P

    def plotSemiParallel(self,P,resid,rank,ix=0):
        ''' Plotting routine if things are semiParallel'''
        import matplotlib.pyplot as plt
        plt.close('all')
        import os

        plt.close('all')
        if not os.path.exists(self.outDir + 'Figs'):
            os.mkdir(self.outDir + 'Figs')

        # vv = S.Ms*S.v
        uu = self.fwd.Ms*self.us
        ub = self.fwd.Ms*self.ub
        skt = self.uHat

        plt.figure(100 + rank + 10*ix)
        plt.plot(np.arange(self.fwd.nSen), skt.real, np.arange(self.fwd.nSen), uu.real)
        plt.savefig(self.outDir + 'Figs/fig' + repr(100+rank+10*ix))

        if (rank==0) & (ix==0):
            # then print some figures
            plt.figure(383)
            plt.plot(resid)
            plt.savefig(self.outDir + 'Figs/fig383')

            plt.figure(387)
            plt.imshow(P.reshape(self.fwd.nRx,self.fwd.nRy), interpolation='nearest')
            plt.colorbar()
            plt.savefig(self.outDir + 'Figs/fig387')

            us = self.fwd.parseFields(self.us)

            plt.figure(76)
            plt.subplot(121)
            plt.imshow(us[0].real)
            plt.colorbar()

            plt.subplot(122)
            plt.imshow(us[0].imag)
            plt.colorbar()
            plt.savefig(self.outDir + 'Figs/fig76')

        # all show!
        # plt.show()
<commit_msg>checking projection.<commit_after>'''
Created on Oct 17, 2012

@author: dstrauss

implementation of contrast source optimization with soft update - i.e. requires one factorization.

'''

import numpy as np
import scipy.sparse as sparse
import scipy.sparse.linalg as lin
from mpi4py import MPI
import sparseTools as spt
import scipy.io as spio
from optimize import optimizer

class problem(optimizer):
    ''' an instance of an optimizer that will solve using the soft contrast X algorithm
    In this algorithm, I'm going to discard the "background field" its just annoying.'''
    def initOpt(self, uHat, D):
        self.rho = D['rho']
        self.xi = D['xi']
        self.uHat = uHat
        self.upperBound = D['uBound']
        self.lmb = D['lmb']
        self.obj = np.zeros(D['maxIter'])

        # add some local vars for ease
        self.s = self.fwd.getS() #  1j*self.muo*self.w
        self.A = self.fwd.nabla2+self.fwd.getk(0)

        # oh. this is going to get strange.
        # integrate TE concepts first.
        # contrast variable
#        self.X = np.zeros(self.fwd.nRx*self.fwd.nRy,dtype='complex128')
        # dual variable
#        self.Z = np.zeros(self.fwd.nRx*self.fwd.nRy,dtype='complex128')
        # scattered fields
        self.us = np.zeros(self.fwd.N,dtype='complex128')
        # just to make life easier:
        self.ub = self.fwd.sol[0] # shouldn't need --> .flatten()

        # create some new operators for doing what is necessary for the
        # contrast X work
        self.X = np.zeros(self.fwd.getXSize(),dtype='complex128')
        self.Z = np.zeros(self.fwd.getXSize(),dtype='complex128')

        # this operator creates a matrix that maps to the X variables which are field specific
        self.fwd.setCTRX()
        # create an operator that makes a projector used for updating u,x jointly
        self.prepProjector()

    def runOpt(self,P):
        ''' to run at each layer at each iteration '''

        # update dual variable
        self.Z = self.Z + (self.X - (self.s*self.fwd.x2u.T*self.us)*(self.fwd.p2x*P))

        uHatLocal =  self.uHat  #remove background field

        # uHatLocal =  self.uHat - self.fwd.Ms*self.ub  #remove background field

        nX = self.fwd.getXSize()
        pm = sparse.spdiags(self.s*self.fwd.p2x*P, 0, nX, nX)
        # print pm.shape
        # print self.fwd.x2u.shape
        ds = pm*self.fwd.x2u.T #  The sampling and material scaling.

        # Construct the KKT Matrix
        bmuu = self.fwd.Ms.T*self.fwd.Ms + self.rho*(ds.T.conj()*ds)
        bmux = -self.rho*ds.T.conj()
        bmul = self.A.T.conj()

        rhsu = self.fwd.Ms.T.conj()*uHatLocal  + self.rho*ds.T.conj()*self.Z
        # - self.rho*(ds.T.conj()*ds)*self.ub

        bmxu = -self.rho*ds
        bmxx = self.rho*sparse.eye(nX, nX)
        bmxl = self.fwd.x2u.T
        rhsx = - self.rho*self.Z
        # self.rho*ds*self.ub

        bmlu = self.A
        bmlx = self.fwd.x2u

        bmll = sparse.coo_matrix((self.fwd.N, self.fwd.N))
        rhsl = np.zeros(self.fwd.N)

        bm = spt.vCat([spt.hCat([bmuu, bmux, bmul]), \
                       spt.hCat([bmxu, bmxx, bmxl]), \
                       spt.hCat([bmlu, bmlx, bmll])])

        rhsbm = np.concatenate((rhsu, rhsx, rhsl))

        updt = lin.spsolve(bm.tocsr(), rhsbm)

        # N = self.nx*self.ny
        self.us = updt[:self.fwd.N]
        self.X = updt[self.fwd.N:(self.fwd.N+nX)]

        usCP,xCP = self.contrastProjector(P)

        print 'udiff ' + repr(np.linalg.norm(usCP-self.us))
        print 'xdiff ' + repr(np.linalg.norm(xCP-self.X))

        obj = np.linalg.norm(uHatLocal-self.fwd.Ms*self.us)
        return obj

    def writeOut(self, rank, ix=0):
        import os
        assert os.path.exists(self.outDir + 'Data')

        us = self.fwd.parseFields(self.us)
        ub = self.fwd.parseFields(self.fwd.sol[0])
        sgmm = self.fwd.parseFields(self.fwd.sigmap[0])
        uTrue = self.fwd.parseFields(self.fwd.sol[1])
        uSmp = self.fwd.Ms*self.fwd.parseFields(self.fwd.sol[1])
        uSolSmp = self.fwd.Ms*self.fwd.parseFields(self.us)

        D = {'f':self.fwd.f, 'angle':self.fwd.incAng, 'sigMat':sgmm[0], 'ub':ub[0], \
             'us':us[0], 'uTrue':uTrue[0], \
             'X':self.X, 'obj':self.obj, 'flavor':self.fwd.flavor, 'uSmp':uSmp, 'uSolSmp':uSolSmp}

        spio.savemat(self.outDir + 'Data/contrastSoftX' + repr(rank) + '_' + repr(ix), D)

    def prepProjector(self):
        ''' create a projection function, stored inside, that does the u,x projection step'''
        n = self.fwd.N
        m = self.fwd.getXSize()

        uu = self.fwd.Ms.T*self.fwd.Ms+self.xi*sparse.eye(n,n)
        ux = sparse.coo_matrix((n,m))
        ul = self.A.T.conj()

        xu = sparse.coo_matrix((m,n))
        xx = self.xi*sparse.eye(m,m)
        xl = self.fwd.x2u.T.conj()

        lu = self.A
        lx = self.fwd.x2u
        ll = sparse.coo_matrix((n,n))

        M = spt.vCat([spt.hCat([uu,ux,ul]),\
                      spt.hCat([xu,xx,xl]),\
                      spt.hCat([lu,lx,ll])])

        self.projector = lin.factorized(M.tocsc())

    def contrastProjector(self,P):
        '''subroutine to actually do the projection '''
        n = self.fwd.N
        m = self.fwd.getXSize()

        # Aa = sparse.eye(n+m,n+m);
        # Bb = -sparse.eye(n+m,n+m);
        eAbs = 1e-5;
        eRel = 1e-5;

        u = np.zeros(n);
        ut = self.us;
        ud = np.zeros(n);

        x = np.zeros(m);
        xt = self.X;
        xd = np.zeros(m);

        z = np.zeros(n+m);

        TT = self.s*sparse.spdiags(self.fwd.p2x*P,0,m,m)*self.fwd.x2u.T

        uu = self.rho*TT.T.conj()*TT + self.xi*sparse.eye(n,n);
        ux = -self.rho*TT.T.conj()
        xu = -self.rho*TT
        xx = self.rho*sparse.eye(m,m) + self.xi*sparse.eye(m,m)

        R = spt.vCat([spt.hCat([uu, ux]), spt.hCat([xu, xx])])
        g = lin.factorized(R.tocsc())

        ePri = 0.0
        eDua = 0.0
        rErr = np.ones(n+m)
        sErr = np.ones(n+m)

        iter = 1
        while (iter<20): # & (np.linalg.norm(rErr)>ePri) & (np.linalg.norm(sErr)>eDua):
            ''' inner loop to solve the projection '''
            iter += 1
            rhs = np.concatenate((self.fwd.Ms.T*self.uHat + self.xi*(ut-ud),\
                                  self.xi*(xt-xd),\
                                  self.fwd.rhs))
            updt = self.projector(rhs)
            u = updt[:n]
            x = updt[n:(n+m)]

            rhs = np.concatenate((self.xi*(u+ud) + self.rho*TT.T.conj()*self.Z,\
                                  self.xi*(x+xd)-self.rho*self.Z))

            zold = z;
            z = g(rhs);
            ut = z[:n]
            xt = z[n:]

            ud = ud + u-ut;
            xd = xd + x-xt;

            sErr = -self.rho*(z-zold);
            rErr = np.concatenate((u,x)) - z;

            ePri = np.sqrt(2*n)*eAbs + eRel*max(np.linalg.norm(np.concatenate((ut,xt))),\
                                                np.linalg.norm(-1.0*z))

            eDua = np.sqrt(2*n)*eAbs + eRel*np.linalg.norm(np.concatenate((ud,xd))*self.rho);

        return u,x

    def aggregatorSemiParallel(self,S, comm):
        ''' Do the aggregation step in parallel whoop! '''
        N = np.size(S)
        n = S[0].fwd.nRx*S[0].fwd.nRy

        uL = sparse.lil_matrix((n,n),dtype='complex128')
        bL = np.zeros(n,dtype='complex128')

#        U = np.zeros((n,N),dtype='complex128')
#        Q = np.zeros((n,N),dtype='complex128')
        nX = self.fwd.getXSize()
        for L in S:
            # for ix in range(N):
#            s = S[ix].s
            # print s
#            U[:,ix] = s*S[ix].fwd.Md*(S[ix].ub + S[ix].us)
#            Q[:,ix] = S[ix].X + S[ix].Z
            # print L.fwd.x2u.shape

            M = L.s*(sparse.spdiags(L.fwd.x2u.T*(L.ub+L.us),0,nX,nX))*self.fwd.p2x
            uL += M.T.conj()*M
            bL += M.T.conj()*(L.X + L.Z)

        U = sparse.lil_matrix((n,n),dtype='complex128')
        B = np.zeros(n,dtype='complex128')

        U = comm.allreduce(uL,U,op=MPI.SUM)
        B = comm.allreduce(bL,B,op=MPI.SUM)

        P = lin.spsolve(U,B)
        # print num
        # print den

        P = P.real

        # print P[1]
        P = np.maximum(P,0)
        # print self.upperBound
        P = np.minimum(P,self.upperBound)
        # print P[1]

        return P

    def plotSemiParallel(self,P,resid,rank,ix=0):
        ''' Plotting routine if things are semiParallel'''
        import matplotlib.pyplot as plt
        plt.close('all')
        import os

        plt.close('all')
        if not os.path.exists(self.outDir + 'Figs'):
            os.mkdir(self.outDir + 'Figs')

        # vv = S.Ms*S.v
        uu = self.fwd.Ms*self.us
        ub = self.fwd.Ms*self.ub
        skt = self.uHat

        plt.figure(100 + rank + 10*ix)
        plt.plot(np.arange(self.fwd.nSen), skt.real, np.arange(self.fwd.nSen), uu.real)
        plt.savefig(self.outDir + 'Figs/fig' + repr(100+rank+10*ix))

        if (rank==0) & (ix==0):
            # then print some figures
            plt.figure(383)
            plt.plot(resid)
            plt.savefig(self.outDir + 'Figs/fig383')

            plt.figure(387)
            plt.imshow(P.reshape(self.fwd.nRx,self.fwd.nRy), interpolation='nearest')
            plt.colorbar()
            plt.savefig(self.outDir + 'Figs/fig387')

            us = self.fwd.parseFields(self.us)

            plt.figure(76)
            plt.subplot(121)
            plt.imshow(us[0].real)
            plt.colorbar()

            plt.subplot(122)
            plt.imshow(us[0].imag)
            plt.colorbar()
            plt.savefig(self.outDir + 'Figs/fig76')

        # all show!
        # plt.show()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User

# Create your models here.

## class Mitgliedsstatus (models.Model):
##     status = models.CharField (max_length=30)
##     bemerkung = models.TextField(blank=True)

##     def __unicode__ (self):
##         return self.status

##     class Meta:
##         verbose_name_plural ="Mitgliedsstatus"

## class Mitglied (models.Model):
##     user = models.OneToOneField (User)
##     status = models.ForeignKey (Mitgliedsstatus)

##     def __unicode__ (self):
##         return self.user.__unicode__()

class Aufgabengruppe (models.Model):
    gruppe = models.CharField (max_length=30)
    verantwortlich = models.ForeignKey (User)
    bemerkung = models.TextField(blank=True)

    def __unicode__ (self):
        return self.gruppe

    class Meta:
        verbose_name_plural = "Aufgabengruppen"

class Aufgabe (models.Model):
    aufgabe = models.CharField (max_length=30)
    datum = models.DateField (blank=True, null=True)
    verantwortlich = models.ForeignKey (User)
    gruppe = models.ForeignKey (Aufgabengruppe)
    anzahl = models.IntegerField ()
    bemerkung = models.TextField(blank=True)

    def __unicode__ (self):
        return self.aufgabe

    class Meta:
        verbose_name_plural = "Aufgaben"

class Meldung (models.Model):
    erstellt = models.DateField (auto_now_add=True)
    veraendert = models.DateField (auto_now=True)
    melder = models.ForeignKey (User)
    aufgabe = models.ForeignKey (Aufgabe)

    def __unicode__ (self):
        return (self.melder.__unicode__() + " ; " +
                self.aufgabe.__unicode__() + " ; " +
                (self.veraendert.strftime("%d/%m/%y")
                 if self.veraendert else "--")
                )

    class Meta:
        verbose_name_plural = "Meldungen"

class Zuteilung (models.Model):
    aufgabe = models.ForeignKey (Aufgabe)
    ausfuehrer = models.ForeignKey (User)
    automatisch = models.BooleanField (default=False)
    class Meta:
        verbose_name_plural = "Zuteilungen"

    def __unicode__ (self):
        return self.aufgabe.__unicode__() + ": " + self.ausfuehrer.__unicode__()

class Leistung (models.Model):
    melder = models.ForeignKey (User)
    aufgabe = models.ForeignKey (Aufgabe)
    erstellt = models.DateField (auto_now_add=True)
    veraendert = models.DateField (auto_now=True)
    wann = models.DateField (help_text="An welchem Tag haben Sie die Leistung erbracht?",
                             )
    zeit = models.DecimalField (max_digits=3,
                                decimal_places = 1,
                                help_text="Wieviel Zeit (in Stunden) haben Sie gearbeitet?",
                                )
    auslagen = models.DecimalField (max_digits=6,
                                    decimal_places = 2,
                                    help_text = "Hatten Sie finanzielle Auslagen? Bitte Belege vorlegen!",
                                    default = 0,
                                    )
    km = models.DecimalField (max_digits= 4,
                              decimal_places = 0,
                              help_text = "Hatten Sie Wegstrecken, für die Sie km-Vergütung erhalten?",
                              default = 0,
                              )

    OFFEN =  'OF'
    ACK = 'AK'
    RUECKFRAGE = 'RU'
    NEG = 'NE'

    STATUS = (
        (OFFEN, 'Offen'),
        (ACK, 'Akzeptiert'),
        (RUECKFRAGE, 'Rückfrage'),
        (NEG, 'Abgelehnt'),
        )

    status = models.CharField (max_length=2,
                               choices = STATUS,
                               default = OFFEN)

    bemerkung = models.TextField (blank=True)
    bemerkungVorstand = models.TextField (blank=True)

    class Meta:
        verbose_name_plural = "Leistungen"

    def __unicode__ (self):
        return (self.melder.__unicode__() + " ; " +
                self.aufgabe.__unicode__() + " ; " +
                (self.veraendert.strftime("%d/%m/%y")
                 if self.veraendert else "--")
                )

<commit_msg>missing verbose names<commit_after># -*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User

# Create your models here.

## class Mitgliedsstatus (models.Model):
##     status = models.CharField (max_length=30)
##     bemerkung = models.TextField(blank=True)

##     def __unicode__ (self):
##         return self.status

##     class Meta:
##         verbose_name_plural ="Mitgliedsstatus"

## class Mitglied (models.Model):
##     user = models.OneToOneField (User)
##     status = models.ForeignKey (Mitgliedsstatus)

##     def __unicode__ (self):
##         return self.user.__unicode__()

class Aufgabengruppe (models.Model):
    gruppe = models.CharField (max_length=30)
    verantwortlich = models.ForeignKey (User)
    bemerkung = models.TextField(blank=True)

    def __unicode__ (self):
        return self.gruppe

    class Meta:
        verbose_name_plural = "Aufgabengruppen"
        verbose_name = "Aufgabengruppe"

class Aufgabe (models.Model):
    aufgabe = models.CharField (max_length=30)
    datum = models.DateField (blank=True, null=True)
    verantwortlich = models.ForeignKey (User)
    gruppe = models.ForeignKey (Aufgabengruppe)
    anzahl = models.IntegerField ()
    bemerkung = models.TextField(blank=True)

    def __unicode__ (self):
        return self.aufgabe

    class Meta:
        verbose_name_plural = "Aufgaben"
        verbose_name = "Aufgabe"

class Meldung (models.Model):
    erstellt = models.DateField (auto_now_add=True)
    veraendert = models.DateField (auto_now=True)
    melder = models.ForeignKey (User)
    aufgabe = models.ForeignKey (Aufgabe)

    def __unicode__ (self):
        return (self.melder.__unicode__() + " ; " +
                self.aufgabe.__unicode__() + " ; " +
                (self.veraendert.strftime("%d/%m/%y")
                 if self.veraendert else "--")
                )

    class Meta:
        verbose_name_plural = "Meldungen"
        verbose_name = "Meldung"

class Zuteilung (models.Model):
    aufgabe = models.ForeignKey (Aufgabe)
    ausfuehrer = models.ForeignKey (User)
    automatisch = models.BooleanField (default=False)
    class Meta:
        verbose_name_plural = "Zuteilungen"

    def __unicode__ (self):
        return self.aufgabe.__unicode__() + ": " + self.ausfuehrer.__unicode__()

    class Meta:
        verbose_name_plural = "Zuteilungen"
        verbose_name = "Zuteilung"

class Leistung (models.Model):
    melder = models.ForeignKey (User)
    aufgabe = models.ForeignKey (Aufgabe)
    erstellt = models.DateField (auto_now_add=True)
    veraendert = models.DateField (auto_now=True)
    wann = models.DateField (help_text="An welchem Tag haben Sie die Leistung erbracht?",
                             )
    zeit = models.DecimalField (max_digits=3,
                                decimal_places = 1,
                                help_text="Wieviel Zeit (in Stunden) haben Sie gearbeitet?",
                                )
    auslagen = models.DecimalField (max_digits=6,
                                    decimal_places = 2,
                                    help_text = "Hatten Sie finanzielle Auslagen? Bitte Belege vorlegen!",
                                    default = 0,
                                    )
    km = models.DecimalField (max_digits= 4,
                              decimal_places = 0,
                              help_text = "Hatten Sie Wegstrecken, für die Sie km-Vergütung erhalten?",
                              default = 0,
                              )

    OFFEN =  'OF'
    ACK = 'AK'
    RUECKFRAGE = 'RU'
    NEG = 'NE'

    STATUS = (
        (OFFEN, 'Offen'),
        (ACK, 'Akzeptiert'),
        (RUECKFRAGE, 'Rückfrage'),
        (NEG, 'Abgelehnt'),
        )

    status = models.CharField (max_length=2,
                               choices = STATUS,
                               default = OFFEN)

    bemerkung = models.TextField (blank=True)
    bemerkungVorstand = models.TextField (blank=True)

    class Meta:
        verbose_name_plural = "Leistungen"
        verbose_name = "Leistung"

    def __unicode__ (self):
        return (self.melder.__unicode__() + " ; " +
                self.aufgabe.__unicode__() + " ; " +
                (self.veraendert.strftime("%d/%m/%y")
                 if self.veraendert else "--")
                )

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, absolute_import, unicode_literals, division

import datetime
import traceback
import simplejson

from aws_federation_proxy import (
    AWSFederationProxy,
    ConfigurationError,
    AWSError,
    PermissionError
)
from functools import wraps
from yamlreader import yaml_load
from bottle import route, abort, request, response, error, default_app
from aws_federation_proxy.util import setup_logging

def with_exception_handling(old_function):
    """Decorator function to ensure proper exception handling"""
    @wraps(old_function)
    def new_function(*args, **kwargs):
        try:
            result = old_function(*args, **kwargs)
        except ConfigurationError as err:
            abort(404, err)
        except AWSError as err:
            abort(502, err)
        except PermissionError as err:
            abort(403, err)
        except Exception:
            message = traceback.format_exc()
            abort(500, "Exception caught {0}".format(message))
        return result
    return new_function

def initialize_federation_proxy(user=None):
    """Get needed config parts and initialize AWSFederationProxy"""
    config_path = request.environ.get('CONFIG_PATH')
    if config_path is None:
        raise Exception("No Config Path specified")
    config = yaml_load(config_path)
    account_config_path = request.environ.get('ACCOUNT_CONFIG_PATH')
    if account_config_path is None:
        raise Exception("No Account Config Path specified")
    account_config = yaml_load(account_config_path)

    if user is None:
        user = get_user(config['api']['user_identification'])

    handler_config = config.get('logging_handler')
    try:
        logger = setup_logging(handler_config, logger_name='AWSFederationProxy')
    except Exception as exc:
        raise ConfigurationError(str(exc))
    proxy = AWSFederationProxy(user=user, config=config,
                               account_config=account_config, logger=logger)
    return proxy

def get_user(user_config):
    """
    user_config = {
        'environment_field': REMOTE_USER
    }
    """
    field = user_config['environment_field']
    if field in request.environ:
        return request.environ[field]

    raise Exception("No {0} specified".format(field))

def build_credentials_dict(credentials):
    """Convert AWS Credential object to a dict"""
    return {
        'Code': 'Success',
        'Type': 'AWS-HMAC',
        'AccessKeyId': credentials.access_key,
        'SecretAccessKey': credentials.secret_key,
        'Token': credentials.session_token,
        'Expiration': credentials.expiration,
        'LastUpdated': datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
    }

@error(403)
@error(404)
@error(500)
@error(502)
def get_error_json(err):
    try:
        proxy = initialize_federation_proxy()
    except Exception:
        user = "Unknown User"
    else:
        user = proxy.user
    response_dict = {
        "status": err.status_code,
        "error": err.status,
        "exception": None,
        "message": str(err.body),
        "traceback": err.traceback
    }
    response.set_header('X-Username', user)
    response.content_type = 'application/json; charset=utf-8'
    return simplejson.dumps(response_dict)

def get_proxy_return_json(user=None):
    """A decorator to set up the proxy and convert output into JSON"""
    def decorator(old_function):
        @wraps(old_function)
        def new_function(*args, **kwargs):
            proxy = initialize_federation_proxy(user=user)
            response.set_header('X-Username', proxy.user)
            return_value = old_function(proxy, *args, **kwargs)
            response.content_type = 'application/json; charset=utf-8'
            return simplejson.dumps(return_value)
        return new_function
    return decorator

@route("/status")
@with_exception_handling
@get_proxy_return_json(user='monitoring')
def get_monitoring_status(proxy):
    """Return status page for monitoring"""
    return {"status": "200", "message": "OK"}

@route('/account')
@with_exception_handling
@get_proxy_return_json()
def get_accountlist(proxy):
    """Return a dict-of-lists of all accounts and roles for the current user"""
    def role_set_to_list(role_set):
        """Convert a set of (role, reason) tuples to a list of roles"""
        return [role for role, reason in role_set]
    accounts_and_roles_with_sets = proxy.get_account_and_role_dict()
    accounts_and_roles = dict(
        (key, role_set_to_list(value)) for (key, value)
        in accounts_and_roles_with_sets.items()
    )
    return accounts_and_roles

@route('/account/<account>/<role>')
@with_exception_handling
@get_proxy_return_json()
def get_credentials_and_console(proxy, account, role):
    """ Return credentials and console url

    Return a dict of credentials (access_key, secret_key and session token)
    and ConsoleURL for the specified role in the specified account
    """
    callback_url = request.query.callbackurl or ""
    credentials = proxy.get_aws_credentials(account, role)
    credentials_dict = build_credentials_dict(credentials)
    credentials_dict['ConsoleUrl'] = proxy.get_console_url(credentials,
                                                           callback_url)
    return credentials_dict

@route('/account/<account>/<role>/credentials')
@with_exception_handling
@get_proxy_return_json()
def get_credentials(proxy, account, role):
    """Return credentials

    Return a dict of credentials (AccessKeyId, SecretAccessKey, Token)
    for the specified role in the specified account.
    """
    credentials = proxy.get_aws_credentials(account, role)
    return build_credentials_dict(credentials)

@route('/account/<account>/<role>/consoleurl')
@with_exception_handling
def get_console(account, role):
    """Return ConsoleURL

    Return string of the console URL for the specified role and account
    """
    proxy = initialize_federation_proxy()
    callback_url = request.query.callbackurl or ""
    credentials = proxy.get_aws_credentials(account, role)
    console_url = proxy.get_console_url(credentials, callback_url)
    response.content_type = 'text/plain; charset=utf-8'
    return str(console_url)

def get_account_and_role(proxy):
    accounts_and_roles = proxy.get_account_and_role_dict()
    if len(accounts_and_roles) != 1:
        raise ConfigurationError("Did not get exactly one account: %s" % (
            accounts_and_roles))

    roles = list(accounts_and_roles.values())[0]
    if len(roles) != 1:
        raise ConfigurationError("Did not get exactly one role: %s" % (
            accounts_and_roles))

    account = list(accounts_and_roles.keys())[0]
    role = list(roles)[0][0]
    return account, role

@route('/meta-data/iam/security-credentials/')
@with_exception_handling
def get_ims_role():
    proxy = initialize_federation_proxy()
    _, role = get_account_and_role(proxy)
    response.content_type = 'text/plain'
    return role

@route('/meta-data/iam/security-credentials/<role>')
@with_exception_handling
@get_proxy_return_json()
def get_ims_credentials(proxy, role):
    account, _ = get_account_and_role(proxy)
    try:
        credentials = proxy.get_aws_credentials(account, role)
    except PermissionError as exc:
        abort(404, exc)
    return build_credentials_dict(credentials)

def get_webapp():
    """Give back the bottle default_app, for direct use in wsgi scripts"""
    return default_app()
<commit_msg>Quickfix, else we get an 500 http error (TypeError: expected byte string object for header name, value of type unicode found)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, absolute_import, division

import datetime
import traceback
import simplejson

from aws_federation_proxy import (
    AWSFederationProxy,
    ConfigurationError,
    AWSError,
    PermissionError
)
from functools import wraps
from yamlreader import yaml_load
from bottle import route, abort, request, response, error, default_app
from aws_federation_proxy.util import setup_logging

def with_exception_handling(old_function):
    """Decorator function to ensure proper exception handling"""
    @wraps(old_function)
    def new_function(*args, **kwargs):
        try:
            result = old_function(*args, **kwargs)
        except ConfigurationError as err:
            abort(404, err)
        except AWSError as err:
            abort(502, err)
        except PermissionError as err:
            abort(403, err)
        except Exception:
            message = traceback.format_exc()
            abort(500, "Exception caught {0}".format(message))
        return result
    return new_function

def initialize_federation_proxy(user=None):
    """Get needed config parts and initialize AWSFederationProxy"""
    config_path = request.environ.get('CONFIG_PATH')
    if config_path is None:
        raise Exception("No Config Path specified")
    config = yaml_load(config_path)
    account_config_path = request.environ.get('ACCOUNT_CONFIG_PATH')
    if account_config_path is None:
        raise Exception("No Account Config Path specified")
    account_config = yaml_load(account_config_path)

    if user is None:
        user = get_user(config['api']['user_identification'])

    handler_config = config.get('logging_handler')
    try:
        logger = setup_logging(handler_config, logger_name='AWSFederationProxy')
    except Exception as exc:
        raise ConfigurationError(str(exc))
    proxy = AWSFederationProxy(user=user, config=config,
                               account_config=account_config, logger=logger)
    return proxy

def get_user(user_config):
    """
    user_config = {
        'environment_field': REMOTE_USER
    }
    """
    field = user_config['environment_field']
    if field in request.environ:
        return request.environ[field]

    raise Exception("No {0} specified".format(field))

def build_credentials_dict(credentials):
    """Convert AWS Credential object to a dict"""
    return {
        'Code': 'Success',
        'Type': 'AWS-HMAC',
        'AccessKeyId': credentials.access_key,
        'SecretAccessKey': credentials.secret_key,
        'Token': credentials.session_token,
        'Expiration': credentials.expiration,
        'LastUpdated': datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
    }

@error(403)
@error(404)
@error(500)
@error(502)
def get_error_json(err):
    try:
        proxy = initialize_federation_proxy()
    except Exception:
        user = "Unknown User"
    else:
        user = proxy.user
    response_dict = {
        "status": err.status_code,
        "error": err.status,
        "exception": None,
        "message": str(err.body),
        "traceback": err.traceback
    }
    response.set_header('X-Username', user)
    response.content_type = 'application/json; charset=utf-8'
    return simplejson.dumps(response_dict)

def get_proxy_return_json(user=None):
    """A decorator to set up the proxy and convert output into JSON"""
    def decorator(old_function):
        @wraps(old_function)
        def new_function(*args, **kwargs):
            proxy = initialize_federation_proxy(user=user)
            response.set_header('X-Username', proxy.user)
            return_value = old_function(proxy, *args, **kwargs)
            response.content_type = 'application/json; charset=utf-8'
            return simplejson.dumps(return_value)
        return new_function
    return decorator

@route("/status")
@with_exception_handling
@get_proxy_return_json(user='monitoring')
def get_monitoring_status(proxy):
    """Return status page for monitoring"""
    return {"status": "200", "message": "OK"}

@route('/account')
@with_exception_handling
@get_proxy_return_json()
def get_accountlist(proxy):
    """Return a dict-of-lists of all accounts and roles for the current user"""
    def role_set_to_list(role_set):
        """Convert a set of (role, reason) tuples to a list of roles"""
        return [role for role, reason in role_set]
    accounts_and_roles_with_sets = proxy.get_account_and_role_dict()
    accounts_and_roles = dict(
        (key, role_set_to_list(value)) for (key, value)
        in accounts_and_roles_with_sets.items()
    )
    return accounts_and_roles

@route('/account/<account>/<role>')
@with_exception_handling
@get_proxy_return_json()
def get_credentials_and_console(proxy, account, role):
    """ Return credentials and console url

    Return a dict of credentials (access_key, secret_key and session token)
    and ConsoleURL for the specified role in the specified account
    """
    callback_url = request.query.callbackurl or ""
    credentials = proxy.get_aws_credentials(account, role)
    credentials_dict = build_credentials_dict(credentials)
    credentials_dict['ConsoleUrl'] = proxy.get_console_url(credentials,
                                                           callback_url)
    return credentials_dict

@route('/account/<account>/<role>/credentials')
@with_exception_handling
@get_proxy_return_json()
def get_credentials(proxy, account, role):
    """Return credentials

    Return a dict of credentials (AccessKeyId, SecretAccessKey, Token)
    for the specified role in the specified account.
    """
    credentials = proxy.get_aws_credentials(account, role)
    return build_credentials_dict(credentials)

@route('/account/<account>/<role>/consoleurl')
@with_exception_handling
def get_console(account, role):
    """Return ConsoleURL

    Return string of the console URL for the specified role and account
    """
    proxy = initialize_federation_proxy()
    callback_url = request.query.callbackurl or ""
    credentials = proxy.get_aws_credentials(account, role)
    console_url = proxy.get_console_url(credentials, callback_url)
    response.content_type = 'text/plain; charset=utf-8'
    return str(console_url)

def get_account_and_role(proxy):
    accounts_and_roles = proxy.get_account_and_role_dict()
    if len(accounts_and_roles) != 1:
        raise ConfigurationError("Did not get exactly one account: %s" % (
            accounts_and_roles))

    roles = list(accounts_and_roles.values())[0]
    if len(roles) != 1:
        raise ConfigurationError("Did not get exactly one role: %s" % (
            accounts_and_roles))

    account = list(accounts_and_roles.keys())[0]
    role = list(roles)[0][0]
    return account, role

@route('/meta-data/iam/security-credentials/')
@with_exception_handling
def get_ims_role():
    proxy = initialize_federation_proxy()
    _, role = get_account_and_role(proxy)
    response.content_type = 'text/plain'
    return role

@route('/meta-data/iam/security-credentials/<role>')
@with_exception_handling
@get_proxy_return_json()
def get_ims_credentials(proxy, role):
    account, _ = get_account_and_role(proxy)
    try:
        credentials = proxy.get_aws_credentials(account, role)
    except PermissionError as exc:
        abort(404, exc)
    return build_credentials_dict(credentials)

def get_webapp():
    """Give back the bottle default_app, for direct use in wsgi scripts"""
    return default_app()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.core.exceptions import ValidationError
from django.db.utils import IntegrityError
from django.test import TestCase

from 臺灣言語資料庫.資料模型 import 來源表
from 臺灣言語平臺.使用者模型 import 使用者表

class 使用者表試驗(TestCase):
	def setUp(self):
		pass
	def tearDown(self):
		pass
	def test_有email佮來源(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		來源 = 來源表. 加來源(來源內容)
		使用者 = 使用者表(email='sui2@pigu.tw',來源=來源)
		使用者.set_unusable_password()
		使用者.full_clean()
	def test_愛有email(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		來源 = 來源表. 加來源(來源內容)
		使用者 = 使用者表(來源=來源)
		self.assertRaises(ValidationError, 使用者.full_clean,)
	def test_愛有來源(self):
		使用者 = 使用者表(email='sui2@pigu.tw')
		self.assertRaises(ValidationError, 使用者.full_clean,)

# 	加使用者
	def test_加使用者(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者 = 使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertEqual(使用者.來源.編號(), 使用者.編號())
	def test_重覆email(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertRaises(IntegrityError, 使用者表.加使用者, 'sui2@pigu.tw', 來源內容,)
	def test_密碼愛鎖起來(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者 = 使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertFalse(使用者.has_usable_password())
<commit_msg>修正使用者表試驗無處理密碼 丞宏<commit_after># -*- coding: utf-8 -*-
from django.core.exceptions import ValidationError
from django.db.utils import IntegrityError
from django.test import TestCase

from 臺灣言語資料庫.資料模型 import 來源表
from 臺灣言語平臺.使用者模型 import 使用者表

class 使用者表試驗(TestCase):
	def setUp(self):
		pass
	def tearDown(self):
		pass
	def test_有email佮來源(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		來源 = 來源表. 加來源(來源內容)
		使用者 = 使用者表(email='sui2@pigu.tw',來源=來源)
		使用者.set_unusable_password()
		使用者.full_clean()
	def test_愛有email(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		來源 = 來源表. 加來源(來源內容)
		使用者 = 使用者表(來源=來源)
		使用者.set_unusable_password()
		self.assertRaises(ValidationError, 使用者.full_clean,)
	def test_愛有來源(self):
		使用者 = 使用者表(email='sui2@pigu.tw')
		使用者.set_unusable_password()
		self.assertRaises(ValidationError, 使用者.full_clean,)

# 	加使用者
	def test_加使用者(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者 = 使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertEqual(使用者.來源.編號(), 使用者.編號())
	def test_重覆email(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertRaises(IntegrityError, 使用者表.加使用者, 'sui2@pigu.tw', 來源內容,)
	def test_密碼愛鎖起來(self):
		來源內容 = {"名":'鄉民', '出世年':'1950', '出世地':'臺灣', }
		使用者 = 使用者表.加使用者('sui2@pigu.tw', 來源內容,)
		self.assertFalse(使用者.has_usable_password())
<|endoftext|>