<commit_before># xpyBuild - eXtensible Python-based Build System
#
# Copyright (c) 2013 - 2017 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: native.py 301527 2017-02-06 15:31:43Z matj $
#

import os, inspect, re, string, time

from buildcommon import *
from basetarget import BaseTarget
from propertysupport import defineOption
from utils.process import call
from pathsets import PathSet, BasePathSet
from buildcontext import getBuildInitializationContext
from buildexceptions import BuildException
from propertyfunctors import make_functor, Composable
from utils.compilers import DefaultCompilers
from utils.fileutils import openForWrite, mkdir, deleteFile, getmtime, exists, normLongPath

defineOption('native.compilers', DefaultCompilers())
defineOption('native.libs', [])
defineOption('native.libpaths', [])
defineOption('native.c.flags', None) # defaults to native.cxx.flags if not set
defineOption('native.cxx.flags', [])
defineOption('native.cxx.path', [])
defineOption('native.include', [])
defineOption('native.link.flags', [])

if isWindows():
	defineOption('native.cxx.exenamefn', FilenameStringFormatter("%s.exe"))
	defineOption('native.cxx.libnamefn', FilenameStringFormatter("%s.dll"))
	defineOption('native.cxx.staticlibnamefn', FilenameStringFormatter("%s.lib"))
	defineOption('native.cxx.objnamefn', FilenameStringFormatter("%s.obj"))
else:
	defineOption('native.cxx.exenamefn', FilenameStringFormatter("%s"))
	defineOption('native.cxx.libnamefn', FilenameStringFormatter("lib%s.so"))
	defineOption('native.cxx.staticlibnamefn', FilenameStringFormatter("lib%s.a"))
	defineOption('native.cxx.objnamefn', FilenameStringFormatter("%s.o"))

class CompilerMakeDependsPathSet(BasePathSet):
	"""
		Use the selection ToolChain to get a list of dependencies from a set of source files
	"""
	def __init__(self, target, src, flags=None, includes=None):
		"""
		@param target: the BaseTarget object for which this path set is being caculated

		@param src: a PathSet of source file paths

		@param flags: additional compiler flags

		@param includes: a list of include directory paths
		"""
		self.target = target
		self.sources = src
		self.flags = flatten([flags]) or []
		self.includes = includes or []
	def __repr__(self):
		return "MakeDepend(%s, %s)" % (self.sources, self.flags)
	def resolveWithDestinations(self, context):
		return [(i, os.path.basename(i)) for i in _resolveUnderlyingDependencies(context)]
	def clean(self):
		dfile = self.target.workDir+'.makedepend'
		log = logging.getLogger('MakeDepend')
		deleteFile(dfile)
	def _resolveUnderlyingDependencies(self, context):
		deplist = None
		options = context.mergeOptions(self.target) # get the merged options
		log = logging.getLogger('MakeDepend')

		dfile = normLongPath(self.target.workDir+'.makedepend')
		testsources = self.sources.resolve(context)
		depsources = self.sources._resolveUnderlyingDependencies(context)

		needsRebuild = not os.path.exists(dfile)
		if needsRebuild:
			log.info("Rebuilding dependencies for %s because cached dependencies file does not exist (%s)" % (self.target, dfile))
		dfiletime = 0 if needsRebuild else getmtime(dfile)
		for x in testsources:
			if not os.path.exists(x):
				return depsources
			elif getmtime(x) > dfiletime:
				if not needsRebuild:	log.info("Rebuilding dependencies for %s because cached dependencies file is older than %s" % (self.target, x))
				needsRebuild = True

		if not needsRebuild: # read in cached dependencies
			deplist = []
			with open(dfile) as f:
				lines = f.readlines()
				header = lines[0].strip()
				lines = lines[1:]
				for d in lines:
					d = d.strip()
					if context._isValidTarget(d) or exists(normLongPath(d)):
						deplist.append(d)
					else:
						needsRebuild = True
						log.warn("Rebuilding dependencies for %s because dependency %s is missing" % (self.target, d))
						break
			if header != str(self):
				log.info("Rebuilding dependencies for %s because target options have changed (%s != %s)" % (self.target, header, str(self)))
			elif not needsRebuild:
				return deplist

		# generate them again
		startt = time.time()
		log.critical("*** Generating dependencies for %s" % self.target)
		deplist = options['native.compilers'].dependencies.depends(context=context, src=testsources, options=options, flags=flatten(options['native.cxx.flags']+[context.expandPropertyValues(x).split(' ') for x in self.flags]), includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))
		deplist += depsources
		mkdir(os.path.dirname(dfile))
		with openForWrite(dfile, 'wb') as f:
			assert not os.linesep in str(self)
			f.write(str(self)+os.linesep)
			for d in deplist:
				f.write(d.encode('UTF-8')+os.linesep)
		if time.time()-startt > 5: # this should usually be pretty quick, so may indicate a real build file mistake
			log.warn('Dependency generation took a long time: %0.1f s to evaluate %s', time.time()-startt, self)

		return deplist

class Cpp(BaseTarget):
	""" A target that compiles a C++ source file to a .o
	"""

	def __init__(self, object, source, includes=None, flags=None, dependencies=None, options=None):
		"""
		@param object: the object file to generate
		@param source: a (list of) source files
		@param includes: a (list of) include directories
		@param flags: a list of additional compiler flags
		@param dependencies: a list of additional dependencies that need to be built
		before this target
		"""
		self.source = PathSet(source)
		self.includes = PathSet(includes or [])
		self.flags = flatten([flags]) or []
		self.options = options or {}
		self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
		BaseTarget.__init__(self, object, [dependencies or [], self.source, self.makedepend])
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].cxxcompiler.compile(context, output=self.path, options=options, flags=flatten(options['native.cxx.flags']+[context.expandPropertyValues(x).split(' ') for x in self.flags]), src=self.source.resolve(context), includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))

	def clean(self, context):
		self.makedepend.clean()
		BaseTarget.clean(self, context)

	def getHashableImplicitInputs(self, context):
		r = super(Cpp, self).getHashableImplicitInputs(context)

		# include input to makedepends, since even without running makedepends
		# we know we're out of date if inputs have changed
		r.append('depends: '+context.expandPropertyValues(str(self.makedepend)))

		return r

class C(BaseTarget):
	""" A target that compiles a C source file to a .o
	"""

	def __init__(self, object, source, includes=None, flags=None, options=None, dependencies=None):
		"""
		@param object: the object file to generate
		@param source: a (list of) source files
		@param includes: a (list of) include directories
		@param flags: a list of additional compiler flags
		@param dependencies: a list of additional dependencies that need to be built
		before this target
		"""
		self.source = PathSet(source)
		self.includes = PathSet(includes or [])
		self.flags = flags or []
		self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
		BaseTarget.__init__(self, object, [dependencies or [], self.makedepend])
		self.options = options or {}
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].ccompiler.compile(context, output=self.path,
				options=options,
				flags=flatten((options['native.c.flags'] or options['native.cxx.flags'])+[context.expandPropertyValues(x).split(' ') for x in self.flags]),
				src=self.source.resolve(context),
				includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))

	def clean(self, context):
		self.makedepend.clean()
		BaseTarget.clean(self, context)

	def getHashableImplicitInputs(self, context):
		r = super(C, self).getHashableImplicitInputs(context)

		# include input to makedepends, since even without running makedepends
		# we know we're out of date if inputs have changed
		r.append(context.expandPropertyValues(str(self.makedepend)))

		return r

class Link(BaseTarget):
	""" A target that links object files to binaries
	"""

	def __init__(self, bin, objects, libs=None, libpaths=None, shared=False, options=None, flags=None, dependencies=None):
		"""
		@param bin: the output binary

		@param objects: a (list of) input object

		@param libs: a (list of) libraries linked against (optional) in platform-neutral format.
		Can include list properties like '${FOO_LIB_NAMES[]}'.

		@param libpaths: a (list of) additional library search directories (optional)

		@param shared: if true compiles to a shared object (.dll or .so) (optional, defaults to false)

		@param flags: a list of additional linker flags

		@param options: a map of options to the underlying operation specific to this target (optional)

		@param dependencies: a list of additional dependencies (targets or files)
		"""
		self.objects = PathSet(objects)
		self.libs = libs or []
		self.libpaths = PathSet(libpaths or [])
		self.shared=shared
		self.options = options
		self.flags = flags or []
		BaseTarget.__init__(self, bin, PathSet(self.objects, (dependencies or [])))
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].linker.link(context, output=self.path,
				options=options,
				flags=options['native.link.flags']+self.flags,
				shared=self.shared,
				src=self.objects.resolve(context),
				libs=flatten([map(string.strip, context.expandPropertyValues(x, expandList=True)) for x in self.libs+options['native.libs'] if x]),
				libdirs=flatten(self.libpaths.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.libpaths']]))

	def getHashableImplicitInputs(self, context):
		r = super(Link, self).getHashableImplicitInputs(context)

		options = context.mergeOptions(self)
		r.append('libs: '+context.expandPropertyValues(str(self.libs+options['native.libs'])))
		r.append('libpaths: '+context.expandPropertyValues(str(self.libpaths)))
		r.append('native.libpaths: %s'%options['native.libpaths'])
		r.append('shared: %s, flags=%s'%(self.shared, self.flags))

		return r

class Ar(BaseTarget):
	""" A target that compiles .a files from collections of .o files
	"""

	def __init__(self, bin, objects):
		"""
		@param bin: the output library

		@param objects: a (list of) input objects

		"""
		self.objects = PathSet(objects)
		BaseTarget.__init__(self, bin, self.objects)
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].archiver.archive(context, output=self.path,
				options=options,
				src=self.objects.resolve(context))

	def getHashableImplicitInputs(self, context):
		r = super(Ar, self).getHashableImplicitInputs(context)

		r.append('objects: %s'%self.objects)

		return r

exename = make_functor(lambda c, i:c.mergeOptions()['native.cxx.exenamefn'](c.expandPropertyValues(i)), name='exename')
objectname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.objnamefn'](c.expandPropertyValues(i)), name='objectname')
libname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.libnamefn'](c.expandPropertyValues(i)), name='libname')
staticlibname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.staticlibnamefn'](c.expandPropertyValues(i)), name='staticlibname')

<commit_msg>CompilerMakeDependsPathSet ought to call baseclass __init__ jsut in case it ever isn't a no-op<commit_after># xpyBuild - eXtensible Python-based Build System
#
# Copyright (c) 2013 - 2017 Software AG, Darmstadt, Germany and/or its licensors
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# $Id: native.py 301527 2017-02-06 15:31:43Z matj $
#

import os, inspect, re, string, time

from buildcommon import *
from basetarget import BaseTarget
from propertysupport import defineOption
from utils.process import call
from pathsets import PathSet, BasePathSet
from buildcontext import getBuildInitializationContext
from buildexceptions import BuildException
from propertyfunctors import make_functor, Composable
from utils.compilers import DefaultCompilers
from utils.fileutils import openForWrite, mkdir, deleteFile, getmtime, exists, normLongPath

defineOption('native.compilers', DefaultCompilers())
defineOption('native.libs', [])
defineOption('native.libpaths', [])
defineOption('native.c.flags', None) # defaults to native.cxx.flags if not set
defineOption('native.cxx.flags', [])
defineOption('native.cxx.path', [])
defineOption('native.include', [])
defineOption('native.link.flags', [])

if isWindows():
	defineOption('native.cxx.exenamefn', FilenameStringFormatter("%s.exe"))
	defineOption('native.cxx.libnamefn', FilenameStringFormatter("%s.dll"))
	defineOption('native.cxx.staticlibnamefn', FilenameStringFormatter("%s.lib"))
	defineOption('native.cxx.objnamefn', FilenameStringFormatter("%s.obj"))
else:
	defineOption('native.cxx.exenamefn', FilenameStringFormatter("%s"))
	defineOption('native.cxx.libnamefn', FilenameStringFormatter("lib%s.so"))
	defineOption('native.cxx.staticlibnamefn', FilenameStringFormatter("lib%s.a"))
	defineOption('native.cxx.objnamefn', FilenameStringFormatter("%s.o"))

class CompilerMakeDependsPathSet(BasePathSet):
	"""
		Use the selection ToolChain to get a list of dependencies from a set of source files
	"""
	def __init__(self, target, src, flags=None, includes=None):
		"""
		@param target: the BaseTarget object for which this path set is being caculated

		@param src: a PathSet of source file paths

		@param flags: additional compiler flags

		@param includes: a list of include directory paths
		"""
		BasePathSet.__init__(self)
		self.target = target
		self.sources = src
		self.flags = flatten([flags]) or []
		self.includes = includes or []

	def __repr__(self):
		return "MakeDepend(%s, %s)" % (self.sources, self.flags)
	def resolveWithDestinations(self, context):
		return [(i, os.path.basename(i)) for i in _resolveUnderlyingDependencies(context)]
	def clean(self):
		dfile = self.target.workDir+'.makedepend'
		log = logging.getLogger('MakeDepend')
		deleteFile(dfile)
	def _resolveUnderlyingDependencies(self, context):
		deplist = None
		options = context.mergeOptions(self.target) # get the merged options
		log = logging.getLogger('MakeDepend')

		dfile = normLongPath(self.target.workDir+'.makedepend')
		testsources = self.sources.resolve(context)
		depsources = self.sources._resolveUnderlyingDependencies(context)

		needsRebuild = not os.path.exists(dfile)
		if needsRebuild:
			log.info("Rebuilding dependencies for %s because cached dependencies file does not exist (%s)" % (self.target, dfile))
		dfiletime = 0 if needsRebuild else getmtime(dfile)
		for x in testsources:
			if not os.path.exists(x):
				return depsources
			elif getmtime(x) > dfiletime:
				if not needsRebuild:	log.info("Rebuilding dependencies for %s because cached dependencies file is older than %s" % (self.target, x))
				needsRebuild = True

		if not needsRebuild: # read in cached dependencies
			deplist = []
			with open(dfile) as f:
				lines = f.readlines()
				header = lines[0].strip()
				lines = lines[1:]
				for d in lines:
					d = d.strip()
					if context._isValidTarget(d) or exists(normLongPath(d)):
						deplist.append(d)
					else:
						needsRebuild = True
						log.warn("Rebuilding dependencies for %s because dependency %s is missing" % (self.target, d))
						break
			if header != str(self):
				log.info("Rebuilding dependencies for %s because target options have changed (%s != %s)" % (self.target, header, str(self)))
			elif not needsRebuild:
				return deplist

		# generate them again
		startt = time.time()
		log.critical("*** Generating dependencies for %s" % self.target)
		deplist = options['native.compilers'].dependencies.depends(context=context, src=testsources, options=options, flags=flatten(options['native.cxx.flags']+[context.expandPropertyValues(x).split(' ') for x in self.flags]), includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))
		deplist += depsources
		mkdir(os.path.dirname(dfile))
		with openForWrite(dfile, 'wb') as f:
			assert not os.linesep in str(self)
			f.write(str(self)+os.linesep)
			for d in deplist:
				f.write(d.encode('UTF-8')+os.linesep)
		if time.time()-startt > 5: # this should usually be pretty quick, so may indicate a real build file mistake
			log.warn('Dependency generation took a long time: %0.1f s to evaluate %s', time.time()-startt, self)

		return deplist

class Cpp(BaseTarget):
	""" A target that compiles a C++ source file to a .o
	"""

	def __init__(self, object, source, includes=None, flags=None, dependencies=None, options=None):
		"""
		@param object: the object file to generate
		@param source: a (list of) source files
		@param includes: a (list of) include directories
		@param flags: a list of additional compiler flags
		@param dependencies: a list of additional dependencies that need to be built
		before this target
		"""
		self.source = PathSet(source)
		self.includes = PathSet(includes or [])
		self.flags = flatten([flags]) or []
		self.options = options or {}
		self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
		BaseTarget.__init__(self, object, [dependencies or [], self.source, self.makedepend])
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].cxxcompiler.compile(context, output=self.path, options=options, flags=flatten(options['native.cxx.flags']+[context.expandPropertyValues(x).split(' ') for x in self.flags]), src=self.source.resolve(context), includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))

	def clean(self, context):
		self.makedepend.clean()
		BaseTarget.clean(self, context)

	def getHashableImplicitInputs(self, context):
		r = super(Cpp, self).getHashableImplicitInputs(context)

		# include input to makedepends, since even without running makedepends
		# we know we're out of date if inputs have changed
		r.append('depends: '+context.expandPropertyValues(str(self.makedepend)))

		return r

class C(BaseTarget):
	""" A target that compiles a C source file to a .o
	"""

	def __init__(self, object, source, includes=None, flags=None, options=None, dependencies=None):
		"""
		@param object: the object file to generate
		@param source: a (list of) source files
		@param includes: a (list of) include directories
		@param flags: a list of additional compiler flags
		@param dependencies: a list of additional dependencies that need to be built
		before this target
		"""
		self.source = PathSet(source)
		self.includes = PathSet(includes or [])
		self.flags = flags or []
		self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
		BaseTarget.__init__(self, object, [dependencies or [], self.makedepend])
		self.options = options or {}
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].ccompiler.compile(context, output=self.path,
				options=options,
				flags=flatten((options['native.c.flags'] or options['native.cxx.flags'])+[context.expandPropertyValues(x).split(' ') for x in self.flags]),
				src=self.source.resolve(context),
				includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))

	def clean(self, context):
		self.makedepend.clean()
		BaseTarget.clean(self, context)

	def getHashableImplicitInputs(self, context):
		r = super(C, self).getHashableImplicitInputs(context)

		# include input to makedepends, since even without running makedepends
		# we know we're out of date if inputs have changed
		r.append(context.expandPropertyValues(str(self.makedepend)))

		return r

class Link(BaseTarget):
	""" A target that links object files to binaries
	"""

	def __init__(self, bin, objects, libs=None, libpaths=None, shared=False, options=None, flags=None, dependencies=None):
		"""
		@param bin: the output binary

		@param objects: a (list of) input object

		@param libs: a (list of) libraries linked against (optional) in platform-neutral format.
		Can include list properties like '${FOO_LIB_NAMES[]}'.

		@param libpaths: a (list of) additional library search directories (optional)

		@param shared: if true compiles to a shared object (.dll or .so) (optional, defaults to false)

		@param flags: a list of additional linker flags

		@param options: a map of options to the underlying operation specific to this target (optional)

		@param dependencies: a list of additional dependencies (targets or files)
		"""
		self.objects = PathSet(objects)
		self.libs = libs or []
		self.libpaths = PathSet(libpaths or [])
		self.shared=shared
		self.options = options
		self.flags = flags or []
		BaseTarget.__init__(self, bin, PathSet(self.objects, (dependencies or [])))
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].linker.link(context, output=self.path,
				options=options,
				flags=options['native.link.flags']+self.flags,
				shared=self.shared,
				src=self.objects.resolve(context),
				libs=flatten([map(string.strip, context.expandPropertyValues(x, expandList=True)) for x in self.libs+options['native.libs'] if x]),
				libdirs=flatten(self.libpaths.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.libpaths']]))

	def getHashableImplicitInputs(self, context):
		r = super(Link, self).getHashableImplicitInputs(context)

		options = context.mergeOptions(self)
		r.append('libs: '+context.expandPropertyValues(str(self.libs+options['native.libs'])))
		r.append('libpaths: '+context.expandPropertyValues(str(self.libpaths)))
		r.append('native.libpaths: %s'%options['native.libpaths'])
		r.append('shared: %s, flags=%s'%(self.shared, self.flags))

		return r

class Ar(BaseTarget):
	""" A target that compiles .a files from collections of .o files
	"""

	def __init__(self, bin, objects):
		"""
		@param bin: the output library

		@param objects: a (list of) input objects

		"""
		self.objects = PathSet(objects)
		BaseTarget.__init__(self, bin, self.objects)
		self.tags('native')

	def run(self, context):
		options = context.mergeOptions(self) # get the merged options

		mkdir(os.path.dirname(self.path))
		options['native.compilers'].archiver.archive(context, output=self.path,
				options=options,
				src=self.objects.resolve(context))

	def getHashableImplicitInputs(self, context):
		r = super(Ar, self).getHashableImplicitInputs(context)

		r.append('objects: %s'%self.objects)

		return r

exename = make_functor(lambda c, i:c.mergeOptions()['native.cxx.exenamefn'](c.expandPropertyValues(i)), name='exename')
objectname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.objnamefn'](c.expandPropertyValues(i)), name='objectname')
libname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.libnamefn'](c.expandPropertyValues(i)), name='libname')
staticlibname = make_functor(lambda c, i:c.mergeOptions()['native.cxx.staticlibnamefn'](c.expandPropertyValues(i)), name='staticlibname')

<|endoftext|>


<commit_before>from __future__ import division

from collections import OrderedDict

import numpy as NP
import scipy.constants
from intervals import FloatInterval

def parse_conductivity(fid):
    """
    Parse a USGS model conductivity file-like object *fid*. Return an
    ordered mapping between layer depth interval, [upper depth,
    lower_depth) in [m], and the conductivity [(Ohm-m)^{-1}]
    """
    for line in fid:
        if line.startswith('*/'):
            if 'thicknesses' in line.split('!')[1].lower():
                thicknesses = map(float, line[1:].split('/')[1].split(',')[:-1])
            elif 'resistivities' in line.split('!')[1].lower():
                resistivites = list(map(float, line[1:].split('/')[1].split(',')[:-1]))
            else:
                break
    last_depth = 0
    bounds = []
    for depth in thicknesses:
        bounds.append(FloatInterval.closed_open(last_depth, last_depth + depth))
        last_depth += depth
    return OrderedDict(zip(bounds,
                           1 / NP.array(resistivites)))

def surface_impedance_1D(conductivity_map, omega):
    """
    Calculate the surface impedance [Ohm] given the 1-D conductivity
    model *conductivity_map* at angular frequencies *omega* [rad].
    """
    # start at bottom layer
    sigma = list(conductivity_map.values())[-1]
    # (5) in NERC, Application Guide: Computing
    # geomagnetically-induced current in the bulk power-system, 2013.
    #
    # http://www.nerc.com/pa/Stand/Project201303GeomagneticDisturbanceMitigation/GIC%20Application%20Guide%202013_approved.pdf
    k = NP.sqrt(1j * omega * scipy.constants.mu_0 * sigma)
    # (6)
    Z = 1j * omega * scipy.constants.mu_0 / k
    # iterate in reversed order (i.e., interior to exterior) and skip the bottom layer
    for interval_i, sigma_i in list(conductivity_map.items())[-1::-1]:
        k = NP.sqrt(1j * omega * scipy.constants.mu_0 * sigma_i)
        # (7)
        A = k * Z / (1j * omega * scipy.constants.mu_0)
        r = (1 - A) / (1 + A)
        # (8)
        d = interval_i.length
        B = r * NP.exp(-2 * k * d)
        Z = 1j * omega * scipy.constants.mu_0 * (1 - B) / (k * (1 + B))
    return Z

if __name__ == '__main__':
    from usgs_conductivity import USGS_MODEL_MAP

    import pylab as PL

    usgs_map_pt1 = USGS_MODEL_MAP['PT_1']
    usgs_map_ip4 = USGS_MODEL_MAP['IP_4']

    N = 1000
    depths = NP.logspace(2, 6, N)

    resistivites_pt1 = NP.empty(N)
    for interval, sigma in usgs_map_pt1.iteritems():
        I = [i for i, d in enumerate(depths) if d in interval]
        resistivites_pt1[I] = 1 / sigma

    resistivites_ip4 = NP.empty(N)
    for interval, sigma in usgs_map_ip4.iteritems():
        I = [i for i, d in enumerate(depths) if d in interval]
        resistivites_ip4[I] = 1 / sigma

    fig = PL.figure(figsize=(8.5, 11))
    PL.loglog(resistivites_pt1,
              NP.array(depths) / 1e3)
    PL.gca().invert_yaxis()
    PL.xlabel('Resistivity [$\Omega$ / m]')
    PL.ylabel('Depth [km]')
    PL.title('1-D Resistivity Model for Piedmont (SE Appalachians) Model PT-1')

    fig = PL.figure(figsize=(8.5, 11))
    PL.loglog(resistivites_ip4,
              NP.array(depths) / 1e3)
    PL.gca().invert_yaxis()
    PL.xlabel('Resistivity [$\Omega$ / m]')
    PL.ylabel('Depth [km]')
    PL.title('1-D Resistivity Model for Interior Plains (Great Plains) Model IP-4')

    f = NP.logspace(-5, -1, N)
    omega = 2 * NP.pi * f

    Z_pt1 = surface_impedance_1D(usgs_map_pt1, omega)
    Z_ip4 = surface_impedance_1D(usgs_map_ip4, omega)

    fig = PL.figure(figsize=(11, 8.5))
    PL.semilogx(f,
                NP.abs(Z_pt1),
                label='PT-1',
                c='b')
    PL.semilogx(f,
                NP.abs(Z_ip4),
                label='IP-4',
                c='g')
    PL.grid(which='both')
    PL.xlabel('Frequency [Hz]')
    PL.ylabel('$|Z(\omega)|$ [Ohm]')
    PL.legend(loc='upper left')
    PL.title('Frequency response of two layered Earth conductivity models')

    PL.show()
<commit_msg>Fix for single interval conductivity maps.<commit_after>from __future__ import division

from collections import OrderedDict

import numpy as NP
import scipy.constants
from intervals import FloatInterval

def parse_conductivity(fid):
    """
    Parse a USGS model conductivity file-like object *fid*. Return an
    ordered mapping between layer depth interval, [upper depth,
    lower_depth) in [m], and the conductivity [(Ohm-m)^{-1}]
    """
    for line in fid:
        if line.startswith('*/'):
            if 'thicknesses' in line.split('!')[1].lower():
                thicknesses = map(float, line[1:].split('/')[1].split(',')[:-1])
            elif 'resistivities' in line.split('!')[1].lower():
                resistivites = list(map(float, line[1:].split('/')[1].split(',')[:-1]))
            else:
                break
    last_depth = 0
    bounds = []
    for depth in thicknesses:
        bounds.append(FloatInterval.closed_open(last_depth, last_depth + depth))
        last_depth += depth
    return OrderedDict(zip(bounds,
                           1 / NP.array(resistivites)))

def surface_impedance_1D(conductivity_map, omega):
    """
    Calculate the surface impedance [Ohm] given the 1-D conductivity
    model *conductivity_map* at angular frequencies *omega* [rad].
    """
    # check that bottom layer is an open half space
    assert list(conductivity_map.keys())[-1].length == float('inf')
    # start at bottom layer
    sigma = list(conductivity_map.values())[-1]
    # (5) in NERC, Application Guide: Computing
    # geomagnetically-induced current in the bulk power-system, 2013.
    #
    # http://www.nerc.com/pa/Stand/Project201303GeomagneticDisturbanceMitigation/GIC%20Application%20Guide%202013_approved.pdf
    k = NP.sqrt(1j * omega * scipy.constants.mu_0 * sigma)
    # (6)
    Z = 1j * omega * scipy.constants.mu_0 / k
    # iterate in reversed order (i.e., interior to exterior) and skip the bottom layer
    for interval_i, sigma_i in list(conductivity_map.items())[-2::-1]:
        k = NP.sqrt(1j * omega * scipy.constants.mu_0 * sigma_i)
        # (7)
        A = k * Z / (1j * omega * scipy.constants.mu_0)
        r = (1 - A) / (1 + A)
        # (8)
        d = interval_i.length
        B = r * NP.exp(-2 * k * d)
        Z = 1j * omega * scipy.constants.mu_0 * (1 - B) / (k * (1 + B))
    return Z

if __name__ == '__main__':
    from .usgs_conductivity import USGS_MODEL_MAP

    import pylab as PL

    usgs_map_pt1 = USGS_MODEL_MAP['PT_1']
    usgs_map_ip4 = USGS_MODEL_MAP['IP_4']

    N = 1000
    depths = NP.logspace(2, 6, N)

    resistivites_pt1 = NP.empty(N)
    for interval, sigma in usgs_map_pt1.items():
        I = [i for i, d in enumerate(depths) if d in interval]
        resistivites_pt1[I] = 1 / sigma

    resistivites_ip4 = NP.empty(N)
    for interval, sigma in usgs_map_ip4.items():
        I = [i for i, d in enumerate(depths) if d in interval]
        resistivites_ip4[I] = 1 / sigma

    fig = PL.figure(figsize=(8.5, 11))
    PL.loglog(resistivites_pt1,
              NP.array(depths) / 1e3)
    PL.gca().invert_yaxis()
    PL.xlabel('Resistivity [$\Omega$ / m]')
    PL.ylabel('Depth [km]')
    PL.title('1-D Resistivity Model for Piedmont (SE Appalachians) Model PT-1')

    fig = PL.figure(figsize=(8.5, 11))
    PL.loglog(resistivites_ip4,
              NP.array(depths) / 1e3)
    PL.gca().invert_yaxis()
    PL.xlabel('Resistivity [$\Omega$ / m]')
    PL.ylabel('Depth [km]')
    PL.title('1-D Resistivity Model for Interior Plains (Great Plains) Model IP-4')

    f = NP.logspace(-5, -1, N)
    omega = 2 * NP.pi * f

    Z_pt1 = surface_impedance_1D(usgs_map_pt1, omega)
    Z_ip4 = surface_impedance_1D(usgs_map_ip4, omega)

    fig = PL.figure(figsize=(11, 8.5))
    PL.semilogx(f,
                NP.abs(Z_pt1),
                label='PT-1',
                c='b')
    PL.semilogx(f,
                NP.abs(Z_ip4),
                label='IP-4',
                c='g')
    PL.grid(which='both')
    PL.xlabel('Frequency [Hz]')
    PL.ylabel('$|Z(\omega)|$ [Ohm]')
    PL.legend(loc='upper left')
    PL.title('Frequency response of two layered Earth conductivity models')

    PL.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Fits tabulated throughput corrections to a model
"""
import argparse

import numpy as np

import matplotlib as mpl
mpl.use('Agg')
mpl.rcParams.update({'font.size': 10})
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.lines as mlines

from sklearn import linear_model
import scipy.optimize

def main():
    # parse command-line arguments
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--verbose", action="store_true",
        help="print verbose output")
    parser.add_argument("-o", "--output", type=str, default=None,
        help="output file base name")
    parser.add_argument("-i", "--input", type=str, default=None,
        help="required input file")
    parser.add_argument("--nexamples", type=int, default=-1,
        help="number of example fits to plot, (array slice 'end' value)")
    parser.add_argument("--seed", type=int, default=2810,
        help="random seed for example choices")
    parser.add_argument("--scipy", action="store_true",
        help="use scipy.optimize instead of sklearn")
    parser.add_argument("--exp-info", type=str, default=None,
        help="add exp info")
    parser.add_argument("--skip-plots", action="store_true",
        help="don't make plots")
    args = parser.parse_args()

    if args.verbose:
        print 'Reading file: %s' % args.input

    # for now, construct the title from the input filename
    title = '-'.join((args.input).split('-')[1:])[:-4]

    # The input data is text file where each line coresponds to
    # a target's throughput correction vector
    data = np.loadtxt(args.input, ndmin=2)

    try:
        nentries, ntokens = data.shape
    except ValueError:
        print data.shape, args.input

    # the first two columns are xfocal and yfocal positions of the target
    nidtokens = 3
    # the rest are the tabulated throughput correction values
    npoints = ntokens - nidtokens

    # the throughput correction vectors span the range 3500A to 10500A
    xvalues = np.linspace(3500, 10500, npoints, endpoint=True)

    nparams = 3
    results = np.empty((nentries, nparams))
    chisqs = np.empty(nentries)

    # fit individual entries
    for i,row in enumerate(data):
        yvalues = row[nidtokens:]
        if args.scipy:
            # chisq function for our model
            def chisq(params):
                sigma = 1e-1
                x0 = np.exp(params[0])
                pred = 1+params[1]*np.log(xvalues/x0)+params[2]*np.log(xvalues/x0)**2
                residuals = (yvalues - pred)/sigma
                return np.dot(residuals,residuals)
            params0 = np.array([np.log(7000),1,-.5])
            result = scipy.optimize.minimize(chisq, params0, options={'maxiter':10000},
                method='Nelder-Mead')
            # save fit results
            results[i,:] = result.x
            results[i,0] = np.exp(result.x[0])
            chisqs[i] = result.fun
            if not result.success:
                # chisq function for our model
                def chisq(params):
                    sigma = 1e-1
                    x0 = params[0]
                    pred = 1+params[1]*np.log(xvalues/x0)+params[2]*np.log(xvalues/x0)**2
                    residuals = (yvalues - pred)/sigma
                    return np.dot(residuals,residuals)
                params0 = np.array([7000,1,-.5])
                result = scipy.optimize.minimize(chisq, params0, options={'maxiter':10000},
                    method='SLSQP', bounds=((1,None),(None,None),(None,None)))
                # save fit results
                results[i,:] = result.x
                chisqs[i] = result.fun
                if not result.success or result.x[0] == 1:
                    print 'failed on %s-%d: %s' % (title, row[0], results[i])
        else:
            # construct a matrix for the model A + B*Log(x) + C*(Log(x))^2
            xmatrix = np.ones((npoints, nparams))
            xmatrix[:,1] = np.log(xvalues)
            xmatrix[:,2] = np.log(xvalues)**2
            regr = linear_model.LinearRegression(fit_intercept=False)
            regr.fit(xmatrix, yvalues)
            a,b,c = regr.coef_
            # transform fit parameters to "physical" params
            x0 = np.exp(-0.5*(b-np.sqrt(b*b+4*(1-a)*c))/c)
            a1 = b + 2*c*np.log(x0)
            a2 = c
            # save fit results
            chisqs[i] = np.nansum((regr.predict(xmatrix)-yvalues)**2)
            results[i,:] = [x0, a1, a2]

    if args.verbose:
        print 'Mean fitted params (x0, a1, a2): (%.4g, %.4g, %.4g)' % tuple(np.nanmean(results, axis=0))
        print 'Mean chisq: %.4g' % np.mean(chisqs)
    else:
        print '%s %.4g %.4g %.4g %.4g' % tuple([title, np.mean(chisqs)] + np.nanmean(results, axis=0).tolist())

    # save results to file
    output = '%s-%s' % (args.output, '-'.join((args.input).split('-')[1:]))
    if args.verbose:
        print 'Saving results to: %s' % output
    np.savetxt(output, results)

    if args.skip_plots:
        return 0

    if args.verbose:
        print 'Creating fit summary plot...'

    # save results summary plot
    fig = plt.figure(figsize=(12,12))

    ax1 = plt.subplot2grid((4,3), (0,0), colspan=3)
    ax2 = plt.subplot2grid((4,3), (1,0))
    ax3 = plt.subplot2grid((4,3), (2,1))
    ax4 = plt.subplot2grid((4,3), (3,2))

    ax5 = plt.subplot2grid((4,3), (2,0))
    ax6 = plt.subplot2grid((4,3), (3,0))
    ax7 = plt.subplot2grid((4,3), (3,1))

    ax8 = plt.subplot2grid((4,3), (1,2))

    # compare the raw throughput corrections with fit results
    plt.sca(ax1)
    # randomly select example fits to plot
    np.random.seed(args.seed)
    for params in np.random.permutation(results)[:args.nexamples]:
        yvalues = 1 + params[1]*np.log(xvalues/params[0]) + params[2]*(np.log(xvalues/params[0]))**2
        plt.plot(xvalues, yvalues, c='black', alpha=.2, lw=.05)
    # shade max-min region of the raw throughput correction vectors
    plt.fill_between(xvalues, data[:,nidtokens:].min(axis=0),data[:,nidtokens:].max(axis=0), alpha=.5, lw=0)
    # manually construct a legend for this plot
    blue_patch = mpatches.Patch(color='blue', alpha=.5, label=r'Prediction Coverage')
    black_line = mlines.Line2D([], [], color='black', alpha=.5, label=r'$1 + a_1 \log(x/x_0) + a_2 (\log(x/x_0))^2 $')
    plt.legend(handles=[blue_patch, black_line], loc=2)
    # add labels and set ranges
    plt.xlabel(r'Wavelength $(\AA)$')
    plt.ylabel('Throughput Correction')
    plt.ylim([0, 3])
    plt.xlim([3500, 10500])
    plt.title(title)
    plt.grid()

    # plot the fit parameter distributions
    def plot_param_dist(params, binspec, color, xlabel):
        xmin, xmax, nxbins = binspec
        plt.hist(params, bins=np.linspace(xmin, xmax, nxbins+1), facecolor=color, alpha=.5)
        plt.ylabel('Counts')
        plt.xlabel(xlabel)
        plt.xlim([xmin, xmax])
        plt.grid()
        textstr = '$\mathrm{mean}=%.2f$\n$\mathrm{median}=%.2f$\n$\mathrm{std}=%.2f$' % (np.nanmean(params), np.nanmedian(params), np.nanstd(params))
        props = dict(boxstyle='round', facecolor='white')
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right', bbox=props)

    plt.sca(ax2)
    plot_param_dist(results[:,0], (3500,7000,50), 'blue', r'$x_0$')

    plt.sca(ax3)
    plot_param_dist(results[:,1], (-0.55,2.5,50), 'green', r'$a_1$')

    plt.sca(ax4)
    plot_param_dist(results[:,2], (-1,0.5,50), 'red', r'$a_2$')

    # plot the fit parameter distributions
    def plot_param_scatter(x, y, xlim, ylim, xlabel, ylabel):
        plt.plot(x, y, '+', ms=5)
        plt.ylabel(ylabel)
        plt.xlabel(xlabel)
        plt.xlim(xlim)
        plt.ylim(ylim)
        plt.grid()
        # calculate correlation coefficient
        corr =  np.corrcoef(x,y)
        rho = corr[0,1]
        # add text box
        textstr = r'$\rho=%.2f$' % rho
        props = dict(boxstyle='round', facecolor='white')
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right', bbox=props)

    plt.sca(ax5)
    plot_param_scatter(results[:,0], results[:,1], (3500,7000), (-0.55,2.5), r'$x_0$', r'$a_1$')

    plt.sca(ax6)
    plot_param_scatter(results[:,0], results[:,2], (3500,7000), (-1,0.5), r'$x_0$', r'$a_2$')

    plt.sca(ax7)
    plot_param_scatter(results[:,1], results[:,2], (-0.55,2.5), (-1,0.5), r'$a_1$', r'$a_2$')

    plt.sca(ax8)
    ax8.axis('off')

    if args.exp_info:
        import json
        expinfo_filename = '%s-%s.json' % (args.exp_info, title)
        with open(expinfo_filename) as jsondata:
            expinfo = json.load(jsondata)
        keyfmt_pairs = [
            ('design_alt', '%.2f'),
            ('mean_alt', '%.2f'),
            ('design_ha', '%.2f'),
            ('mean_ha', '%.2f'),
            ('SEEING50', '%.2f'),
            ('mean_psf_fwhm', '%.2f')
        ]
        textstr = ''
        textstr += 'n_entries: %d\n' % nentries
        textstr += 'mean_chisq: %.4g\n' % np.mean(chisqs)
        textstr += '\n'.join([('%s: '+fmt) % (key, expinfo[key]) for key,fmt in keyfmt_pairs])
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right')

    plot_name = output[:-4]+'.png'
    if args.verbose:
        print 'Saving fit summary figure to file: %s' % plot_name
    fig.savefig(plot_name, bbox_inches='tight')

if __name__ == '__main__':
    main()
<commit_msg>add nexp to extra plot info<commit_after>#!/usr/bin/env python
"""
Fits tabulated throughput corrections to a model
"""
import argparse

import numpy as np

import matplotlib as mpl
mpl.use('Agg')
mpl.rcParams.update({'font.size': 10})
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.lines as mlines

from sklearn import linear_model
import scipy.optimize

def main():
    # parse command-line arguments
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--verbose", action="store_true",
        help="print verbose output")
    parser.add_argument("-o", "--output", type=str, default=None,
        help="output file base name")
    parser.add_argument("-i", "--input", type=str, default=None,
        help="required input file")
    parser.add_argument("--nexamples", type=int, default=-1,
        help="number of example fits to plot, (array slice 'end' value)")
    parser.add_argument("--seed", type=int, default=2810,
        help="random seed for example choices")
    parser.add_argument("--scipy", action="store_true",
        help="use scipy.optimize instead of sklearn")
    parser.add_argument("--exp-info", type=str, default=None,
        help="add exp info")
    parser.add_argument("--skip-plots", action="store_true",
        help="don't make plots")
    args = parser.parse_args()

    if args.verbose:
        print 'Reading file: %s' % args.input

    # for now, construct the title from the input filename
    title = '-'.join((args.input).split('-')[1:])[:-4]

    # The input data is text file where each line coresponds to
    # a target's throughput correction vector
    data = np.loadtxt(args.input, ndmin=2)

    try:
        nentries, ntokens = data.shape
    except ValueError:
        print data.shape, args.input

    # the first two columns are xfocal and yfocal positions of the target
    nidtokens = 3
    # the rest are the tabulated throughput correction values
    npoints = ntokens - nidtokens

    # the throughput correction vectors span the range 3500A to 10500A
    xvalues = np.linspace(3500, 10500, npoints, endpoint=True)

    nparams = 3
    results = np.empty((nentries, nparams))
    chisqs = np.empty(nentries)

    # fit individual entries
    for i,row in enumerate(data):
        yvalues = row[nidtokens:]
        if args.scipy:
            # chisq function for our model
            def chisq(params):
                sigma = 1e-1
                x0 = np.exp(params[0])
                pred = 1+params[1]*np.log(xvalues/x0)+params[2]*np.log(xvalues/x0)**2
                residuals = (yvalues - pred)/sigma
                return np.dot(residuals,residuals)
            params0 = np.array([np.log(7000),1,-.5])
            result = scipy.optimize.minimize(chisq, params0, options={'maxiter':10000},
                method='Nelder-Mead')
            # save fit results
            results[i,:] = result.x
            results[i,0] = np.exp(result.x[0])
            chisqs[i] = result.fun
            if not result.success:
                # chisq function for our model
                def chisq(params):
                    sigma = 1e-1
                    x0 = params[0]
                    pred = 1+params[1]*np.log(xvalues/x0)+params[2]*np.log(xvalues/x0)**2
                    residuals = (yvalues - pred)/sigma
                    return np.dot(residuals,residuals)
                params0 = np.array([7000,1,-.5])
                result = scipy.optimize.minimize(chisq, params0, options={'maxiter':10000},
                    method='SLSQP', bounds=((1,None),(None,None),(None,None)))
                # save fit results
                results[i,:] = result.x
                chisqs[i] = result.fun
                if not result.success or result.x[0] == 1:
                    print 'failed on %s-%d: %s' % (title, row[0], results[i])
        else:
            # construct a matrix for the model A + B*Log(x) + C*(Log(x))^2
            xmatrix = np.ones((npoints, nparams))
            xmatrix[:,1] = np.log(xvalues)
            xmatrix[:,2] = np.log(xvalues)**2
            regr = linear_model.LinearRegression(fit_intercept=False)
            regr.fit(xmatrix, yvalues)
            a,b,c = regr.coef_
            # transform fit parameters to "physical" params
            x0 = np.exp(-0.5*(b-np.sqrt(b*b+4*(1-a)*c))/c)
            a1 = b + 2*c*np.log(x0)
            a2 = c
            # save fit results
            chisqs[i] = np.nansum((regr.predict(xmatrix)-yvalues)**2)
            results[i,:] = [x0, a1, a2]

    if args.verbose:
        print 'Mean fitted params (x0, a1, a2): (%.4g, %.4g, %.4g)' % tuple(np.nanmean(results, axis=0))
        print 'Mean chisq: %.4g' % np.mean(chisqs)
    else:
        print '%s %.4g %.4g %.4g %.4g' % tuple([title, np.mean(chisqs)] + np.nanmean(results, axis=0).tolist())

    # save results to file
    output = '%s-%s' % (args.output, '-'.join((args.input).split('-')[1:]))
    if args.verbose:
        print 'Saving results to: %s' % output
    np.savetxt(output, results)

    if args.skip_plots:
        return 0

    if args.verbose:
        print 'Creating fit summary plot...'

    # save results summary plot
    fig = plt.figure(figsize=(12,12))

    ax1 = plt.subplot2grid((4,3), (0,0), colspan=3)
    ax2 = plt.subplot2grid((4,3), (1,0))
    ax3 = plt.subplot2grid((4,3), (2,1))
    ax4 = plt.subplot2grid((4,3), (3,2))

    ax5 = plt.subplot2grid((4,3), (2,0))
    ax6 = plt.subplot2grid((4,3), (3,0))
    ax7 = plt.subplot2grid((4,3), (3,1))

    ax8 = plt.subplot2grid((4,3), (1,2))

    # compare the raw throughput corrections with fit results
    plt.sca(ax1)
    # randomly select example fits to plot
    np.random.seed(args.seed)
    for params in np.random.permutation(results)[:args.nexamples]:
        yvalues = 1 + params[1]*np.log(xvalues/params[0]) + params[2]*(np.log(xvalues/params[0]))**2
        plt.plot(xvalues, yvalues, c='black', alpha=.2, lw=.05)
    # shade max-min region of the raw throughput correction vectors
    plt.fill_between(xvalues, data[:,nidtokens:].min(axis=0),data[:,nidtokens:].max(axis=0), alpha=.5, lw=0)
    # manually construct a legend for this plot
    blue_patch = mpatches.Patch(color='blue', alpha=.5, label=r'Prediction Coverage')
    black_line = mlines.Line2D([], [], color='black', alpha=.5, label=r'$1 + a_1 \log(x/x_0) + a_2 (\log(x/x_0))^2 $')
    plt.legend(handles=[blue_patch, black_line], loc=2)
    # add labels and set ranges
    plt.xlabel(r'Wavelength $(\AA)$')
    plt.ylabel('Throughput Correction')
    plt.ylim([0, 3])
    plt.xlim([3500, 10500])
    plt.title(title)
    plt.grid()

    # plot the fit parameter distributions
    def plot_param_dist(params, binspec, color, xlabel):
        xmin, xmax, nxbins = binspec
        plt.hist(params, bins=np.linspace(xmin, xmax, nxbins+1), facecolor=color, alpha=.5)
        plt.ylabel('Counts')
        plt.xlabel(xlabel)
        plt.xlim([xmin, xmax])
        plt.grid()
        textstr = '$\mathrm{mean}=%.2f$\n$\mathrm{median}=%.2f$\n$\mathrm{std}=%.2f$' % (np.nanmean(params), np.nanmedian(params), np.nanstd(params))
        props = dict(boxstyle='round', facecolor='white')
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right', bbox=props)

    plt.sca(ax2)
    plot_param_dist(results[:,0], (3500,7000,50), 'blue', r'$x_0$')

    plt.sca(ax3)
    plot_param_dist(results[:,1], (-0.55,2.5,50), 'green', r'$a_1$')

    plt.sca(ax4)
    plot_param_dist(results[:,2], (-1,0.5,50), 'red', r'$a_2$')

    # plot the fit parameter distributions
    def plot_param_scatter(x, y, xlim, ylim, xlabel, ylabel):
        plt.plot(x, y, '+', ms=5)
        plt.ylabel(ylabel)
        plt.xlabel(xlabel)
        plt.xlim(xlim)
        plt.ylim(ylim)
        plt.grid()
        # calculate correlation coefficient
        corr =  np.corrcoef(x,y)
        rho = corr[0,1]
        # add text box
        textstr = r'$\rho=%.2f$' % rho
        props = dict(boxstyle='round', facecolor='white')
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right', bbox=props)

    plt.sca(ax5)
    plot_param_scatter(results[:,0], results[:,1], (3500,7000), (-0.55,2.5), r'$x_0$', r'$a_1$')

    plt.sca(ax6)
    plot_param_scatter(results[:,0], results[:,2], (3500,7000), (-1,0.5), r'$x_0$', r'$a_2$')

    plt.sca(ax7)
    plot_param_scatter(results[:,1], results[:,2], (-0.55,2.5), (-1,0.5), r'$a_1$', r'$a_2$')

    plt.sca(ax8)
    ax8.axis('off')

    if args.exp_info:
        import json
        expinfo_filename = '%s-%s.json' % (args.exp_info, title)
        with open(expinfo_filename) as jsondata:
            expinfo = json.load(jsondata)
        keyfmt_pairs = [
            ('design_alt', '%.2f'),
            ('mean_alt', '%.2f'),
            ('design_ha', '%.2f'),
            ('mean_ha', '%.2f'),
            ('SEEING50', '%.2f'),
            ('mean_psf_fwhm', '%.2f')
        ]
        textstr = ''
        textstr += 'n_entries: %d\n' % nentries
        textstr += 'mean_chisq: %.4g\n' % np.mean(chisqs)
        textstr += 'nexp: %d\n' % len(expinfo['exposures'])
        textstr += '\n'.join([('%s: '+fmt) % (key, expinfo[key]) for key,fmt in keyfmt_pairs])
        plt.text(0.95, 0.95, textstr, transform=plt.gca().transAxes, va='top', ha='right')

    plot_name = output[:-4]+'.png'
    if args.verbose:
        print 'Saving fit summary figure to file: %s' % plot_name
    fig.savefig(plot_name, bbox_inches='tight')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""

Actor Runtime
=============

The runtime provides the functions to create an actor from a behavior,
and to report an error.  Subclass from `AbstractRuntime` to create a
runtime class.

`SimpleRuntime.create` just creates the actor, and
`SimpleRuntime.error` prints the error message to stdout.

The behaviors are defined as::

    @behavior
    def my_beh(arg1, ..., self, msg):
        ...

where `arg1, ...` are 0 or more arguments to be passed at creation
time. `self` refers to the actor in context, and `msg` is the message
being passed.

To create or change behavior, `self` provides the methods `create` and
`become`. To send a message to an actor `x` use `x << msg`.  For
example::

    @behavior
    def my_beh(n, self, msg):
        if n > 0:
            another = self.create(my_beh, n-1)
            another << 'a message'
        self.become(sink_beh)

    @behavior
    def sink_beh(self, msg):
        pass

"""

from functools import wraps
import pprint

from .singleton import Singleton
from .eventloop import EventLoop

class AbstractRuntime(object, metaclass=Singleton):

    def create(self, behavior, *args):
        raise NotImplementedError()

    def error(self, message):
        raise NotImplementedError()

class SimpleRuntime(AbstractRuntime):

    def create(self, behavior, *args):
        return Actor(self, behavior, *args)

    def error(self, message):
        print('ERROR: {0}'.format(pprint.pformat(message)))

class Runtime(SimpleRuntime):

    def __init__(self):
        super().__init__()
        self.evloop = EventLoop()
        self.evloop.run_in_thread()

class Actor(object):

    def __init__(self, runtime, behavior, *args):
        self.runtime = runtime
        self.become(behavior, *args)
        self.ev_loop = EventLoop()

    def become(self, behavior, *args):
        self._behavior = behavior(*args)

    def send(self, msg):
        def event():
            self._behavior(self, msg)
        self.ev_loop.schedule((event, self.error))

    def create(self, behavior, *args):
        return self.runtime.create(behavior, *args)

    def error(self, message):
        self.runtime.error(message)

    def __lshift__(self, msg):
        """Syntax sugar for sending a message.

        Use as `x << msg`.

        """
        self.send(msg)

def behavior(f):
    """Decorator for declaring a function as behavior.

    Use as::

        @behavior
        def fun(x, y, self, msg):
            ...

    And create or become this behavior by passing the two arguments
    `x` and `y`.

    """
    @wraps(f)
    def wrapper(*args):
        def _f(self, msg):
            f(*(args + (self, msg)))
        return _f
    return wrapper

<commit_msg>Add another syntax for sending messages<commit_after>"""

Actor Runtime
=============

The runtime provides the functions to create an actor from a behavior,
and to report an error.  Subclass from `AbstractRuntime` to create a
runtime class.

`SimpleRuntime.create` just creates the actor, and
`SimpleRuntime.error` prints the error message to stdout.

The behaviors are defined as::

    @behavior
    def my_beh(arg1, ..., self, msg):
        ...

where `arg1, ...` are 0 or more arguments to be passed at creation
time. `self` refers to the actor in context, and `msg` is the message
being passed.

To create or change behavior, `self` provides the methods `create` and
`become`. To send a message to an actor `x` use `x << msg`.  For
example::

    @behavior
    def my_beh(n, self, msg):
        if n > 0:
            another = self.create(my_beh, n-1)
            another << 'a message'
        self.become(sink_beh)

    @behavior
    def sink_beh(self, msg):
        pass

"""

from functools import wraps
import pprint

from .singleton import Singleton
from .eventloop import EventLoop

class AbstractRuntime(object, metaclass=Singleton):

    def create(self, behavior, *args):
        raise NotImplementedError()

    def error(self, message):
        raise NotImplementedError()

class SimpleRuntime(AbstractRuntime):

    def create(self, behavior, *args):
        return Actor(self, behavior, *args)

    def error(self, message):
        print('ERROR: {0}'.format(pprint.pformat(message)))

class Runtime(SimpleRuntime):

    def __init__(self):
        super().__init__()
        self.evloop = EventLoop()
        self.evloop.run_in_thread()

class Actor(object):

    def __init__(self, runtime, behavior, *args):
        self.runtime = runtime
        self.become(behavior, *args)
        self.ev_loop = EventLoop()

    def become(self, behavior, *args):
        self._behavior = behavior(*args)

    def send(self, msg):
        def event():
            self._behavior(self, msg)
        self.ev_loop.schedule((event, self.error))

    def create(self, behavior, *args):
        return self.runtime.create(behavior, *args)

    def error(self, message):
        self.runtime.error(message)

    def __lshift__(self, msg):
        """Syntax sugar for sending a message.

        Use as `x << msg`.

        """
        self.send(msg)

    def __call__(self, msg):
        self.send(msg)

def behavior(f):
    """Decorator for declaring a function as behavior.

    Use as::

        @behavior
        def fun(x, y, self, msg):
            ...

    And create or become this behavior by passing the two arguments
    `x` and `y`.

    """
    @wraps(f)
    def wrapper(*args):
        def _f(self, msg):
            f(*(args + (self, msg)))
        return _f
    return wrapper

<|endoftext|>


<commit_before>from __future__ import division

import sys
import logging
import math
from datetime import timedelta
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

import numpy as NP
import pylab as PL
import matplotlib.dates as mdates
import pandas as PD

from pyglow.pyglow import Point

from date import dt_parser

RED = NP.array([255, 204, 204]) / 255
ORANGE = NP.array([255, 204, 154]) / 255
YELLOW = NP.array([255, 255, 204]) / 255
GREEN = NP.array([204, 255, 204]) / 255
BLUE = NP.array([189, 227, 255]) / 255

GREEN2 = NP.array([52, 253, 47]) / 255
YELLOW2 = NP.array([254, 219, 49]) / 255
RED2 = NP.array([252, 38, 28]) / 255

KP_COLORS_DISPLAY = {0: 'w',
                     1: 'w',
                     2: 'w',
                     3: 'w',
                     4: 'w',
                     5: BLUE,
                     6: GREEN,
                     7: YELLOW,
                     8: ORANGE,
                     9: RED}

KP_COLORS_DOCUMENT = {0: GREEN2,
                      1: GREEN2,
                      2: GREEN2,
                      3: GREEN2,
                      4: YELLOW2,
                      5: RED2,
                      6: RED2,
                      7: RED2,
                      8: RED2,
                      9: RED2}

KP_HATCH_DOCUMENT = {0: '',
                     1: '',
                     2: '',
                     3: '',
                     4: '/',
                     5: '\\',
                     6: '\\',
                     7: '\\',
                     8: '\\',
                     9: '\\'}

"""
Mapping between style identifier and the tuple of color and hatch
mappings.
"""
STYLE_MAP = {'display':  (KP_COLORS_DISPLAY,
                          False),
             'document': (KP_COLORS_DOCUMENT,
                          KP_HATCH_DOCUMENT)}

def plot_indices(d1,
                 d2,
                 fig=None,
                 style='display',
                 bar_lw=0.2,
                 dst_lw=1.75,
                 edgecolor=(0.1, 0.1, 0.1)):
    """
    """
    # gather Kp
    kp = []
    kp_dt = []
    for dt in PD.date_range(d1, d2, freq='3H'):
        point = Point(dt, 0, 0, 0)
        kp.append(point.kp)
        kp_dt.append(dt)
    # gather Dst
    dst = []
    dst_dt = []
    for dt in PD.date_range(d1, d2, freq='1H'):
        point = Point(dt, 0, 0, 0)
        dst.append(point.dst)
        dst_dt.append(dt)
    # create plot
    N_days = (dst_dt[-1] - dst_dt[0]).total_seconds() / 60 / 60 / 24
    if fig is None:
        fig = PL.figure(figsize=(11 * N_days / 6, 5))
    kp_colors, kp_hatch = STYLE_MAP[style]
    # Kp subplot
    ax1 = PL.subplot(111)
    left = NP.array(kp_dt) - timedelta(hours=1.5)
    width = 3 / 24
    height = kp
    color = [kp_colors[int(math.floor(x))] for x in kp]
    hatch = [kp_hatch[int(math.floor(x))] if kp_hatch else None for x in kp]
    bars = PL.bar(left,
                  height,
                  width=width,
                  color=color,
                  linewidth=bar_lw,
                  edgecolor=edgecolor)
    for bar, hatch_i in zip(bars, hatch):
        bar.set_hatch(hatch_i)
    ax1.xaxis_date()
    PL.ylim(0, 9)
    PL.xlabel('UT')
    PL.ylabel('3-hour Kp index')
    # Dst subplot
    ax2 = ax1.twinx()
    PL.plot_date(dst_dt,
                 dst,
                 lw=dst_lw,
                 marker=None,
                 ls='-')
    PL.ylabel('Hourly DST [nT]')
    PL.title('GFZ $K_p$ and Dst Indices {:%Y-%m-%d %H:%M} to '
             '{:%Y-%m-%d %H:%M}'.format(dst_dt[0],
                                        dst_dt[-1]))
    return fig

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Create plot of the Kp and Dst indices.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('pdf_fname',
                        type=str,
                        help='file to store plot')
    parser.add_argument('d1',
                        type=dt_parser,
                        help='start date/time')
    parser.add_argument('d2',
                        type=dt_parser,
                        help='end date/time')
    parser.add_argument('--style',
                        '-s',
                        type=str,
                        choices=sorted(STYLE_MAP),
                        default='display',
                        help='plot style (display is more colorful and meant for screen display whereas document is high contrast and uses hatches for both color and black-white interpretation and meant for use in publication)')
    args = parser.parse_args(argv[1:])

    plot_indices(args.d1,
                 args.d2,
                 style=args.style)

    PL.savefig(args.pdf_fname,
               bbox_inches='tight')

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<commit_msg>Added optional stats option to plot_indices.<commit_after>from __future__ import division

import sys
import logging
import math
from datetime import timedelta
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from collections import namedtuple

import numpy as NP
import pylab as PL
import matplotlib.dates as mdates
import pandas as PD

from pyglow.pyglow import Point

from date import dt_parser
from ..stats.stats import Stats

RED = NP.array([255, 204, 204]) / 255
ORANGE = NP.array([255, 204, 154]) / 255
YELLOW = NP.array([255, 255, 204]) / 255
GREEN = NP.array([204, 255, 204]) / 255
BLUE = NP.array([189, 227, 255]) / 255

GREEN2 = NP.array([52, 253, 47]) / 255
YELLOW2 = NP.array([254, 219, 49]) / 255
RED2 = NP.array([252, 38, 28]) / 255

KP_COLORS_DISPLAY = {0: 'w',
                     1: 'w',
                     2: 'w',
                     3: 'w',
                     4: 'w',
                     5: BLUE,
                     6: GREEN,
                     7: YELLOW,
                     8: ORANGE,
                     9: RED}

KP_COLORS_DOCUMENT = {0: GREEN2,
                      1: GREEN2,
                      2: GREEN2,
                      3: GREEN2,
                      4: YELLOW2,
                      5: RED2,
                      6: RED2,
                      7: RED2,
                      8: RED2,
                      9: RED2}

KP_HATCH_DOCUMENT = {0: '',
                     1: '',
                     2: '',
                     3: '',
                     4: '/',
                     5: '\\',
                     6: '\\',
                     7: '\\',
                     8: '\\',
                     9: '\\'}

"""
Mapping between style identifier and the tuple of color and hatch
mappings.
"""
STYLE_MAP = {'display':  (KP_COLORS_DISPLAY,
                          False),
             'document': (KP_COLORS_DOCUMENT,
                          KP_HATCH_DOCUMENT)}

class IndexStats(namedtuple('IndiciesStats', 'kp dst')):
    pass

def plot_indices(d1,
                 d2,
                 fig=None,
                 style='display',
                 bar_lw=0.2,
                 dst_lw=1.75,
                 edgecolor=(0.1, 0.1, 0.1),
                 stats=False):
    """
    """
    # gather Kp
    kp = []
    kp_dt = []
    for dt in PD.date_range(d1, d2, freq='3H'):
        point = Point(dt, 0, 0, 0)
        kp.append(point.kp)
        kp_dt.append(dt)
    # gather Dst
    dst = []
    dst_dt = []
    for dt in PD.date_range(d1, d2, freq='1H'):
        point = Point(dt, 0, 0, 0)
        dst.append(point.dst)
        dst_dt.append(dt)
    # create plot
    N_days = (dst_dt[-1] - dst_dt[0]).total_seconds() / 60 / 60 / 24
    if fig is None:
        fig = PL.figure(figsize=(11 * N_days / 6, 5))
    kp_colors, kp_hatch = STYLE_MAP[style]
    # Kp subplot
    ax1 = PL.subplot(111)
    left = NP.array(kp_dt) - timedelta(hours=1.5)
    width = 3 / 24
    height = kp
    color = [kp_colors[int(math.floor(x))] for x in kp]
    hatch = [kp_hatch[int(math.floor(x))] if kp_hatch else None for x in kp]
    bars = PL.bar(left,
                  height,
                  width=width,
                  color=color,
                  linewidth=bar_lw,
                  edgecolor=edgecolor)
    for bar, hatch_i in zip(bars, hatch):
        bar.set_hatch(hatch_i)
    ax1.xaxis_date()
    PL.ylim(0, 9)
    PL.xlabel('UT')
    PL.ylabel('3-hour Kp index')
    # Dst subplot
    ax2 = ax1.twinx()
    PL.plot_date(dst_dt,
                 dst,
                 lw=dst_lw,
                 marker=None,
                 ls='-')
    PL.ylabel('Hourly DST [nT]')
    PL.title('GFZ $K_p$ and Dst Indices {:%Y-%m-%d %H:%M} to '
             '{:%Y-%m-%d %H:%M}'.format(dst_dt[0],
                                        dst_dt[-1]))
    #return fig
    if stats:
        index_stats = IndexStats(Stats(*kp),
                                 Stats(*dst))
        return index_stats, fig
    else:
        return fig

def main(argv=None):
    if argv is None:
        argv = sys.argv

    parser = ArgumentParser('Create plot of the Kp and Dst indices.',
                            formatter_class=ArgumentDefaultsHelpFormatter)
    parser.add_argument('pdf_fname',
                        type=str,
                        help='file to store plot')
    parser.add_argument('d1',
                        type=dt_parser,
                        help='start date/time')
    parser.add_argument('d2',
                        type=dt_parser,
                        help='end date/time')
    parser.add_argument('--style',
                        '-s',
                        type=str,
                        choices=sorted(STYLE_MAP),
                        default='display',
                        help='plot style (display is more colorful and meant for screen display whereas document is high contrast and uses hatches for both color and black-white interpretation and meant for use in publication)')
    args = parser.parse_args(argv[1:])

    plot_indices(args.d1,
                 args.d2,
                 style=args.style)

    PL.savefig(args.pdf_fname,
               bbox_inches='tight')

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    sys.exit(main())
<|endoftext|>


<commit_before>"""Cucumber json output formatter."""

import json
import math
import os
import time
from typing import Any, Dict, List

from _pytest.config import Config
from _pytest.config.argparsing import Parser
from _pytest.reports import TestReport
from _pytest.terminal import TerminalReporter

def add_options(parser: Parser) -> None:
    """Add pytest-bdd options."""
    group = parser.getgroup("bdd", "Cucumber JSON")
    group.addoption(
        "--cucumberjson",
        "--cucumber-json",
        action="store",
        dest="cucumber_json_path",
        metavar="path",
        default=None,
        help="create cucumber json style report file at given path.",
    )

def configure(config: Config) -> None:
    cucumber_json_path = config.option.cucumber_json_path
    # prevent opening json log on worker nodes (xdist)
    if cucumber_json_path and not hasattr(config, "workerinput"):
        config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path)
        config.pluginmanager.register(config._bddcucumberjson)

def unconfigure(config: Config) -> None:
    xml = getattr(config, "_bddcucumberjson", None)
    if xml is not None:
        del config._bddcucumberjson
        config.pluginmanager.unregister(xml)

class LogBDDCucumberJSON:

    """Logging plugin for cucumber like json output."""

    def __init__(self, logfile: str) -> None:
        logfile = os.path.expanduser(os.path.expandvars(logfile))
        self.logfile = os.path.normpath(os.path.abspath(logfile))
        self.features = {}

    def append(self, obj):
        self.features[-1].append(obj)

    def _get_result(self, step: Dict[str, Any], report: TestReport, error_message: bool = False) -> Dict[str, Any]:
        """Get scenario test run result.

        :param step: `Step` step we get result for
        :param report: pytest `Report` object
        :return: `dict` in form {"status": "<passed|failed|skipped>", ["error_message": "<error_message>"]}
        """
        result = {}
        if report.passed or not step["failed"]:  # ignore setup/teardown
            result = {"status": "passed"}
        elif report.failed and step["failed"]:
            result = {"status": "failed", "error_message": str(report.longrepr) if error_message else ""}
        elif report.skipped:
            result = {"status": "skipped"}
        result["duration"] = int(math.floor((10 ** 9) * step["duration"]))  # nanosec
        return result

    def _serialize_tags(self, item: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Serialize item's tags.

        :param item: json-serialized `Scenario` or `Feature`.
        :return: `list` of `dict` in the form of:
            [
                {
                    "name": "<tag>",
                    "line": 2,
                }
            ]
        """
        return [{"name": tag, "line": item["line_number"] - 1} for tag in item["tags"]]

    def pytest_runtest_logreport(self, report: TestReport) -> None:
        try:
            scenario = report.scenario
        except AttributeError:
            # skip reporting for non-bdd tests
            return

        if not scenario["steps"] or report.when != "call":
            # skip if there isn't a result or scenario has no steps
            return

        def stepmap(step: Dict[str, Any]) -> Dict[str, Any]:
            error_message = False
            if step["failed"] and not scenario.setdefault("failed", False):
                scenario["failed"] = True
                error_message = True

            step_name = step["name"]

            return {
                "keyword": step["keyword"],
                "name": step_name,
                "line": step["line_number"],
                "match": {"location": ""},
                "result": self._get_result(step, report, error_message),
            }

        if scenario["feature"]["filename"] not in self.features:
            self.features[scenario["feature"]["filename"]] = {
                "keyword": "Feature",
                "uri": scenario["feature"]["rel_filename"],
                "name": scenario["feature"]["name"] or scenario["feature"]["rel_filename"],
                "id": scenario["feature"]["rel_filename"].lower().replace(" ", "-"),
                "line": scenario["feature"]["line_number"],
                "description": scenario["feature"]["description"],
                "tags": self._serialize_tags(scenario["feature"]),
                "elements": [],
            }

        self.features[scenario["feature"]["filename"]]["elements"].append(
            {
                "keyword": "Scenario",
                "id": report.item["name"],
                "name": scenario["name"],
                "line": scenario["line_number"],
                "description": "",
                "tags": self._serialize_tags(scenario),
                "type": "scenario",
                "steps": [stepmap(step) for step in scenario["steps"]],
            }
        )

    def pytest_sessionstart(self) -> None:
        self.suite_start_time = time.time()

    def pytest_sessionfinish(self) -> None:
        with open(self.logfile, "w", encoding="utf-8") as logfile:
            logfile.write(json.dumps(list(self.features.values())))

    def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None:
        terminalreporter.write_sep("-", "generated json file: %s" % (self.logfile))
<commit_msg>Fix annotations<commit_after>"""Cucumber json output formatter."""

import json
import math
import os
import time
from typing import TYPE_CHECKING, Any, Dict, List

if TYPE_CHECKING:
    from _pytest.config import Config
    from _pytest.config.argparsing import Parser
    from _pytest.reports import TestReport
    from _pytest.terminal import TerminalReporter

def add_options(parser: "Parser") -> None:
    """Add pytest-bdd options."""
    group = parser.getgroup("bdd", "Cucumber JSON")
    group.addoption(
        "--cucumberjson",
        "--cucumber-json",
        action="store",
        dest="cucumber_json_path",
        metavar="path",
        default=None,
        help="create cucumber json style report file at given path.",
    )

def configure(config: "Config") -> None:
    cucumber_json_path = config.option.cucumber_json_path
    # prevent opening json log on worker nodes (xdist)
    if cucumber_json_path and not hasattr(config, "workerinput"):
        config._bddcucumberjson = LogBDDCucumberJSON(cucumber_json_path)
        config.pluginmanager.register(config._bddcucumberjson)

def unconfigure(config: "Config") -> None:
    xml = getattr(config, "_bddcucumberjson", None)
    if xml is not None:
        del config._bddcucumberjson
        config.pluginmanager.unregister(xml)

class LogBDDCucumberJSON:

    """Logging plugin for cucumber like json output."""

    def __init__(self, logfile: str) -> None:
        logfile = os.path.expanduser(os.path.expandvars(logfile))
        self.logfile = os.path.normpath(os.path.abspath(logfile))
        self.features = {}

    # TODO: Unused method?
    def append(self, obj):
        self.features[-1].append(obj)

    def _get_result(self, step: Dict[str, Any], report: "TestReport", error_message: bool = False) -> Dict[str, Any]:
        """Get scenario test run result.

        :param step: `Step` step we get result for
        :param report: pytest `Report` object
        :return: `dict` in form {"status": "<passed|failed|skipped>", ["error_message": "<error_message>"]}
        """
        result = {}
        if report.passed or not step["failed"]:  # ignore setup/teardown
            result = {"status": "passed"}
        elif report.failed and step["failed"]:
            result = {"status": "failed", "error_message": str(report.longrepr) if error_message else ""}
        elif report.skipped:
            result = {"status": "skipped"}
        result["duration"] = int(math.floor((10 ** 9) * step["duration"]))  # nanosec
        return result

    def _serialize_tags(self, item: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Serialize item's tags.

        :param item: json-serialized `Scenario` or `Feature`.
        :return: `list` of `dict` in the form of:
            [
                {
                    "name": "<tag>",
                    "line": 2,
                }
            ]
        """
        return [{"name": tag, "line": item["line_number"] - 1} for tag in item["tags"]]

    def pytest_runtest_logreport(self, report: "TestReport") -> None:
        try:
            scenario = report.scenario
        except AttributeError:
            # skip reporting for non-bdd tests
            return

        if not scenario["steps"] or report.when != "call":
            # skip if there isn't a result or scenario has no steps
            return

        def stepmap(step: Dict[str, Any]) -> Dict[str, Any]:
            error_message = False
            if step["failed"] and not scenario.setdefault("failed", False):
                scenario["failed"] = True
                error_message = True

            step_name = step["name"]

            return {
                "keyword": step["keyword"],
                "name": step_name,
                "line": step["line_number"],
                "match": {"location": ""},
                "result": self._get_result(step, report, error_message),
            }

        if scenario["feature"]["filename"] not in self.features:
            self.features[scenario["feature"]["filename"]] = {
                "keyword": "Feature",
                "uri": scenario["feature"]["rel_filename"],
                "name": scenario["feature"]["name"] or scenario["feature"]["rel_filename"],
                "id": scenario["feature"]["rel_filename"].lower().replace(" ", "-"),
                "line": scenario["feature"]["line_number"],
                "description": scenario["feature"]["description"],
                "tags": self._serialize_tags(scenario["feature"]),
                "elements": [],
            }

        self.features[scenario["feature"]["filename"]]["elements"].append(
            {
                "keyword": "Scenario",
                "id": report.item["name"],
                "name": scenario["name"],
                "line": scenario["line_number"],
                "description": "",
                "tags": self._serialize_tags(scenario),
                "type": "scenario",
                "steps": [stepmap(step) for step in scenario["steps"]],
            }
        )

    def pytest_sessionstart(self) -> None:
        self.suite_start_time = time.time()

    def pytest_sessionfinish(self) -> None:
        with open(self.logfile, "w", encoding="utf-8") as logfile:
            logfile.write(json.dumps(list(self.features.values())))

    def pytest_terminal_summary(self, terminalreporter: "TerminalReporter") -> None:
        terminalreporter.write_sep("-", f"generated json file: {self.logfile}")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from rdflib import URIRef, Literal, Namespace
from rulesets.ruleset import RulesetCommon
from nltk.corpus import alpino

ONTOLEX = Namespace("http://www.w3.org/ns/lemon/ontolex#")
LEXINFO = Namespace("http://www.lexinfo.net/ontology/2.0/lexinfo#")
SKOSTHES = Namespace("http://purl.org/iso25964/skos-thes#")
LANGUAGE = "nl"

class Ruleset(RulesetCommon):
	def __init__(self,config,dont_ask=False):
		RulesetCommon.__init__(self,config,dont_ask)
		global ONTOLEX
		global LEXINFO
		global SKOSTHES

		# for now use alpino, we should be able to configure this
		self.worddb = alpino.words()
		self.language = LANGUAGE
		self.lang_id = self.db.languages[LANGUAGE]

	def adjectiveAntonyms(self):
		""" Based on word characteristics. Trying to prefix 'on' to a word. """
		pos_id = self.db.posses["adjective"]
		lexicalEntryIDs = {}

		for lexicalEntryIdentifier in self.g.subjects(LEXINFO.partOfSpeech,LEXINFO.adjective):
			label = self.getLabel(lexicalEntryIdentifier)
			if label[:2] != "on":
				lexicalEntryIDs[lexicalEntryIdentifier] = label

		for lexicalEntryIdentifier in lexicalEntryIDs:
			source_value = lexicalEntryIDs[lexicalEntryIdentifier]
			guess_antonym = "on" + source_value

			if self.userCheck("tegenstelling", source_value, guess_antonym):
				sensecount = self.__countLexicalenses(lexicalEntryIdentifier)
				if sensecount > 1:
					print("multiple senses detected, store manually")
					continue
				elif sensecount == 1:
					# check antonym
					lexicalSenseIdentifier = self.g.value(URIRef(lexicalEntryIdentifier),ONTOLEX.sense,None)
					if (URIRef(lexicalSenseIdentifier),LEXINFO.antonym,None) in self.g:
						print("antonym already detected, moving on")
						continue

				# at this point sensecount is 0, or it is safe to move on
				# we might want to store now, first the guess antonym lexicalEntry, then the sense
				self.db.storeCanonical(guess_antonym,self.lang_id,pos_id)
				self.db.addSense(source_value,pos_id,"lexinfo","antonym",guess_antonym,pos_id)

	def adjectiveMaterialNouns(self):
		""" Get all material nouns, based on being part of material category."""
		materialSenseIDs = self.getLexicalSenseIDsByReference(["http://www.wikidata.org/entity/Q11344"])
		source_pos_id = self.db.posses["noun"]
		target_pos_id = self.db.posses["adjective"]

		# now get all specific values in lexicalEntries
		for mID in materialSenseIDs:
			for senseID in self.g.objects(URIRef(mID),SKOSTHES.narrowerInstantial):
				lexicalEntryID = str(self.g.value(None,ONTOLEX.sense,URIRef(senseID)))
				self.lexicalEntries[lexicalEntryID] = self.getLabel(senseID)

		for lexicalEntryID in self.lexicalEntries:
			label = self.lexicalEntries[lexicalEntryID]
			guess_adjective = self.__getNounStem(label) + "en"

			if guess_adjective in self.worddb:
				if self.userCheck("bijvoegelijk naamwoord", label, guess_adjective):
					self.db.storeCanonical(guess_adjective,self.lang_id,target_pos_id)
					self.db.addSense(label,source_pos_id,"skos","related",guess_adjective,target_pos_id)

	def verbRelatedNouns(self):
		""" example. delen -> deling """
		source_pos_id = self.db.posses["verb"]
		target_pos_id = self.db.posses["noun"]
		lexicalEntryIDs = {}

		for lexicalEntryIdentifier in self.g.subjects(LEXINFO.partOfSpeech,LEXINFO.verb):
			self.lexicalEntries[lexicalEntryIdentifier] = self.getLabel(lexicalEntryIdentifier)

		for lexicalEntryIdentifier in self.lexicalEntries:
			source_value = self.lexicalEntries[lexicalEntryIdentifier]
			guess_noun = source_value[:-2] + "ing"

			# also, perhaps make this configurable as well
			if guess_noun in self.worddb:
				if self.userCheck("gerelateerd", source_value, guess_noun):
					sensecount = self.__countLexicalenses(lexicalEntryIdentifier)
					if sensecount > 1:
						print("multiple senses detected, store manually")
						continue
					elif sensecount == 1:
						# check if relation to term exists
						if self.checkLexicalEntryExists(guess_noun, LEXINFO.noun):
							# bit lazy here, but just playing now
							print("entry found for target, so maybe it is connected already")
							continue

					# at this point sensecount is 0, or it is safe to move on
					self.db.storeCanonical(guess_noun,self.lang_id,target_pos_id)
					self.db.addSense(source_value,source_pos_id,"skos","related",guess_noun,target_pos_id)

	def nounPlurals(self):
		# Finding plurals for nouns without a plural form. """
		self.setProcessableEntries(LEXINFO.noun,LEXINFO.number,LEXINFO.plural)
		source_pos_id = self.db.posses["noun"]
		target_pos_id = self.db.posses["noun"]

		for lexicalEntryID in self.lexicalEntries:
			label = self.lexicalEntries[lexicalEntryID]

			if label[-2:] == "er" or label[-2:] == "ie" or label[-2:] == "en":
				guess_plural = label + "s"
			else:
				guess_plural = self.__getNounStem(label) + "en"

			if guess_plural in self.worddb:
				if self.userCheck("meervoud", label, guess_plural):
					# first get the database identifier and store the plural
					lex_id = self.db.getLexicalEntryIDByIdentifier(str(lexicalEntryID))
					self.db.storeOtherForm(lex_id,guess_plural,self.lang_id,["number:plural"])

					# then do the same for the canonicalForm, and store the singular
					lexicalFormID = self.g.value(URIRef(lexicalEntryID),ONTOLEX.canonicalForm,None)
					form_id = self.db.getLexicalFormID(str(lexicalFormID))
					self.db.storeFormProperty(form_id,self.db.morphosyntactics["number:singular"])
					self.db.DB.commit()

	def nounGender(self):
		""" Using rules from http://www.inventio.nl/genus/uitleg.html to detect word gender. """
		# Finding gender for noun forms without a gender.
		self.setProcessableForms(LEXINFO.noun,LEXINFO.gender)
		geoSenseIDs = self.getLexicalSenseIDsByReference(["http://www.wikidata.org/entity/Q6256"])

		for lexicalFormID in self.lexicalForms:
			label = self.lexicalForms[lexicalFormID]["label"]
			lexicalEntryID = self.lexicalForms[lexicalFormID]["lexicalEntryID"]
			guess_gender = ""

			# we want to check all forms, unless the sense gives us other info (only in the case of countries and placenames)
			# when first char is uppercase, we're gonna assume sense lookup first
			if label[0].isupper():
				guess_gender = self.__getNounGenderBySense(lexicalEntryID,geoSenseIDs)
			else:
				if label[-3:] == "ing" and self.checkLexicalEntryExists(label[:-3] + "en",LEXINFO.verb):
					guess_gender = "feminine"

			if self.userCheck("geslacht",label,guess_gender):
				form_id = self.db.getLexicalFormID(str(lexicalFormID))
				self.db.storeFormProperty(form_id,self.db.morphosyntactics["gender:" + guess_gender])
				self.db.DB.commit()

	def __getNounGenderBySense(self,lexicalEntryID,geoSenseIDs):
		for geoSenseID in geoSenseIDs:
			if self.__checkSenseRelation(URIRef(lexicalEntryID),SKOSTHES.broaderInstantial,URIRef(geoSenseID)):
				return "neuter"
		return ""

	def __countLexicalenses(self,lexicalEntryIdentifier):
		c = 0
		for lexicalSenseIdentifier in self.g.objects(URIRef(lexicalEntryIdentifier),ONTOLEX.sense):
			c = c + 1
		return c

	def __checkSenseRelation(self,lexicalEntryID,checkPredicate,checkObject):
		""" Given a lexicalEntryID, check all related senses to see if the requested relation is present."""
		for lexicalSenseID in self.g.objects(URIRef(lexicalEntryID),ONTOLEX.sense):
			if (URIRef(lexicalSenseID),checkPredicate,checkObject) in self.g:
				return True
		return False

	def __getNounStem(self,word):
		""" For example, boom -> bom -> bom + en = bomen."""
		if word[-3:-1] in [ "aa", "ee", "oo" ]:
			  return word[:-3] + word[-3:-2] + word[-1]
		else:
			return word
<commit_msg>put worddb in set for faster lookup<commit_after># -*- coding: utf-8 -*-

from rdflib import URIRef, Literal, Namespace
from rulesets.ruleset import RulesetCommon
from nltk.corpus import alpino

ONTOLEX = Namespace("http://www.w3.org/ns/lemon/ontolex#")
LEXINFO = Namespace("http://www.lexinfo.net/ontology/2.0/lexinfo#")
SKOSTHES = Namespace("http://purl.org/iso25964/skos-thes#")
LANGUAGE = "nl"

class Ruleset(RulesetCommon):
	def __init__(self,config,dont_ask=False):
		RulesetCommon.__init__(self,config,dont_ask)
		global ONTOLEX
		global LEXINFO
		global SKOSTHES

		# for now use alpino, we should be able to configure this
		self.worddb = set(alpino.words())
		self.language = LANGUAGE
		self.lang_id = self.db.languages[LANGUAGE]

	def adjectiveAntonyms(self):
		""" Based on word characteristics. Trying to prefix 'on' to a word. """
		pos_id = self.db.posses["adjective"]
		lexicalEntryIDs = {}

		for lexicalEntryIdentifier in self.g.subjects(LEXINFO.partOfSpeech,LEXINFO.adjective):
			label = self.getLabel(lexicalEntryIdentifier)
			if label[:2] != "on":
				lexicalEntryIDs[lexicalEntryIdentifier] = label

		for lexicalEntryIdentifier in lexicalEntryIDs:
			source_value = lexicalEntryIDs[lexicalEntryIdentifier]
			guess_antonym = "on" + source_value

			if self.userCheck("tegenstelling", source_value, guess_antonym):
				sensecount = self.__countLexicalenses(lexicalEntryIdentifier)
				if sensecount > 1:
					print("multiple senses detected, store manually")
					continue
				elif sensecount == 1:
					# check antonym
					lexicalSenseIdentifier = self.g.value(URIRef(lexicalEntryIdentifier),ONTOLEX.sense,None)
					if (URIRef(lexicalSenseIdentifier),LEXINFO.antonym,None) in self.g:
						print("antonym already detected, moving on")
						continue

				# at this point sensecount is 0, or it is safe to move on
				# we might want to store now, first the guess antonym lexicalEntry, then the sense
				self.db.storeCanonical(guess_antonym,self.lang_id,pos_id)
				self.db.addSense(source_value,pos_id,"lexinfo","antonym",guess_antonym,pos_id)

	def adjectiveMaterialNouns(self):
		""" Get all material nouns, based on being part of material category."""
		materialSenseIDs = self.getLexicalSenseIDsByReference(["http://www.wikidata.org/entity/Q11344"])
		source_pos_id = self.db.posses["noun"]
		target_pos_id = self.db.posses["adjective"]

		# now get all specific values in lexicalEntries
		for mID in materialSenseIDs:
			for senseID in self.g.objects(URIRef(mID),SKOSTHES.narrowerInstantial):
				lexicalEntryID = str(self.g.value(None,ONTOLEX.sense,URIRef(senseID)))
				self.lexicalEntries[lexicalEntryID] = self.getLabel(senseID)

		for lexicalEntryID in self.lexicalEntries:
			label = self.lexicalEntries[lexicalEntryID]
			guess_adjective = self.__getNounStem(label) + "en"

			if guess_adjective in self.worddb:
				if self.userCheck("bijvoegelijk naamwoord", label, guess_adjective):
					self.db.storeCanonical(guess_adjective,self.lang_id,target_pos_id)
					self.db.addSense(label,source_pos_id,"skos","related",guess_adjective,target_pos_id)

	def verbRelatedNouns(self):
		""" example. delen -> deling """
		source_pos_id = self.db.posses["verb"]
		target_pos_id = self.db.posses["noun"]
		lexicalEntryIDs = {}

		for lexicalEntryIdentifier in self.g.subjects(LEXINFO.partOfSpeech,LEXINFO.verb):
			self.lexicalEntries[lexicalEntryIdentifier] = self.getLabel(lexicalEntryIdentifier)

		for lexicalEntryIdentifier in self.lexicalEntries:
			source_value = self.lexicalEntries[lexicalEntryIdentifier]
			guess_noun = source_value[:-2] + "ing"

			# also, perhaps make this configurable as well
			if guess_noun in self.worddb:
				if self.userCheck("gerelateerd", source_value, guess_noun):
					sensecount = self.__countLexicalenses(lexicalEntryIdentifier)
					if sensecount > 1:
						print("multiple senses detected, store manually")
						continue
					elif sensecount == 1:
						# check if relation to term exists
						if self.checkLexicalEntryExists(guess_noun, LEXINFO.noun):
							# bit lazy here, but just playing now
							print("entry found for target, so maybe it is connected already")
							continue

					# at this point sensecount is 0, or it is safe to move on
					self.db.storeCanonical(guess_noun,self.lang_id,target_pos_id)
					self.db.addSense(source_value,source_pos_id,"skos","related",guess_noun,target_pos_id)

	def nounPlurals(self):
		# Finding plurals for nouns without a plural form. """
		self.setProcessableEntries(LEXINFO.noun,LEXINFO.number,LEXINFO.plural)
		source_pos_id = self.db.posses["noun"]
		target_pos_id = self.db.posses["noun"]

		for lexicalEntryID in self.lexicalEntries:
			label = self.lexicalEntries[lexicalEntryID]

			if label[-2:] == "er" or label[-2:] == "ie" or label[-2:] == "en":
				guess_plural = label + "s"
			else:
				guess_plural = self.__getNounStem(label) + "en"

			if guess_plural in self.worddb:
				if self.userCheck("meervoud", label, guess_plural):
					# first get the database identifier and store the plural
					lex_id = self.db.getLexicalEntryIDByIdentifier(str(lexicalEntryID))
					self.db.storeOtherForm(lex_id,guess_plural,self.lang_id,["number:plural"])

					# then do the same for the canonicalForm, and store the singular
					lexicalFormID = self.g.value(URIRef(lexicalEntryID),ONTOLEX.canonicalForm,None)
					form_id = self.db.getLexicalFormID(str(lexicalFormID))
					self.db.storeFormProperty(form_id,self.db.morphosyntactics["number:singular"])
					self.db.DB.commit()

	def nounGender(self):
		""" Using rules from http://www.inventio.nl/genus/uitleg.html to detect word gender. """
		# Finding gender for noun forms without a gender.
		self.setProcessableForms(LEXINFO.noun,LEXINFO.gender)
		geoSenseIDs = self.getLexicalSenseIDsByReference(["http://www.wikidata.org/entity/Q6256"])

		for lexicalFormID in self.lexicalForms:
			label = self.lexicalForms[lexicalFormID]["label"]
			lexicalEntryID = self.lexicalForms[lexicalFormID]["lexicalEntryID"]
			guess_gender = ""

			# we want to check all forms, unless the sense gives us other info (only in the case of countries and placenames)
			# when first char is uppercase, we're gonna assume sense lookup first
			if label[0].isupper():
				guess_gender = self.__getNounGenderBySense(lexicalEntryID,geoSenseIDs)
			else:
				if label[-3:] == "ing" and self.checkLexicalEntryExists(label[:-3] + "en",LEXINFO.verb):
					guess_gender = "feminine"

			if self.userCheck("geslacht",label,guess_gender):
				form_id = self.db.getLexicalFormID(str(lexicalFormID))
				self.db.storeFormProperty(form_id,self.db.morphosyntactics["gender:" + guess_gender])
				self.db.DB.commit()

	def __getNounGenderBySense(self,lexicalEntryID,geoSenseIDs):
		for geoSenseID in geoSenseIDs:
			if self.__checkSenseRelation(URIRef(lexicalEntryID),SKOSTHES.broaderInstantial,URIRef(geoSenseID)):
				return "neuter"
		return ""

	def __countLexicalenses(self,lexicalEntryIdentifier):
		c = 0
		for lexicalSenseIdentifier in self.g.objects(URIRef(lexicalEntryIdentifier),ONTOLEX.sense):
			c = c + 1
		return c

	def __checkSenseRelation(self,lexicalEntryID,checkPredicate,checkObject):
		""" Given a lexicalEntryID, check all related senses to see if the requested relation is present."""
		for lexicalSenseID in self.g.objects(URIRef(lexicalEntryID),ONTOLEX.sense):
			if (URIRef(lexicalSenseID),checkPredicate,checkObject) in self.g:
				return True
		return False

	def __getNounStem(self,word):
		""" For example, boom -> bom -> bom + en = bomen."""
		if word[-3:-1] in [ "aa", "ee", "oo" ]:
			  return word[:-3] + word[-3:-2] + word[-1]
		else:
			return word
<|endoftext|>