<commit_before>import random
import hashlib
import base64

def generate_key():
    key = hashlib.sha224(str(random.getrandbits(256))).digest()
    key = base64.b64encode(key,
            random.choice(['rA','aZ','gQ','hH','hG','aR','DD']))
    key = key.rstrip('==')
    return key

def chunked(seq, n):
    """By Ned Batchelder.
    Yield successive n-sized chunks from seq.

    >>> for group in chunked(range(8), 3):
    ...     print group
    [0, 1, 2]
    [3, 4, 5]
    [6, 7]
    """
    for i in xrange(0, len(seq), n):
        yield seq[i:i+n]

def flatten(lst):
    for elem in lst:
        if hasattr(elem, '__iter__'):
            for e in flatten(elem):
                yield e
        else:
            yield elem

def find_dict_key(dictionary, search_value):
    for key, value in dictionary.iteritems():
        if value == search_value:
            return key

def extract(dictionary, keys):
    """Returns a new dictionary with only the keys from the dictionary passed in
    specified in the keys list.

    """
    return dict((key, dictionary[key]) for key in keys if key in dictionary)

<commit_msg>Return lowercased random tokens for consistency.<commit_after>import random
import hashlib
import base64

def generate_key():
    key = hashlib.sha224(str(random.getrandbits(256))).digest()
    key = base64.b64encode(key,
            random.choice(['rA','aZ','gQ','hH','hG','aR','DD']))
    key = key.rstrip('==').lower()
    return key

def chunked(seq, n):
    """By Ned Batchelder.
    Yield successive n-sized chunks from seq.

    >>> for group in chunked(range(8), 3):
    ...     print group
    [0, 1, 2]
    [3, 4, 5]
    [6, 7]
    """
    for i in xrange(0, len(seq), n):
        yield seq[i:i+n]

def flatten(lst):
    for elem in lst:
        if hasattr(elem, '__iter__'):
            for e in flatten(elem):
                yield e
        else:
            yield elem

def find_dict_key(dictionary, search_value):
    for key, value in dictionary.iteritems():
        if value == search_value:
            return key

def extract(dictionary, keys):
    """Returns a new dictionary with only the keys from the dictionary passed in
    specified in the keys list.

    """
    return dict((key, dictionary[key]) for key in keys if key in dictionary)

<|endoftext|>


<commit_before>#!/usr/bin/python -B -u

import fileinput, glob, os, pprint, re, sys, tempfile, shutil, string

sys.path.append ('../..')

from bockbuild.darwinprofile import DarwinProfile
from bockbuild.util import *
from packages import MonoReleasePackages

class MonoReleaseProfile (DarwinProfile, MonoReleasePackages):
	def __init__ (self):
		self.MONO_ROOT = "/Library/Frameworks/Mono.framework"
		self.RELEASE_VERSION = "2.11.2" # REMEMBER TO UPDATE
		self.BUILD_NUMBER = "0"
		self.MRE_GUID = "432959f9-ce1b-47a7-94d3-eb99cb2e1aa8"
		self.MDK_GUID = "964ebddd-1ffe-47e7-8128-5ce17ffffb05"

		# Create the updateid
		parts = self.RELEASE_VERSION.split(".")
		version_list = ( parts + ["0"] * (3 - len(parts)) )[:4]
		for i in range(1,3):
			version_list [i] = version_list [i].zfill (2)
		self.updateid = "".join (version_list)
		self.updateid += self.BUILD_NUMBER.replace (".", "").zfill (9 - len (self.updateid))

		versions_root = os.path.join (self.MONO_ROOT, "Versions")
		self.release_root = os.path.join (versions_root, self.RELEASE_VERSION)

		DarwinProfile.__init__ (self, self.release_root)
		MonoReleasePackages.__init__ (self)

		self.self_dir = os.path.realpath (os.path.dirname (sys.argv[0]))
		self.packaging_dir = os.path.join (self.self_dir, "packaging")

		aclocal_dir = os.path.join (self.prefix, "share", "aclocal")
		if not os.path.exists(aclocal_dir):
			os.makedirs (aclocal_dir)

	def framework_path (self, subdir):
		return os.path.join (self.prefix, subdir)

	def remove_files (self, subdir = "lib", prefix = "*"):
		dir = os.path.join (self.prefix, subdir)
		print "Removing %s files in %s" % (prefix, dir)
		backtick ('find %s -name "%s" -delete' % (dir, prefix))

	def include_libgdiplus (self):
		config = os.path.join (self.prefix, "etc", "mono", "config")
		temp = config + ".tmp"
		lib = self.framework_path("lib")
		with open(config) as c:
			with open(temp, "w") as output:
				for line in c:
					if re.search(r'</configuration>', line):
						# Insert libgdiplus entries before the end of the file
						gdipluspath = os.path.join (lib, "libgdiplus.dylib")
						output.write('\t<dllmap dll="gdiplus" target="%s" />\n' % gdipluspath)
						output.write('\t<dllmap dll="gdiplus.dll" target="%s" />\n' % gdipluspath)
					output.write(line)

		os.rename(temp, config)

	def make_package_symlinks(self, root):
		os.symlink (self.prefix, os.path.join (root, "Versions", "Current"))
		currentlink = os.path.join (self.MONO_ROOT, "Versions", "Current")
		links = [
			("bin", "Commands"),
			("include", "Headers"),
			("lib", "Libraries"),
			("", "Home"),
			(os.path.join ("lib", "libmono-2.0.dylib"), "Mono")
		]
		for srcname, destname in links:
			src  = os.path.join (currentlink, srcname)
			dest = os.path.join (root, destname)
			if os.path.exists (dest):
				os.unlink (dest)
			os.symlink (src, dest)

	# creates and returns the path to a working directory containing:
	#   PKGROOT/ - this root will be bundled into the .pkg and extracted at /
	#   uninstallMono.sh - copied onto the DMG
	#   Info{_sdk}.plist - used by packagemaker to make the installer
	#   resources/ - other resources used by packagemaker for the installer
	def setup_working_dir (self):
		tmpdir = tempfile.mkdtemp()
		monoroot = os.path.join (tmpdir, "PKGROOT", self.MONO_ROOT[1:])
		versions = os.path.join (monoroot, "Versions")
		os.makedirs (versions)

		# setup metadata
		backtick ('rsync -aP %s/* %s' % (self.packaging_dir, tmpdir))
		parameter_map = {
			'@@MONO_VERSION@@': self.RELEASE_VERSION,
			'@@MONO_RELEASE@@': self.BUILD_NUMBER,
			'@@MONO_VERSION_RELEASE@@': self.RELEASE_VERSION + '_' + self.BUILD_NUMBER,
			'@@MONO_PACKAGE_GUID@@': self.MRE_GUID,
			'@@MONO_CSDK_GUID@@': self.MDK_GUID,
			'@@MONO_VERSION_RELEASE_INT@@': self.updateid,
			'@@PACKAGES@@': string.join (set([root for root,ext in map(os.path.splitext, os.listdir (self.build_root))]), "\\\n"),
			'@@DEP_PACKAGES@@': ""
		}
		for dirpath, d, files in os.walk (tmpdir):
			for name in files:
				if not name.startswith('.'):
					replace_in_file (os.path.join (dirpath, name), parameter_map)

		self.make_package_symlinks(monoroot)

		# copy to package root
		backtick ('rsync -aP "%s" "%s"' % (self.release_root, versions))

		return tmpdir

	def apply_blacklist (self, working_dir, blacklist_name):
		blacklist = os.path.join (working_dir, blacklist_name)
		root = os.path.join (working_dir, "PKGROOT", self.release_root[1:])
		backtick (blacklist + ' "%s"' % root)

	def run_package_maker (self, working_dir, pkg_file_name, title):
		packagemaker = '/Developer/Applications/Utilities/PackageMaker.app/Contents/MacOS/PackageMaker'
		output = os.path.join (working_dir, pkg_file_name)
		cmd = ' '.join([packagemaker,

			"--resources '%s/resources'" % working_dir,
			"--info '%s/Info.plist'" % working_dir,
			"--root '%s/PKGROOT'" % working_dir,

			"--out '%s'" % output,
			"--title '%s'" % title,
			"-x '.DS_Store'",
			"--verbose"
		])
		backtick (cmd)
		return output

	def make_updateinfo (self, working_dir, guid):
		with open(os.path.join (working_dir, "PKGROOT", self.release_root[1:], "updateinfo"), "w") as updateinfo:
			updateinfo.write (guid + ' ' + self.updateid + "\n")

	def make_dmg (self, output, title, *contents):
		dmgroot = tempfile.mkdtemp()
		backtick ('rsync -aP "%s" "%s"' % ('" "'.join (contents), dmgroot))
		backtick ('hdiutil create -ov -srcfolder "%s" -volname "%s" "%s"' % (dmgroot, title, output))
		shutil.rmtree (dmgroot)

	def build_package (self):
		out_path = self.self_dir
		working = self.setup_working_dir ()

		mre_dmg = os.path.join (out_path, "MonoFramework-MRE-%s.macos10.xamarin.x86.dmg" % self.RELEASE_VERSION)
		mdk_dmg = os.path.join (out_path, "MonoFramework-MDK-%s.macos10.xamarin.x86.dmg" % self.RELEASE_VERSION)
		uninstall_script = os.path.join (working, "uninstallMono.sh")

		# make the MDK
		title = 'Mono Framework MDK ' + self.RELEASE_VERSION
		self.apply_blacklist (working, 'mdk_blacklist.sh')
		self.make_updateinfo (working, self.MDK_GUID)
		mdk_pkg = self.run_package_maker (working, "MonoFramework-MDK-%s.macos10.xamarin.x86.pkg" % self.RELEASE_VERSION, title)
		print "Saving: " + mdk_dmg
		self.make_dmg (mdk_dmg, title, mdk_pkg, uninstall_script)

		# make the MRE
		title = 'Mono Framework MRE ' + self.RELEASE_VERSION
		self.apply_blacklist (working, 'mre_blacklist.sh')
		self.make_updateinfo (working, self.MRE_GUID)
		mre_pkg = self.run_package_maker (working, "MonoFramework-MRE-%s.macos10.xamarin.x86.pkg" % self.RELEASE_VERSION, title)
		print "Saving: " + mre_dmg
		self.make_dmg (mre_dmg, title, mre_pkg, uninstall_script)

		shutil.rmtree (working)

	# THIS IS THE MAIN METHOD FOR MAKING A PACKAGE
	def package (self):
		self.remove_files (prefix = '*.la')
		self.remove_files (prefix = '*.a')
		self.include_libgdiplus ()
		# must apply blacklist first here because PackageMaker follows symlinks :(
		backtick (os.path.join (self.packaging_dir, 'mdk_blacklist.sh') + ' ' + self.release_root)
		self.build_package ()

MonoReleaseProfile ().build ()

profname = "mono-mac-release-env"
dir = os.path.realpath (os.path.dirname (sys.argv[0]))
envscript = '''#!/bin/sh
PROFNAME="%s"
INSTALLDIR=%s/build-root/_install
export DYLD_FALLBACK_LIBRARY_PATH="$INSTALLDIR/lib:/lib:/usr/lib:$DYLD_FALLBACK_LIBRARY_PATH"
export C_INCLUDE_PATH="$INSTALLDIR/include:$C_INCLUDE_PATH"
export ACLOCAL_PATH="$INSTALLDIR/share/aclocal:$ACLOCAL_PATH"
export ACLOCAL_FLAGS="-I $INSTALLDIR/share/aclocal $ACLOCAL_FLAGS"
export PKG_CONFIG_PATH="$INSTALLDIR/lib/pkgconfig:$INSTALLDIR/lib64/pkgconfig:$INSTALLDIR/share/pkgconfig:$PKG_CONFIG_PATH"
export CONFIG_SITE="$INSTALLDIR/$PROFNAME-config.site"
export MONO_GAC_PREFIX="$INSTALLDIR:MONO_GAC_PREFIX"
export MONO_ADDINS_REGISTRY="$INSTALLDIR/addinreg"
export PATH="$INSTALLDIR/bin:$PATH"
export MONO_INSTALL_PREFIX="$INSTALLDIR"

#mkdir -p "$INSTALLDIR"
#echo "test \"\$prefix\" = NONE && prefix=\"$INSTALLDIR\"" > $CONFIG_SITE

PS1="[$PROFNAME] \w @ "
''' % ( profname, dir )

with open(os.path.join (dir, profname), 'w') as f:
	f.write (envscript)
<commit_msg>[mono-mac-release] Fix import<commit_after>#!/usr/bin/python -B -u

import fileinput, glob, os, pprint, re, sys, tempfile, shutil, string

sys.path.append ('../..')

from bockbuild.darwinprofile import DarwinProfile
from bockbuild.util.util import *
from packages import MonoReleasePackages

class MonoReleaseProfile (DarwinProfile, MonoReleasePackages):
	def __init__ (self):
		self.MONO_ROOT = "/Library/Frameworks/Mono.framework"
		self.RELEASE_VERSION = "2.11.2" # REMEMBER TO UPDATE
		self.BUILD_NUMBER = "0"
		self.MRE_GUID = "432959f9-ce1b-47a7-94d3-eb99cb2e1aa8"
		self.MDK_GUID = "964ebddd-1ffe-47e7-8128-5ce17ffffb05"

		# Create the updateid
		parts = self.RELEASE_VERSION.split(".")
		version_list = ( parts + ["0"] * (3 - len(parts)) )[:4]
		for i in range(1,3):
			version_list [i] = version_list [i].zfill (2)
		self.updateid = "".join (version_list)
		self.updateid += self.BUILD_NUMBER.replace (".", "").zfill (9 - len (self.updateid))

		versions_root = os.path.join (self.MONO_ROOT, "Versions")
		self.release_root = os.path.join (versions_root, self.RELEASE_VERSION)

		DarwinProfile.__init__ (self, self.release_root)
		MonoReleasePackages.__init__ (self)

		self.self_dir = os.path.realpath (os.path.dirname (sys.argv[0]))
		self.packaging_dir = os.path.join (self.self_dir, "packaging")

		aclocal_dir = os.path.join (self.prefix, "share", "aclocal")
		if not os.path.exists(aclocal_dir):
			os.makedirs (aclocal_dir)

	def framework_path (self, subdir):
		return os.path.join (self.prefix, subdir)

	def remove_files (self, subdir = "lib", prefix = "*"):
		dir = os.path.join (self.prefix, subdir)
		print "Removing %s files in %s" % (prefix, dir)
		backtick ('find %s -name "%s" -delete' % (dir, prefix))

	def include_libgdiplus (self):
		config = os.path.join (self.prefix, "etc", "mono", "config")
		temp = config + ".tmp"
		lib = self.framework_path("lib")
		with open(config) as c:
			with open(temp, "w") as output:
				for line in c:
					if re.search(r'</configuration>', line):
						# Insert libgdiplus entries before the end of the file
						gdipluspath = os.path.join (lib, "libgdiplus.dylib")
						output.write('\t<dllmap dll="gdiplus" target="%s" />\n' % gdipluspath)
						output.write('\t<dllmap dll="gdiplus.dll" target="%s" />\n' % gdipluspath)
					output.write(line)

		os.rename(temp, config)

	def make_package_symlinks(self, root):
		os.symlink (self.prefix, os.path.join (root, "Versions", "Current"))
		currentlink = os.path.join (self.MONO_ROOT, "Versions", "Current")
		links = [
			("bin", "Commands"),
			("include", "Headers"),
			("lib", "Libraries"),
			("", "Home"),
			(os.path.join ("lib", "libmono-2.0.dylib"), "Mono")
		]
		for srcname, destname in links:
			src  = os.path.join (currentlink, srcname)
			dest = os.path.join (root, destname)
			if os.path.exists (dest):
				os.unlink (dest)
			os.symlink (src, dest)

	# creates and returns the path to a working directory containing:
	#   PKGROOT/ - this root will be bundled into the .pkg and extracted at /
	#   uninstallMono.sh - copied onto the DMG
	#   Info{_sdk}.plist - used by packagemaker to make the installer
	#   resources/ - other resources used by packagemaker for the installer
	def setup_working_dir (self):
		tmpdir = tempfile.mkdtemp()
		monoroot = os.path.join (tmpdir, "PKGROOT", self.MONO_ROOT[1:])
		versions = os.path.join (monoroot, "Versions")
		os.makedirs (versions)

		# setup metadata
		backtick ('rsync -aP %s/* %s' % (self.packaging_dir, tmpdir))
		parameter_map = {
			'@@MONO_VERSION@@': self.RELEASE_VERSION,
			'@@MONO_RELEASE@@': self.BUILD_NUMBER,
			'@@MONO_VERSION_RELEASE@@': self.RELEASE_VERSION + '_' + self.BUILD_NUMBER,
			'@@MONO_PACKAGE_GUID@@': self.MRE_GUID,
			'@@MONO_CSDK_GUID@@': self.MDK_GUID,
			'@@MONO_VERSION_RELEASE_INT@@': self.updateid,
			'@@PACKAGES@@': string.join (set([root for root,ext in map(os.path.splitext, os.listdir (self.build_root))]), "\\\n"),
			'@@DEP_PACKAGES@@': ""
		}
		for dirpath, d, files in os.walk (tmpdir):
			for name in files:
				if not name.startswith('.'):
					replace_in_file (os.path.join (dirpath, name), parameter_map)

		self.make_package_symlinks(monoroot)

		# copy to package root
		backtick ('rsync -aP "%s" "%s"' % (self.release_root, versions))

		return tmpdir

	def apply_blacklist (self, working_dir, blacklist_name):
		blacklist = os.path.join (working_dir, blacklist_name)
		root = os.path.join (working_dir, "PKGROOT", self.release_root[1:])
		backtick (blacklist + ' "%s"' % root)

	def run_package_maker (self, working_dir, pkg_file_name, title):
		packagemaker = '/Developer/Applications/Utilities/PackageMaker.app/Contents/MacOS/PackageMaker'
		output = os.path.join (working_dir, pkg_file_name)
		cmd = ' '.join([packagemaker,

			"--resources '%s/resources'" % working_dir,
			"--info '%s/Info.plist'" % working_dir,
			"--root '%s/PKGROOT'" % working_dir,

			"--out '%s'" % output,
			"--title '%s'" % title,
			"-x '.DS_Store'",
			"--verbose"
		])
		backtick (cmd)
		return output

	def make_updateinfo (self, working_dir, guid):
		with open(os.path.join (working_dir, "PKGROOT", self.release_root[1:], "updateinfo"), "w") as updateinfo:
			updateinfo.write (guid + ' ' + self.updateid + "\n")

	def make_dmg (self, output, title, *contents):
		dmgroot = tempfile.mkdtemp()
		backtick ('rsync -aP "%s" "%s"' % ('" "'.join (contents), dmgroot))
		backtick ('hdiutil create -ov -srcfolder "%s" -volname "%s" "%s"' % (dmgroot, title, output))
		shutil.rmtree (dmgroot)

	def build_package (self):
		out_path = self.self_dir
		working = self.setup_working_dir ()

		mre_dmg = os.path.join (out_path, "MonoFramework-MRE-%s.macos10.xamarin.x86.dmg" % self.RELEASE_VERSION)
		mdk_dmg = os.path.join (out_path, "MonoFramework-MDK-%s.macos10.xamarin.x86.dmg" % self.RELEASE_VERSION)
		uninstall_script = os.path.join (working, "uninstallMono.sh")

		# make the MDK
		title = 'Mono Framework MDK ' + self.RELEASE_VERSION
		self.apply_blacklist (working, 'mdk_blacklist.sh')
		self.make_updateinfo (working, self.MDK_GUID)
		mdk_pkg = self.run_package_maker (working, "MonoFramework-MDK-%s.macos10.xamarin.x86.pkg" % self.RELEASE_VERSION, title)
		print "Saving: " + mdk_dmg
		self.make_dmg (mdk_dmg, title, mdk_pkg, uninstall_script)

		# make the MRE
		title = 'Mono Framework MRE ' + self.RELEASE_VERSION
		self.apply_blacklist (working, 'mre_blacklist.sh')
		self.make_updateinfo (working, self.MRE_GUID)
		mre_pkg = self.run_package_maker (working, "MonoFramework-MRE-%s.macos10.xamarin.x86.pkg" % self.RELEASE_VERSION, title)
		print "Saving: " + mre_dmg
		self.make_dmg (mre_dmg, title, mre_pkg, uninstall_script)

		shutil.rmtree (working)

	# THIS IS THE MAIN METHOD FOR MAKING A PACKAGE
	def package (self):
		self.remove_files (prefix = '*.la')
		self.remove_files (prefix = '*.a')
		self.include_libgdiplus ()
		# must apply blacklist first here because PackageMaker follows symlinks :(
		backtick (os.path.join (self.packaging_dir, 'mdk_blacklist.sh') + ' ' + self.release_root)
		self.build_package ()

MonoReleaseProfile ().build ()

profname = "mono-mac-release-env"
dir = os.path.realpath (os.path.dirname (sys.argv[0]))
envscript = '''#!/bin/sh
PROFNAME="%s"
INSTALLDIR=%s/build-root/_install
export DYLD_FALLBACK_LIBRARY_PATH="$INSTALLDIR/lib:/lib:/usr/lib:$DYLD_FALLBACK_LIBRARY_PATH"
export C_INCLUDE_PATH="$INSTALLDIR/include:$C_INCLUDE_PATH"
export ACLOCAL_PATH="$INSTALLDIR/share/aclocal:$ACLOCAL_PATH"
export ACLOCAL_FLAGS="-I $INSTALLDIR/share/aclocal $ACLOCAL_FLAGS"
export PKG_CONFIG_PATH="$INSTALLDIR/lib/pkgconfig:$INSTALLDIR/lib64/pkgconfig:$INSTALLDIR/share/pkgconfig:$PKG_CONFIG_PATH"
export CONFIG_SITE="$INSTALLDIR/$PROFNAME-config.site"
export MONO_GAC_PREFIX="$INSTALLDIR:MONO_GAC_PREFIX"
export MONO_ADDINS_REGISTRY="$INSTALLDIR/addinreg"
export PATH="$INSTALLDIR/bin:$PATH"
export MONO_INSTALL_PREFIX="$INSTALLDIR"

#mkdir -p "$INSTALLDIR"
#echo "test \"\$prefix\" = NONE && prefix=\"$INSTALLDIR\"" > $CONFIG_SITE

PS1="[$PROFNAME] \w @ "
''' % ( profname, dir )

with open(os.path.join (dir, profname), 'w') as f:
	f.write (envscript)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This module provides classes to perform fitting of two structures.
"""

from __future__ import division

__author__ = "Stephen Dacek, William Davidson Richards, Shyue Ping Ong"
__copyright__ = "Copyright 2011, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Stephen Dacek"
__email__ = "sdacek@mit.edu"
__status__ = "Beta"
__date__ = "Dec 3, 2012"

from pymatgen.core.structure_modifier import BasisChange
from pymatgen.core.structure import Structure
from pymatgen.core.structure_modifier import StructureEditor
from pymatgen.core.lattice import Lattice
from pymatgen.transformations.standard_transformations import \
    PrimitiveCellTransformation
from pymatgen.util.coord_utils import find_in_coord_list_pbc

import numpy as np
import itertools

def basis_change(coords, new_lattice):
    #Convert cartesian coordinates to new basis
    frac_coords = []
    for i in range(len(coords)):
        frac_coords.append(new_lattice.get_fractional_coords(coords[i]))

    return frac_coords

def cmp_struct(s1, s2, frac_tol):
    #compares the fractional coordinates
    for s1_coords, s2_coords in zip(s1, s2):
        #Available vectors
        avail = [1] * len(s1_coords)
        for coord in s1_coords:
            ind = find_in_coord_list_pbc(s2_coords, coord, frac_tol)
            #if more than one match found, take closest
            if len(ind) > 1:
                fcoords = np.tile(coord, (len(s2_coords[ind]), 1))
                fdist = np.mod(s2_coords[ind], 1) - np.mod(fcoords, 1)
                fdist -= np.round(fdist)
                avail[np.where(
                    np.all(np.abs(fdist) == np.min(np.abs(fdist)), axis=1))[
                      0]] = 0
            elif len(ind) and avail[ind]:
                avail[ind] = 0
            else:
                return False
    return True

def compare_structures(struct1, struct2, ltol=0.2, atol=5,
                       primitive_cell=False,
                       equivalent_species=[]):
    # Same number of sites
    if struct1.num_sites != struct2.num_sites:
        return False

    #primitive cell transformation - Needs work
    if primitive_cell:
        prim = PrimitiveCellTransformation()
        struct1 = prim.apply_transformation(struct1)
        struct2 = prim.apply_transformation(struct2)

    #compute niggli lattices,
    nl1 = struct1.lattice.get_niggli_reduced_lattice()
    nl2 = struct2.lattice.get_niggli_reduced_lattice()
    struct1 = BasisChange(struct1, nl1).modified_structure
    struct2 = BasisChange(struct2, nl2).modified_structure

    #rescale lattice to same volume
    se = StructureEditor(struct2)
    nl2 = Lattice(nl2.matrix * (nl1.volume / nl2.volume) ** (1.0 / 3))
    se.modify_lattice(nl2)
    struct2 = se.modified_structure

    #fractional tolerance
    frac_tol = np.array([ltol / i for i in struct1.lattice.abc])

    #get possible new lattice vectors
    ds = Structure(struct2.lattice, ['X'], [[0, 0, 0]])
    nv = []
    for i in range(3):
        l = struct1.lattice.abc[i]
        nvi = []
        vs = ds.get_neighbors_in_shell([0, 0, 0], l, ltol * l)
        for site, dist in vs:
            nvi.append(site.coords)
        nv.append(nvi)

    #generate structure coordinate lists
    species_list = []
    s1 = []
    for site in struct1:
        ind = None
        for i, specie in enumerate(species_list):
            if site.specie == specie:
                ind = i
                break

        if ind is not None:
            s1[ind] = np.append(s1[ind], [site.frac_coords], axis=0)
        else:
            s1.append(np.array([site.frac_coords]))
            species_list.append(site.specie)

    zipped = sorted(zip(s1, species_list), key=lambda x: len(x[0]))
    s1 = [x[0] for x in zipped]
    species_list = [x[1] for x in zipped]
    s2_cart = [[] for i in s1]

    for site in struct2:
        ind = None
        for i, specie in enumerate(species_list):
            if site.specie == specie:
                ind = i
                break
            #get cart coords for s2
        if s2_cart[ind]:
            s2_cart[ind] = np.append(s2_cart[ind], [site.coords], axis=0)
        else:
            s2_cart[ind] = np.array([site.coords])

    #translate s1
    s1_translation = s1[0][0]

    for i in range(len(species_list)):
        s1[i] = (s1[i] - s1_translation) % 1

    #do permutations of vectors, check for equality
    for a, b, c in itertools.product(nv[0], nv[1], nv[2]):
        if np.linalg.det([a, b, c]) == 0: #invalid lattice
            continue

        nl = Lattice([a, b, c])
        if np.allclose(nl.angles, struct1.lattice.angles, rtol=0, atol=atol):
            #Basis Change into new lattice
            s2 = basis_change(s2_cart, nl)

            for coord in s2[0]:
                t_s2 = []
                for coords in s2:
                    t_s2.append((coords - coord) % 1)
                if cmp_struct(s1, t_s2, frac_tol):
                    return True
    return False
<commit_msg>Minor fix to support disodered structures.<commit_after>#!/usr/bin/env python

"""
This module provides classes to perform fitting of two structures.
"""

from __future__ import division

__author__ = "Stephen Dacek, William Davidson Richards, Shyue Ping Ong"
__copyright__ = "Copyright 2011, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Stephen Dacek"
__email__ = "sdacek@mit.edu"
__status__ = "Beta"
__date__ = "Dec 3, 2012"

from pymatgen.core.structure_modifier import BasisChange
from pymatgen.core.structure import Structure
from pymatgen.core.structure_modifier import StructureEditor
from pymatgen.core.lattice import Lattice
from pymatgen.transformations.standard_transformations import \
    PrimitiveCellTransformation
from pymatgen.util.coord_utils import find_in_coord_list_pbc

import numpy as np
import itertools

def basis_change(coords, new_lattice):
    #Convert cartesian coordinates to new basis
    frac_coords = []
    for i in range(len(coords)):
        frac_coords.append(new_lattice.get_fractional_coords(coords[i]))

    return frac_coords

def cmp_struct(s1, s2, frac_tol):
    #compares the fractional coordinates
    for s1_coords, s2_coords in zip(s1, s2):
        #Available vectors
        avail = [1] * len(s1_coords)
        for coord in s1_coords:
            ind = find_in_coord_list_pbc(s2_coords, coord, frac_tol)
            #if more than one match found, take closest
            if len(ind) > 1:
                fcoords = np.tile(coord, (len(s2_coords[ind]), 1))
                fdist = np.mod(s2_coords[ind], 1) - np.mod(fcoords, 1)
                fdist -= np.round(fdist)
                avail[np.where(
                    np.all(np.abs(fdist) == np.min(np.abs(fdist)), axis=1))[
                      0]] = 0
            elif len(ind) and avail[ind]:
                avail[ind] = 0
            else:
                return False
    return True

def compare_structures(struct1, struct2, ltol=0.2, atol=5,
                       primitive_cell=False,
                       equivalent_species=[]):
    # Same number of sites
    if struct1.num_sites != struct2.num_sites:
        return False

    #primitive cell transformation - Needs work
    if primitive_cell:
        prim = PrimitiveCellTransformation()
        struct1 = prim.apply_transformation(struct1)
        struct2 = prim.apply_transformation(struct2)

    #compute niggli lattices,
    nl1 = struct1.lattice.get_niggli_reduced_lattice()
    nl2 = struct2.lattice.get_niggli_reduced_lattice()
    struct1 = BasisChange(struct1, nl1).modified_structure
    struct2 = BasisChange(struct2, nl2).modified_structure

    #rescale lattice to same volume
    se = StructureEditor(struct2)
    nl2 = Lattice(nl2.matrix * (nl1.volume / nl2.volume) ** (1.0 / 3))
    se.modify_lattice(nl2)
    struct2 = se.modified_structure

    #fractional tolerance
    frac_tol = np.array([ltol / i for i in struct1.lattice.abc])

    #get possible new lattice vectors
    ds = Structure(struct2.lattice, ['X'], [[0, 0, 0]])
    nv = []
    for i in range(3):
        l = struct1.lattice.abc[i]
        nvi = []
        vs = ds.get_neighbors_in_shell([0, 0, 0], l, ltol * l)
        for site, dist in vs:
            nvi.append(site.coords)
        nv.append(nvi)

    #generate structure coordinate lists
    species_list = []
    s1 = []
    for site in struct1:
        ind = None
        for i, species in enumerate(species_list):
            if site.species_and_occu == species:
                ind = i
                break

        if ind is not None:
            s1[ind] = np.append(s1[ind], [site.frac_coords], axis=0)
        else:
            s1.append(np.array([site.frac_coords]))
            species_list.append(site.species_and_occu)

    zipped = sorted(zip(s1, species_list), key=lambda x: len(x[0]))
    s1 = [x[0] for x in zipped]
    species_list = [x[1] for x in zipped]
    s2_cart = [[] for i in s1]

    for site in struct2:
        ind = None
        for i, species in enumerate(species_list):
            if site.species_and_occu == species:
                ind = i
                break
            #get cart coords for s2
        if s2_cart[ind] != []:
            s2_cart[ind] = np.append(s2_cart[ind], [site.coords], axis=0)
        else:
            s2_cart[ind] = np.array([site.coords])

    #translate s1
    s1_translation = s1[0][0]

    for i in range(len(species_list)):
        s1[i] = (s1[i] - s1_translation) % 1

    #do permutations of vectors, check for equality
    for a, b, c in itertools.product(nv[0], nv[1], nv[2]):
        if np.linalg.det([a, b, c]) == 0: #invalid lattice
            continue

        nl = Lattice([a, b, c])
        if np.allclose(nl.angles, struct1.lattice.angles, rtol=0, atol=atol):
            #Basis Change into new lattice
            s2 = basis_change(s2_cart, nl)

            for coord in s2[0]:
                t_s2 = []
                for coords in s2:
                    t_s2.append((coords - coord) % 1)
                if cmp_struct(s1, t_s2, frac_tol):
                    return True
    return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

{
    'name': 'Calendar',
    'version': '1.0',
    'sequence': 130,
    'depends': ['base', 'mail', 'base_automation', 'web_calendar'],
    'summary': 'Personal & Shared Calendar',
    'description': """
This is a full-featured calendar system.
========================================

It supports:
------------
    - Calendar of events
    - Recurring events

If you need to manage your meetings, you should install the CRM module.
    """,
    'category': 'Extra Tools',
    'website': 'https://www.odoo.com/page/crm',
    'demo': [
        'data/calendar_demo.xml'
    ],
    'data': [
        'security/ir.model.access.csv',
        'security/calendar_security.xml',
        'data/calendar_cron.xml',
        'data/calendar_data.xml',
        'views/calendar_templates.xml',
        'views/calendar_views.xml',
    ],
    'qweb': ['static/src/xml/*.xml'],
    'installable': True,
    'application': True,
    'auto_install': False,
}
<commit_msg>[REF] calendar: remove dependency to base automation<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

{
    'name': 'Calendar',
    'version': '1.0',
    'sequence': 130,
    'depends': ['base', 'mail', 'web_calendar'],
    'summary': 'Personal & Shared Calendar',
    'description': """
This is a full-featured calendar system.
========================================

It supports:
------------
    - Calendar of events
    - Recurring events

If you need to manage your meetings, you should install the CRM module.
    """,
    'category': 'Extra Tools',
    'website': 'https://www.odoo.com/page/crm',
    'demo': [
        'data/calendar_demo.xml'
    ],
    'data': [
        'security/ir.model.access.csv',
        'security/calendar_security.xml',
        'data/calendar_cron.xml',
        'data/calendar_data.xml',
        'views/calendar_templates.xml',
        'views/calendar_views.xml',
    ],
    'qweb': ['static/src/xml/*.xml'],
    'installable': True,
    'application': True,
    'auto_install': False,
}
<|endoftext|>


<commit_before>from django.shortcuts import get_object_or_404, render
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.views import generic
from django.views.generic.edit import FormView

from .models import *
from .forms import *

class IndexView(generic.ListView):
    template_name = 'shop/index.html'
    context_object_name = 'all_product'

    def get_queryset(self):
        return Product.objects.all()

class DetailView(generic.DetailView):
    model = Product
    template_name = 'shop/detail.html'

class RegisterView(FormView):
    template_name = 'shop/regis.html'
    form_class = RegisterForm
    success_url = '/shop/'

    def form_valid(self, form):
        return super(RegisterView, self).form_valid(form)
<commit_msg>update views.py<commit_after>from django.shortcuts import get_object_or_404, render
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.views import generic,View
from django.views.generic.edit import FormView
from django.views.generic import TemplateView

from .models import *
from .forms import *

class IndexView(generic.ListView):
    template_name = 'shop/index.html'
    context_object_name = 'all_product'

    def get_queryset(self):
        return Product.objects.all()

class DetailView(generic.DetailView):
    model = Product
    template_name = 'shop/detail.html'

class ContactView(TemplateView):
    model = Product
    template_name = "shop/contact-us-page.html"

class RegisterView(View):
    template_name = 'shop/regis.html'
    # success_url = '/shop/'

    #def form_valid(self, form):
    def get(self, request):
        return render(request, self.template_name, {})

    def post(self, request):
        forms = {}
        for key in request.POST.keys():
            forms[key] = request.POST.get(key, '')
        newuser = MyUser(is_superuser=0,
                    username=forms['display_name'],
                    first_name=forms['first_name'],
                    last_name=forms['last_name'],
                    email=forms['email'])
        newuser.set_password(forms['password'])
        newuser.save()
        #return redirect('/shop')

        return render(request, self.template_name, forms)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import ast
import copy
import dateutil.parser
import dicom
import elasticsearch
import json
import logging

from api import config
from api.web import encoder

es = config.es
db = config.db

DE_INDEX = 'data_explorer'

ANALYSIS = {
            'analyzer' : {
                'str_search_analyzer' : {
                    'tokenizer' : 'keyword',
                    'filter' : ['lowercase']
                },

                'str_index_analyzer' : {
                    'tokenizer' : 'keyword',
                    'filter' : ['lowercase', 'substring']
                }
            },
            'filter' : {
                'substring': {
                    'type': 'nGram',
                    'min_gram': 2,
                    'max_gram': 50,
                    'token_chars': []
                }
            }
        }

DYNAMIC_TEMPLATES = [
    {
            '_id': {
                'match': '_id',
                'match_mapping_type' : 'string',
                'mapping': {
                   'type': 'keyword',
                   'index': 'not_analyzed'
                }
            }
        },
    {
            'long_fields' : {
                'match_mapping_type' : 'long',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'integer_fields' : {
                'match_mapping_type' : 'integer',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'double_fields' : {
                'match_mapping_type' : 'double',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'float_fields' : {
                'match_mapping_type' : 'float',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'short_fields' : {
                'match_mapping_type' : 'short',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'byte_fields' : {
                'match_mapping_type' : 'byte',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
        {
            'hash': {
                'match': 'hash',
                'match_mapping_type' : 'string',
                'mapping': {
                   'type': 'text',
                   'index': 'not_analyzed'
                }
            }
        },
        {
            'string_fields' : {
                'match': '*',
                'match_mapping_type' : 'string',
                'mapping' : {
                    'type': 'text',
                    'analyzer': 'str_search_analyzer',
                    "fields": {
                        "raw": {
                            "type": "keyword",
                            "index": "not_analyzed",
                            "ignore_above": 256
                        }
                    }
                }
            }
        }
]

def datetime(str_datetime):
    pass

def age(str_age):
    pass

BLACKLIST_KEYS = ['template', 'roles', 'permissions', 'analyses', 'files', 'collections']

SKIPPED = []
SKIPPED2ELECTRICBOOGALOO = ['PixelSpacing', 'ImageOrientationPatient', 'PatientAge', 'ImagePositionPatient']

# TODO: Choose integer/long and float/double where appropriate
VR_TYPES = {
        'AE': ['string', str],
        'AS': ['long', int],
        'AT': ['long', int],
        'CS': ['string', str],
        'DA': ['date', datetime],
        'DS': ['float', float],
        'DT': ['date', datetime],
        'FD': ['float', float],
        'FL': ['float', float],
        'IS': ['long', int],
        'LO': ['string', str],
        'LT': ['string', str],
        'NONE': None,
        'OB': None,
        'OB or OW': None,
        'OF': None,
        'OW': None,
        'PN': ['string', str],
        'SH': ['string', str],
        'SL': ['long', int],
        'SQ': None,
        'SS': ['long', int],
        'ST': ['string', str],
        'TM': ['time', datetime],
        'UI': ['string', str],
        'UL': ['long', int],
        'US': ['long', int],
        'US or OW': ['long', int],
        'US or SS': ['long', int],
        'US or SS or OW': ['long', int],
        'UT': ['string', str]
    }

def create_mappings():
    public_dict = dicom._dicom_dict.DicomDictionary
    field_mappings = {}
    for k,v in public_dict.iteritems():
        vr_type = v[0]
        field_name = v[4]
        vr_mapping = VR_TYPES.get(vr_type)
        if vr_mapping:
            field_type = vr_mapping[0]
            if field_type == 'string' and vr_type not in ['UT', 'LT', 'ST']:
                field_mappings[field_name+'_term'] = {'type': 'string', 'index': 'not_analyzed'}
            field_mappings[field_name] = {'type': field_type}
            if field_type == 'time':
                # Actually store as date, format as time:
                field_mappings[field_name] = {'type': 'date', 'format': 'time'}
        else:
            pass
            #logging.warn('Skipping field {} of VR type {}'.format(field_name, vr_type))

    return field_mappings

def cast_date(dcm_date):
    """
    Cast DICOM date string (YYYYMMDD) into ElasticSearch pre-defined strict_date format (yyyy-MM-dd)
    """
    return dcm_date[:4] + '-' + dcm_date[4:6] + '-' + dcm_date[6:]

def cast_time(dcm_time):
    """
    Cast DICOM time string (HHMMSS.FRAC)
    into ElasticSearch pre-defined strict_time format (HH:mm:ss.SSSZZ)
    """
    # TODO: this fxn needs to be tested on real data
    if len(dcm_time) < 6:
        return None
    hours = dcm_time[:2]
    minutes = dcm_time[2:4]
    seconds = dcm_time[4:6]
    if len(dcm_time) > 7:
        fraction_str = dcm_time[7:]
        fraction = float(dcm_time[7:])/10^(len(fraction_str))
        fraction = int(fraction*1000)
    else:
        fraction = 0
    return '%s:%s:%s.%03d00' % (hours, minutes, seconds, fraction)

def cast_datetime(dcm_datetime):
    """
    Cast DICOM datetime string (YYYYMMDDHHMMSS.FFFFFF)
    into ElasticSearch pre-defined basic_date_time format (yyyyMMdd'T'HHmmss.SSSZ)
    """
    # TODO: this fxn needs to be tested on real data
    year = dcm_datetime[:4]
    month = dcm_datetime[4:6]
    day = dcm_datetime[6:8]
    if len(dcm_datetime) > 8:
        hours = dcm_datetime[8:10]
        minutes = dcm_datetime[10:12]
        seconds = dcm_datetime[12:14]
    else:
        hours = '00'
        minutes = '00'
        seconds = '00'
    if len(dcm_datetime) > 15:
        fraction_str = dcm_datetime[15:]
        fraction = float(dcm_datetime[15:])/10^(len(fraction_str))
        fraction = int(fraction*1000)
    else:
        fraction = 0
    return '%s%s%sT%s%s%s.%03d0' % (year, month, day, hours, minutes, seconds, fraction)

def cast_age(dcm_age):
    """ Cast DICOM age string into seconds"""
    # TODO: this fxn needs to be tested on real data
    unit = dcm_age[-1]
    if unit not in ['D', 'W', 'M', 'Y']:
        return None
    multipliers = dict(D=60*60*24,
                       W=60*60*24*7,
                       M=60*60*24*30,
                       Y=60*60*24*365)
    value = int(dcm_age[:-1])
    seconds = multipliers[unit]*value
    return seconds

def value_is_array(value):
    if type(value) != unicode:
        return False
    if len(value) < 2:
        return False
    if value[0] == '[' and value[-1] == ']':
        return True
    return False

def cast_array_from_string(string):
    array = None
    try:
        array = ast.literal_eval(string)
    except:
        config.log.warn('Tried to cast string {} as array, failed.'.format(string))

    if array:
        new_array = []
        for element in array:
            try:
                element = int(element)
            except:
                try:
                    element = float(element)
                except:
                    pass
            new_array.append(element)
        return new_array
    else:
        return string

def remove_blacklisted_keys(obj):
    for key in BLACKLIST_KEYS:
        obj.pop(key, None)

def handle_files(parent, files, dicom_mappings, permissions, doc):
    doc['container_type'] = 'file'
    for f in files:
        # f.pop('info', None)
        doc['file'] = f
        # doc = {
        #     'file': f,
        #     'permissions': permissions
        # }
        # if f.get('type', '') == 'dicom' and f.get('info'):
        #     dicom_data = f.pop('info')
        #     term_fields = {}
        #     modified_data = {}
        #     for skipped in SKIPPED:
        #         dicom_data.pop(skipped, None)
        #     for k,v in dicom_data.iteritems():

        #         try:

        #             # Arrays are saved as strings in
        #             if value_is_array(v):
        #                 config.log.debug('calling array for {} and value {}'.format(k, v))
        #                 v = cast_array_from_string(v)
        #             if 'datetime' in k.lower():
        #                 config.log.debug('called datetime for {} and value {}'.format(k, v))
        #                 v = cast_datetime(str(v))
        #             elif 'date' in k.lower():
        #                 config.log.debug('called date for {} and value {}'.format(k, v))
        #                 v = cast_date(str(v))
        #             # elif 'time' in k.lower():
        #             #     # config.log.debug('called time for {} and value {}'.format(k, v))
        #             #     # v = cast_time(str(v))
        #             elif 'Age' in k:
        #                 config.log.debug('called age for {} and value {}'.format(k, v))
        #                 v = cast_age(str(v))
        #         except:
        #             pass

        #         term_field_name = k+'_term'
        #         if term_field_name in dicom_mappings and type(v) in [unicode, str]:
        #             term_fields[k+'_term'] = str(v)
        #         modified_data[k] = v

        #     modified_data.update(term_fields)
        #     doc['dicom_header'] = modified_data

        generated_id = str(parent['_id']) + '_' + f['name']

        doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
        try:
            # es.index(index=DE_INDEX, id=generated_id, parent=str(parent['_id']), doc_type='file', body=doc)
            es.index(index=DE_INDEX, id=generated_id, doc_type='flywheel', body=doc_s)
        except:
            return

if __name__ == '__main__':

    if es.indices.exists(DE_INDEX):
        print 'Removing existing data explorer index...'
        res = es.indices.delete(index=DE_INDEX)
        print 'response: {}'.format(res)

    # mappings = create_mappings()

    request = {
        'settings': {
            'number_of_shards': 1,
            'number_of_replicas': 0,
            'analysis' : ANALYSIS
        },
        'mappings': {
            '_default_' : {
                '_all' : {'enabled' : True},
                'dynamic_templates': DYNAMIC_TEMPLATES
            },
            'flywheel': {},
            # 'file': {
            #     '_parent': {
            #         'type': 'flywheel'
            #     },
            #     'properties': {
            #         'dicom_header': {
            #             'properties': mappings
            #         }
            #     }
            # }
        }
    }

    print 'creating {} index ...'.format(DE_INDEX)
    res = es.indices.create(index=DE_INDEX, body=request)
    print 'response: {}'.format(res)

    # mappings = es.indices.get_mapping(index=DE_INDEX, doc_type='flywheel')

    # dicom_mappings = mappings[DE_INDEX]['mappings']['file']['properties']['dicom_header']['properties']
    dicom_mappings = None

    permissions = []

    groups = db.groups.find({})
    for g in groups:
        remove_blacklisted_keys(g)

        projects = db.projects.find({'group': g['_id']})
        for p in projects:

            files = p.pop('files', [])
            # Set permissions for documents
            permissions = p.pop('permissions', [])
            remove_blacklisted_keys(p)

            doc = {
                'project':              p,
                'group':                g,
                'permissions':          permissions,
                'container_type':       'project'

            }

            doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
            es.index(index=DE_INDEX, id=str(p['_id']), doc_type='flywheel', body=doc_s)

            handle_files(p, files, dicom_mappings, permissions, doc)

            sessions = db.sessions.find({'project': p['_id']})
            for s in sessions:
                subject = s.pop('subject', {})

                analyses = s.pop('analyses', [])
                files = s.pop('files', [])
                remove_blacklisted_keys(s)

                doc = {
                    'project':              p,
                    'group':                g,
                    'session':              s,
                    'subject':              subject,
                    'permissions':          permissions,
                    'container_type':       'session'

                }

                doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                es.index(index=DE_INDEX, id=str(s['_id']), doc_type='flywheel', body=doc_s)

                handle_files(s, files, dicom_mappings, permissions, doc)

                for an in analyses:
                    files = an.pop('files', [])
                    doc = {
                        'analysis':             an,
                        'session':              s,
                        'subject':              subject,
                        'project':              p,
                        'group':                g,
                        'permissions':          permissions,
                        'container_type':       'analysis'

                    }

                    doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                    es.index(index=DE_INDEX, id=str(an['_id']), doc_type='flywheel', body=doc_s)

                    files = [f for f in files if f.get('output')]

                    handle_files(an, files, dicom_mappings, permissions, doc)

                acquisitions = db.acquisitions.find({'session': s['_id']})
                for a in acquisitions:
                    a.pop('info', None)
                    files = a.pop('files', [])
                    remove_blacklisted_keys(a)

                    doc = {
                        'acquisition':          a,
                        'session':              s,
                        'subject':              subject,
                        'project':              p,
                        'group':                g,
                        'permissions':          permissions,
                        'container_type':       'acquisition'

                    }

                    doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                    es.index(index=DE_INDEX, id=str(a['_id']), doc_type='flywheel', body=doc_s)

                    handle_files(a, files, dicom_mappings, permissions, doc)

<commit_msg>Up fields limit to 2000<commit_after>#!/usr/bin/env python

import ast
import copy
import dateutil.parser
import dicom
import elasticsearch
import json
import logging

from api import config
from api.web import encoder

es = config.es
db = config.db

DE_INDEX = 'data_explorer'

ANALYSIS = {
            'analyzer' : {
                'str_search_analyzer' : {
                    'tokenizer' : 'keyword',
                    'filter' : ['lowercase']
                },

                'str_index_analyzer' : {
                    'tokenizer' : 'keyword',
                    'filter' : ['lowercase', 'substring']
                }
            },
            'filter' : {
                'substring': {
                    'type': 'nGram',
                    'min_gram': 2,
                    'max_gram': 50,
                    'token_chars': []
                }
            }
        }

DYNAMIC_TEMPLATES = [
    {
            '_id': {
                'match': '_id',
                'match_mapping_type' : 'string',
                'mapping': {
                   'type': 'keyword',
                   'index': 'not_analyzed'
                }
            }
        },
    {
            'long_fields' : {
                'match_mapping_type' : 'long',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'integer_fields' : {
                'match_mapping_type' : 'integer',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'double_fields' : {
                'match_mapping_type' : 'double',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'float_fields' : {
                'match_mapping_type' : 'float',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'short_fields' : {
                'match_mapping_type' : 'short',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
    {
            'byte_fields' : {
                'match_mapping_type' : 'byte',
                'mapping' : {
                    'ignore_malformed': True
                }
            }
        },
        {
            'hash': {
                'match': 'hash',
                'match_mapping_type' : 'string',
                'mapping': {
                   'type': 'text',
                   'index': 'not_analyzed'
                }
            }
        },
        {
            'string_fields' : {
                'match': '*',
                'match_mapping_type' : 'string',
                'mapping' : {
                    'type': 'text',
                    'analyzer': 'str_search_analyzer',
                    "fields": {
                        "raw": {
                            "type": "keyword",
                            "index": "not_analyzed",
                            "ignore_above": 256
                        }
                    }
                }
            }
        }
]

def datetime(str_datetime):
    pass

def age(str_age):
    pass

BLACKLIST_KEYS = ['template', 'roles', 'permissions', 'analyses', 'files', 'collections']

SKIPPED = []
SKIPPED2ELECTRICBOOGALOO = ['PixelSpacing', 'ImageOrientationPatient', 'PatientAge', 'ImagePositionPatient']

# TODO: Choose integer/long and float/double where appropriate
VR_TYPES = {
        'AE': ['string', str],
        'AS': ['long', int],
        'AT': ['long', int],
        'CS': ['string', str],
        'DA': ['date', datetime],
        'DS': ['float', float],
        'DT': ['date', datetime],
        'FD': ['float', float],
        'FL': ['float', float],
        'IS': ['long', int],
        'LO': ['string', str],
        'LT': ['string', str],
        'NONE': None,
        'OB': None,
        'OB or OW': None,
        'OF': None,
        'OW': None,
        'PN': ['string', str],
        'SH': ['string', str],
        'SL': ['long', int],
        'SQ': None,
        'SS': ['long', int],
        'ST': ['string', str],
        'TM': ['time', datetime],
        'UI': ['string', str],
        'UL': ['long', int],
        'US': ['long', int],
        'US or OW': ['long', int],
        'US or SS': ['long', int],
        'US or SS or OW': ['long', int],
        'UT': ['string', str]
    }

def create_mappings():
    public_dict = dicom._dicom_dict.DicomDictionary
    field_mappings = {}
    for k,v in public_dict.iteritems():
        vr_type = v[0]
        field_name = v[4]
        vr_mapping = VR_TYPES.get(vr_type)
        if vr_mapping:
            field_type = vr_mapping[0]
            if field_type == 'string' and vr_type not in ['UT', 'LT', 'ST']:
                field_mappings[field_name+'_term'] = {'type': 'string', 'index': 'not_analyzed'}
            field_mappings[field_name] = {'type': field_type}
            if field_type == 'time':
                # Actually store as date, format as time:
                field_mappings[field_name] = {'type': 'date', 'format': 'time'}
        else:
            pass
            #logging.warn('Skipping field {} of VR type {}'.format(field_name, vr_type))

    return field_mappings

def cast_date(dcm_date):
    """
    Cast DICOM date string (YYYYMMDD) into ElasticSearch pre-defined strict_date format (yyyy-MM-dd)
    """
    return dcm_date[:4] + '-' + dcm_date[4:6] + '-' + dcm_date[6:]

def cast_time(dcm_time):
    """
    Cast DICOM time string (HHMMSS.FRAC)
    into ElasticSearch pre-defined strict_time format (HH:mm:ss.SSSZZ)
    """
    # TODO: this fxn needs to be tested on real data
    if len(dcm_time) < 6:
        return None
    hours = dcm_time[:2]
    minutes = dcm_time[2:4]
    seconds = dcm_time[4:6]
    if len(dcm_time) > 7:
        fraction_str = dcm_time[7:]
        fraction = float(dcm_time[7:])/10^(len(fraction_str))
        fraction = int(fraction*1000)
    else:
        fraction = 0
    return '%s:%s:%s.%03d00' % (hours, minutes, seconds, fraction)

def cast_datetime(dcm_datetime):
    """
    Cast DICOM datetime string (YYYYMMDDHHMMSS.FFFFFF)
    into ElasticSearch pre-defined basic_date_time format (yyyyMMdd'T'HHmmss.SSSZ)
    """
    # TODO: this fxn needs to be tested on real data
    year = dcm_datetime[:4]
    month = dcm_datetime[4:6]
    day = dcm_datetime[6:8]
    if len(dcm_datetime) > 8:
        hours = dcm_datetime[8:10]
        minutes = dcm_datetime[10:12]
        seconds = dcm_datetime[12:14]
    else:
        hours = '00'
        minutes = '00'
        seconds = '00'
    if len(dcm_datetime) > 15:
        fraction_str = dcm_datetime[15:]
        fraction = float(dcm_datetime[15:])/10^(len(fraction_str))
        fraction = int(fraction*1000)
    else:
        fraction = 0
    return '%s%s%sT%s%s%s.%03d0' % (year, month, day, hours, minutes, seconds, fraction)

def cast_age(dcm_age):
    """ Cast DICOM age string into seconds"""
    # TODO: this fxn needs to be tested on real data
    unit = dcm_age[-1]
    if unit not in ['D', 'W', 'M', 'Y']:
        return None
    multipliers = dict(D=60*60*24,
                       W=60*60*24*7,
                       M=60*60*24*30,
                       Y=60*60*24*365)
    value = int(dcm_age[:-1])
    seconds = multipliers[unit]*value
    return seconds

def value_is_array(value):
    if type(value) != unicode:
        return False
    if len(value) < 2:
        return False
    if value[0] == '[' and value[-1] == ']':
        return True
    return False

def cast_array_from_string(string):
    array = None
    try:
        array = ast.literal_eval(string)
    except:
        config.log.warn('Tried to cast string {} as array, failed.'.format(string))

    if array:
        new_array = []
        for element in array:
            try:
                element = int(element)
            except:
                try:
                    element = float(element)
                except:
                    pass
            new_array.append(element)
        return new_array
    else:
        return string

def remove_blacklisted_keys(obj):
    for key in BLACKLIST_KEYS:
        obj.pop(key, None)

def handle_files(parent, files, dicom_mappings, permissions, doc):
    doc['container_type'] = 'file'
    for f in files:
        # f.pop('info', None)
        doc['file'] = f
        # doc = {
        #     'file': f,
        #     'permissions': permissions
        # }
        # if f.get('type', '') == 'dicom' and f.get('info'):
        #     dicom_data = f.pop('info')
        #     term_fields = {}
        #     modified_data = {}
        #     for skipped in SKIPPED:
        #         dicom_data.pop(skipped, None)
        #     for k,v in dicom_data.iteritems():

        #         try:

        #             # Arrays are saved as strings in
        #             if value_is_array(v):
        #                 config.log.debug('calling array for {} and value {}'.format(k, v))
        #                 v = cast_array_from_string(v)
        #             if 'datetime' in k.lower():
        #                 config.log.debug('called datetime for {} and value {}'.format(k, v))
        #                 v = cast_datetime(str(v))
        #             elif 'date' in k.lower():
        #                 config.log.debug('called date for {} and value {}'.format(k, v))
        #                 v = cast_date(str(v))
        #             # elif 'time' in k.lower():
        #             #     # config.log.debug('called time for {} and value {}'.format(k, v))
        #             #     # v = cast_time(str(v))
        #             elif 'Age' in k:
        #                 config.log.debug('called age for {} and value {}'.format(k, v))
        #                 v = cast_age(str(v))
        #         except:
        #             pass

        #         term_field_name = k+'_term'
        #         if term_field_name in dicom_mappings and type(v) in [unicode, str]:
        #             term_fields[k+'_term'] = str(v)
        #         modified_data[k] = v

        #     modified_data.update(term_fields)
        #     doc['dicom_header'] = modified_data

        generated_id = str(parent['_id']) + '_' + f['name']

        doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
        try:
            # es.index(index=DE_INDEX, id=generated_id, parent=str(parent['_id']), doc_type='file', body=doc)
            es.index(index=DE_INDEX, id=generated_id, doc_type='flywheel', body=doc_s)
        except:
            return

if __name__ == '__main__':

    if es.indices.exists(DE_INDEX):
        print 'Removing existing data explorer index...'
        res = es.indices.delete(index=DE_INDEX)
        print 'response: {}'.format(res)

    # mappings = create_mappings()

    request = {
        'settings': {
            "index.mapping.total_fields.limit": 2000,
            'number_of_shards': 1,
            'number_of_replicas': 0,
            'analysis' : ANALYSIS
        },
        'mappings': {
            '_default_' : {
                '_all' : {'enabled' : True},
                'dynamic_templates': DYNAMIC_TEMPLATES
            },
            'flywheel': {},
            # 'file': {
            #     '_parent': {
            #         'type': 'flywheel'
            #     },
            #     'properties': {
            #         'dicom_header': {
            #             'properties': mappings
            #         }
            #     }
            # }
        }
    }

    print 'creating {} index ...'.format(DE_INDEX)
    res = es.indices.create(index=DE_INDEX, body=request)
    print 'response: {}'.format(res)

    # mappings = es.indices.get_mapping(index=DE_INDEX, doc_type='flywheel')

    # dicom_mappings = mappings[DE_INDEX]['mappings']['file']['properties']['dicom_header']['properties']
    dicom_mappings = None

    permissions = []

    groups = db.groups.find({})
    for g in groups:
        remove_blacklisted_keys(g)

        projects = db.projects.find({'group': g['_id']})
        for p in projects:

            files = p.pop('files', [])
            # Set permissions for documents
            permissions = p.pop('permissions', [])
            remove_blacklisted_keys(p)

            doc = {
                'project':              p,
                'group':                g,
                'permissions':          permissions,
                'container_type':       'project'

            }

            doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
            es.index(index=DE_INDEX, id=str(p['_id']), doc_type='flywheel', body=doc_s)

            handle_files(p, files, dicom_mappings, permissions, doc)

            sessions = db.sessions.find({'project': p['_id']})
            for s in sessions:
                subject = s.pop('subject', {})

                analyses = s.pop('analyses', [])
                files = s.pop('files', [])
                remove_blacklisted_keys(s)

                doc = {
                    'project':              p,
                    'group':                g,
                    'session':              s,
                    'subject':              subject,
                    'permissions':          permissions,
                    'container_type':       'session'

                }

                doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                es.index(index=DE_INDEX, id=str(s['_id']), doc_type='flywheel', body=doc_s)

                handle_files(s, files, dicom_mappings, permissions, doc)

                for an in analyses:
                    files = an.pop('files', [])
                    doc = {
                        'analysis':             an,
                        'session':              s,
                        'subject':              subject,
                        'project':              p,
                        'group':                g,
                        'permissions':          permissions,
                        'container_type':       'analysis'

                    }

                    doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                    es.index(index=DE_INDEX, id=str(an['_id']), doc_type='flywheel', body=doc_s)

                    files = [f for f in files if f.get('output')]

                    handle_files(an, files, dicom_mappings, permissions, doc)

                acquisitions = db.acquisitions.find({'session': s['_id']})
                for a in acquisitions:
                    a.pop('info', None)
                    files = a.pop('files', [])
                    remove_blacklisted_keys(a)

                    doc = {
                        'acquisition':          a,
                        'session':              s,
                        'subject':              subject,
                        'project':              p,
                        'group':                g,
                        'permissions':          permissions,
                        'container_type':       'acquisition'

                    }

                    doc_s = json.dumps(doc, default=encoder.custom_json_serializer)
                    es.index(index=DE_INDEX, id=str(a['_id']), doc_type='flywheel', body=doc_s)

                    handle_files(a, files, dicom_mappings, permissions, doc)

<|endoftext|>


<commit_before># coding=utf-8
#This file is part of QNET.
#
#    QNET is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#    QNET is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with QNET.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2012-2013, Nikolas Tezak
#
###########################################################################
"""
QNET contains symbolic algebra packages to handle:
 - quantum operators --> `qnet.algebra.operator_algebra`
 - quantum feedback circuits --> `qnet.algebra.circuit_algebra`
 - quantum super operators --> `qnet.algebra.super_operator_algebra`
 - quantum states --> `qnet.algebra.state_algebra`

It also features a parser for quantum feedback circuits specified
via Quantum Hardware Description Language:

    parse_qhdl.py -f MyCircuit.qhdl [-l|-L]

See our full docs for more information.
"""
__version__ = "1.2.4"
<commit_msg>bumped version<commit_after># coding=utf-8
#This file is part of QNET.
#
#    QNET is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#    QNET is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with QNET.  If not, see <http://www.gnu.org/licenses/>.
#
# Copyright (C) 2012-2013, Nikolas Tezak
#
###########################################################################
"""
QNET contains symbolic algebra packages to handle:
 - quantum operators --> `qnet.algebra.operator_algebra`
 - quantum feedback circuits --> `qnet.algebra.circuit_algebra`
 - quantum super operators --> `qnet.algebra.super_operator_algebra`
 - quantum states --> `qnet.algebra.state_algebra`

It also features a parser for quantum feedback circuits specified
via Quantum Hardware Description Language:

    parse_qhdl.py -f MyCircuit.qhdl [-l|-L]

See our full docs for more information.
"""
__version__ = "1.2.5"
<|endoftext|>


<commit_before>import numpy

try:
    from vigra import vigranumpycmodule as vm
except ImportError:
    try:
        import vigranumpycmodule as vm
    except ImportError:
        sys.exit("vigranumpycmodule not found!")

class SegmentationBase(object):
    def __init__(self):
        self.smoothing = ''
        pass

    def segment(self):
        pass

class LocallyDominantSegmentation2D(SegmentationBase):
    def __init__(self, shape, sigma=2.0, smoothing='Gaussian'):
        SegmentationBase.__init__(self)
        self.smoothing = smoothing
        self.sigma = sigma
        self.shape = list(shape[0:2])
        self.shape.append(-1)
        self.result = None

    def segment(self, propmap):
        if not propmap.dtype == numpy.float32:
            propmap = propmap.astype(numpy.float32)

        propmap_ = propmap.reshape(self.shape)
        print propmap_.shape
        if self.smoothing in ['Gaussian']:
            res = numpy.zeros( propmap_.shape, dtype=numpy.float32)
            for k in range(0, propmap_.shape[2]):
                res[:,:,k] = vm.gaussianSmooth2d(propmap_[:,:,k], self.sigma)
        else:
            print "Invalid option for smoothing: %s" % self.smoothing
            return None

        self.result = numpy.argmax(res, axis=2)
        #vm.writeImage(self.result.astype(numpy.uint8),'c:/il_seg.jpg')

if __name__ == "__main__":
    a = numpy.random.rand(256,256,4)
    s = LocallyDominantSegmentation()
    r = s.segment(a)
    print r
        <commit_msg>update to new vigra structure<commit_after>import numpy

try:
    import vigra
except ImportError:
    sys.exit("vigra module not found!")

class SegmentationBase(object):
    def __init__(self):
        self.smoothing = ''
        pass

    def segment(self):
        pass

class LocallyDominantSegmentation2D(SegmentationBase):
    def __init__(self, shape, sigma=2.0, smoothing='Gaussian'):
        SegmentationBase.__init__(self)
        self.smoothing = smoothing
        self.sigma = sigma
        self.shape = list(shape[0:2])
        self.shape.append(-1)
        self.result = None

    def segment(self, propmap):
        if not propmap.dtype == numpy.float32:
            propmap = propmap.astype(numpy.float32)

        propmap_ = propmap.reshape(self.shape)
        print propmap_.shape
        if self.smoothing in ['Gaussian']:
            res = numpy.zeros( propmap_.shape, dtype=numpy.float32)
            for k in range(0, propmap_.shape[2]):
                res[:,:,k] = vigra.filters.gaussianSmooth2d(propmap_[:,:,k], self.sigma)
        else:
            print "Invalid option for smoothing: %s" % self.smoothing
            return None

        self.result = numpy.argmax(res, axis=2)
        #vm.writeImage(self.result.astype(numpy.uint8),'c:/il_seg.jpg')

if __name__ == "__main__":
    a = numpy.random.rand(256,256,4)
    s = LocallyDominantSegmentation()
    r = s.segment(a)
    print r
        <|endoftext|>


<commit_before>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions
UseExceptions()

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_with_missing_file(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/fake.unw.rsc"
		self.assertRaises(IOError, roipac.to_ehdr_header, src)

	def test_to_ehdr_header_with_dir(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs"
		self.assertRaises(IOError, roipac.to_ehdr_header, src)

	def test_to_ehdr_header_with_dem(self):
		dem_hdr = "../../tests/dem/sydney_trimmed.dem.rsc"
		act = roipac.to_ehdr_header(dem_hdr)
		self.assertEqual(act, dem_hdr[:-7] + "hdr")

		with open(act) as f:
			lines = [line.strip() for line in f.readlines()]
			values = [line.split() for line in lines]

		self.assertTrue(['ncols', '47'] in values)
		self.assertTrue(['nrows', '72'] in values)

		# TODO: test different forms of cellsize
		self.assertTrue(['cellsize', '0.000833333'] in values)
		#self.assertTrue(['xdim', '0.000277777777782'] in values)
		#self.assertTrue(['ydim', '-0.000277777777782'] in values)

		self.assertFalse(['nodata', '0'] in values)
		self.assertFalse(['nbands', '1'] in values)
		self.assertTrue(['byteorder', 'lsb'] in values)
		self.assertFalse(['layout', 'bil'] in values)
		self.assertTrue(['nbits', '16'] in values)
		self.assertTrue(['pixeltype', 'signedint'] in values)
		os.remove(act)

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA

		# ensure decent data is retrieved
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0) # ignore max as NODATA is 0
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<commit_msg>Refactored location of sydney_test DEM.<commit_after>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions
UseExceptions()

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_with_missing_file(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/fake.unw.rsc"
		self.assertRaises(IOError, roipac.to_ehdr_header, src)

	def test_to_ehdr_header_with_dir(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs"
		self.assertRaises(IOError, roipac.to_ehdr_header, src)

	def test_to_ehdr_header_with_dem(self):
		dem_hdr = "../../tests/sydney_test/dem/sydney_trimmed.dem.rsc"
		act = roipac.to_ehdr_header(dem_hdr)
		self.assertEqual(act, dem_hdr[:-7] + "hdr")

		with open(act) as f:
			lines = [line.strip() for line in f.readlines()]
			values = [line.split() for line in lines]

		self.assertTrue(['ncols', '47'] in values)
		self.assertTrue(['nrows', '72'] in values)

		# TODO: test different forms of cellsize
		self.assertTrue(['cellsize', '0.000833333'] in values)
		#self.assertTrue(['xdim', '0.000277777777782'] in values)
		#self.assertTrue(['ydim', '-0.000277777777782'] in values)

		self.assertFalse(['nodata', '0'] in values)
		self.assertFalse(['nbands', '1'] in values)
		self.assertTrue(['byteorder', 'lsb'] in values)
		self.assertFalse(['layout', 'bil'] in values)
		self.assertTrue(['nbits', '16'] in values)
		self.assertTrue(['pixeltype', 'signedint'] in values)
		os.remove(act)

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA

		# ensure decent data is retrieved
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0) # ignore max as NODATA is 0
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<|endoftext|>