<commit_before>'''
###############################################################################
Controller:  Overall simulation controller class
###############################################################################
'''
import pickle as _pickle

class Controller(dict):
    r"""

    """
    # The following __instance__ class variable and subclassed __new__ method
    # makes the Controller class a 'Singleton'.  This way, the _sim attribute
    # of every OpenPNM object is the same, AND if you create a sim on the
    # command line (sim = OpenPNM.Base.Controller()) it will be the same sim!
    __instance__ = None
    def __new__(cls, *args,**kwargs):
        if Controller.__instance__ is None:
            Controller.__instance__ = dict.__new__(cls)
        return Controller.__instance__

    def __init__(self):
        r'''
        '''
        pass

    def __str__(self):
        header = ('-'*60)
        print(header)
        print("{a:<25s} {b:<25s}".format(a='Class', b='Object Name'))
        print(header)
        for item in self.keys():
            print("{a:<25s} {b:<25s}".format(a=self[item].__class__.__name__, b=item))
        return ''

    def network(self):
        r'''
        Returns a list of all Network objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericNetwork')

    def geometries(self):
        r'''
        Returns a list of all Geometry objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericGeometry')

    def phases(self):
        r'''
        Returns a list of all Phase objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericPhase')

    def physics(self):
        r'''
        Returns a list of all Physics objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericPhysics')

    def algorithms(self):
        r'''
        Returns a list of all Algorithm objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericAlgorithm')

    def _get_objects(self,obj_type):
        temp = []
        for obj in self.keys():
            mro = [item.__name__ for item in self[obj].__class__.__mro__]
            if obj_type in mro:
                temp.append(self[obj])
        return temp

    def clear(self):
        r'''
        This is an overloaded version of the standard dict's ``clear`` method.
        This completely clears the Controller object's dict as expected, but
        also removes links to the Controller object in all simulation objects.

        Notes
        -----
        When removing links in all simulation objects to the Controller it
        replaces their ``simulation`` attribute with a standard ``dict``.  This
        ``dict`` can be reassociated with a Controller object using ``update``,
        which has been overloaded to do so.
        '''
        temp = {}
        for item in self.keys():
            self[item].simulation = temp
        self.__dict__ = {}
        super(Controller,self).clear()

    def update(self,dict_):
        r'''
        This is an overloaded version of the standard dict's ``update`` method.
        It accepts a dictionary argument, which is injects into the Controller
        object, but is also associates the Controller object with all simulation
        objects that were in the received dict.

        Parameters
        ----------
        dict_ : dictionary
            A Python dictionary contain {key : value} pairs in the form of
            {obj.name : obj}.

        Notes
        -----
        When the ``clear`` method of a Controller object is called, it sets the
        ``simulation`` attribute of all simulation objects to a standard
        ``dict``.  The overloading of this method allows such a ``dict`` to be
        reassociated with a Controller object.
        '''
        super(Controller,self).update(dict_)
        for item in self.keys():
            self[item]._sim = self

    def purge_object(self,obj):
        r'''
        Remove an object from the simulation

        Parameters
        ----------
        obj : OpenPNM Object
            The object to be removed from the simulation.  This method removes
            all traces of the object from everywhere in the simulation,
            including all the object tracking lists and label dictionaries of
            every object.

        Notes
        -----
        To only remove an object from the Contoller object, without purging all
        traces from the simulation, use the dictionary's native ``pop`` method.
        '''
        name = obj.name
        for item in self.keys():
            # Remove label arrays from all other objects
            self[item].pop('pore.'+name,None)
            self[item].pop('throat.'+name,None)
            # Remove associations on other objects
            self[item]._geometries[:] = [x for x in self[item]._geometries if x is not obj]
            self[item]._phases[:] = [x for x in self[item]._phases if x is not obj]
            self[item]._physics[:] = [x for x in self[item]._physics if x is not obj]
        # Set object's simulation to an empty dict
        self[name]._sim = {}
        # Remove object from simulation dict
        del self[name]

    def clone_object(self,obj):
        r'''
        Clone an OpenPNM Object, without associating the new object with the
        parent simulation.

        Parameters
        ----------
        obj : OpenPNM Object
            The object to be cloned can be any OpenPNM Object

        Returns
        -------
        A clone of the specified object is returned, but it retains all its links
        to the objects associated with the original object.  The cloned object is
        not associated with the Network.

        Notes
        -----
        This method is intended to create a disposable object, for instance, to
        receive simulation data without overwriting existing data.

        '''
        a = _pickle.dumps(obj)
        obj_new = _pickle.loads(a)
        super(Controller,self).update(obj)
        return obj_new

    def save_object(self,obj,filename=''):
        r'''
        Save a single OpenPNM object to a 'pno' file.

        Parameters
        ----------
        obj : OpenPNM object
            The object to save.  All the associations are removed, so upon
            reloading the object needs to be reconnected manually to a
            simulation.
        filename : string (optional)
            The file name to use when saving.  If no name is given the object
            name is used.
        '''
        if filename == '':
            filename = obj.name
        else:
            filename = filename.split('.')[0]

        obj._sim = {}
        obj._net = []
        obj._geometries = []
        obj._physics = []
        obj._phases = []

        #Save nested dictionary pickle
        _pickle.dump(obj,open(filename+'.pno','wb'))

    def load_object(self,filename):
        r'''
        Load a single object saved as a 'pno' file.

        Parameters
        ----------
        filename : string
            The file name to load.  The file extension must be 'pno'.
        '''
        filename = filename.split('.')[0]
        obj = _pickle.load(open(filename+'.pno','rb'))
        obj.simulation = self

    def save(self,filename=''):
        r'''
        Save the entire state of a simulation to a 'pnm' file.

        Parameters
        ----------
        filename : string, optional
            The file name to save as. If none is given the name of the Network
            object is used.
        '''
        if filename == '':
            filename = self.network()[0].name
        else:
            filename = filename.split('.')[0]

        #Save nested dictionary pickle
        _pickle.dump(self,open(filename+'.pnm','wb'))

    def load(self,filename):
        r'''
        Load an entire simulation from a 'pnm' file.

        Parameters
        ----------
        filename : string
            The file name of the simulation to load.

        Notes
        -----
        This calls the ``clear`` method of the Controller object, so it will
        over write the calling objects information AND remove any references
        to the calling object from existing simulation objects.
        '''
        filename = filename.split('.')[0]
        if self != {}:
            print('Warning: Loading data onto non-empty controller object, existing data will be lost')
            self.clear()
        self = _pickle.load(open(filename+'.pnm','rb'))

    def export(self,filename='',fileformat='VTK'):
        r'''
        Export simulation data to the specified file format.

        Parameters
        ----------
        filename : string, optional
            The file name to save as.  If no name is given then the name of
            suppiled object is used.  If no object is given, the name of the
            Network is used.
        fileformat : string
            The type of file to create.  Options are:

            1. VTK: Suitable for visualizing in VTK capable software such as Paraview
            2. MAT: Suitable for loading data into Matlab for post-processing

        '''
        import OpenPNM.Utilities.IO as io
        if fileformat == 'VTK':
            net = self.network()[0]
            phases = net._phases
            io.VTK.save(filename=filename,network=net,phases=phases)
            return
        if fileformat == 'MAT':
            net = self.network()[0]
            phases = net._phases
            io.MAT.save(filename=filename,network=net,phases=phases)
            return

if __name__ == '__main__':
    sim = Controller()

<commit_msg>Playing around with printing a hierarchy of objects<commit_after>'''
###############################################################################
Controller:  Overall simulation controller class
###############################################################################
'''
import pickle as _pickle

class Controller(dict):
    r"""

    """
    # The following __instance__ class variable and subclassed __new__ method
    # makes the Controller class a 'Singleton'.  This way, the _sim attribute
    # of every OpenPNM object is the same, AND if you create a sim on the
    # command line (sim = OpenPNM.Base.Controller()) it will be the same sim!
    __instance__ = None
    def __new__(cls, *args,**kwargs):
        if Controller.__instance__ is None:
            Controller.__instance__ = dict.__new__(cls)
        return Controller.__instance__

    def __init__(self):
        r'''
        '''
        pass

    def __str__(self):
        header = ('-'*60)
        print(header)
        print("{a:<25s} {b:<25s}".format(a='Class', b='Object Name'))
        print(header)
        for item in self.keys():
            print("{a:<25s} {b:<25s}".format(a=self[item].__class__.__name__, b=item))
        return ''

    def show_tree(self):
        r'''
        Prints a heirarchical list of object associations
        '''
        header = ('-'*60)
        net = self.network()[0]
        print(header)
        print('Network: '+net.name)
        for geom in self.geometries():
            print('+ '+'Geometry: '+geom.name)
        for phase in self.phases():
            if len(phase.phases())==0:
                print('+ '+'Pure Phase: '+phase.name)
            if len(phase.phases())>1:
                print('+ '+'Mixture Phase: '+phase.name)
                comps = phase.phases()
                for compname in comps:
                    print('+++ '+'Component Phase: '+self[compname].name)
            for phys in self.physics():
                if phase in phys._phases:
                    print('++ '+'Physics: '+phys.name)

    def network(self):
        r'''
        Returns a list of all Network objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericNetwork')

    def geometries(self):
        r'''
        Returns a list of all Geometry objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericGeometry')

    def phases(self):
        r'''
        Returns a list of all Phase objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericPhase')

    def physics(self):
        r'''
        Returns a list of all Physics objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericPhysics')

    def algorithms(self):
        r'''
        Returns a list of all Algorithm objects in the simulation.
        '''
        return self._get_objects(obj_type='GenericAlgorithm')

    def _get_objects(self,obj_type):
        temp = []
        for obj in self.keys():
            mro = [item.__name__ for item in self[obj].__class__.__mro__]
            if obj_type in mro:
                temp.append(self[obj])
        return temp

    def clear(self):
        r'''
        This is an overloaded version of the standard dict's ``clear`` method.
        This completely clears the Controller object's dict as expected, but
        also removes links to the Controller object in all simulation objects.

        Notes
        -----
        When removing links in all simulation objects to the Controller it
        replaces their ``simulation`` attribute with a standard ``dict``.  This
        ``dict`` can be reassociated with a Controller object using ``update``,
        which has been overloaded to do so.
        '''
        temp = {}
        for item in self.keys():
            self[item].simulation = temp
        self.__dict__ = {}
        super(Controller,self).clear()

    def update(self,dict_):
        r'''
        This is an overloaded version of the standard dict's ``update`` method.
        It accepts a dictionary argument, which is injects into the Controller
        object, but is also associates the Controller object with all simulation
        objects that were in the received dict.

        Parameters
        ----------
        dict_ : dictionary
            A Python dictionary contain {key : value} pairs in the form of
            {obj.name : obj}.

        Notes
        -----
        When the ``clear`` method of a Controller object is called, it sets the
        ``simulation`` attribute of all simulation objects to a standard
        ``dict``.  The overloading of this method allows such a ``dict`` to be
        reassociated with a Controller object.
        '''
        super(Controller,self).update(dict_)
        for item in self.keys():
            self[item]._sim = self

    def purge_object(self,obj):
        r'''
        Remove an object from the simulation

        Parameters
        ----------
        obj : OpenPNM Object
            The object to be removed from the simulation.  This method removes
            all traces of the object from everywhere in the simulation,
            including all the object tracking lists and label dictionaries of
            every object.

        Notes
        -----
        To only remove an object from the Contoller object, without purging all
        traces from the simulation, use the dictionary's native ``pop`` method.
        '''
        name = obj.name
        for item in self.keys():
            # Remove label arrays from all other objects
            self[item].pop('pore.'+name,None)
            self[item].pop('throat.'+name,None)
            # Remove associations on other objects
            self[item]._geometries[:] = [x for x in self[item]._geometries if x is not obj]
            self[item]._phases[:] = [x for x in self[item]._phases if x is not obj]
            self[item]._physics[:] = [x for x in self[item]._physics if x is not obj]
        # Set object's simulation to an empty dict
        self[name]._sim = {}
        # Remove object from simulation dict
        del self[name]

    def clone_object(self,obj):
        r'''
        Clone an OpenPNM Object, without associating the new object with the
        parent simulation.

        Parameters
        ----------
        obj : OpenPNM Object
            The object to be cloned can be any OpenPNM Object

        Returns
        -------
        A clone of the specified object is returned, but it retains all its links
        to the objects associated with the original object.  The cloned object is
        not associated with the Network.

        Notes
        -----
        This method is intended to create a disposable object, for instance, to
        receive simulation data without overwriting existing data.

        '''
        a = _pickle.dumps(obj)
        obj_new = _pickle.loads(a)
        super(Controller,self).update(obj)
        return obj_new

    def save_object(self,obj,filename=''):
        r'''
        Save a single OpenPNM object to a 'pno' file.

        Parameters
        ----------
        obj : OpenPNM object
            The object to save.  All the associations are removed, so upon
            reloading the object needs to be reconnected manually to a
            simulation.
        filename : string (optional)
            The file name to use when saving.  If no name is given the object
            name is used.
        '''
        if filename == '':
            filename = obj.name
        else:
            filename = filename.split('.')[0]

        obj._sim = {}
        obj._net = []
        obj._geometries = []
        obj._physics = []
        obj._phases = []

        #Save nested dictionary pickle
        _pickle.dump(obj,open(filename+'.pno','wb'))

    def load_object(self,filename):
        r'''
        Load a single object saved as a 'pno' file.

        Parameters
        ----------
        filename : string
            The file name to load.  The file extension must be 'pno'.
        '''
        filename = filename.split('.')[0]
        obj = _pickle.load(open(filename+'.pno','rb'))
        obj.simulation = self

    def save(self,filename=''):
        r'''
        Save the entire state of a simulation to a 'pnm' file.

        Parameters
        ----------
        filename : string, optional
            The file name to save as. If none is given the name of the Network
            object is used.
        '''
        if filename == '':
            filename = self.network()[0].name
        else:
            filename = filename.split('.')[0]

        #Save nested dictionary pickle
        _pickle.dump(self,open(filename+'.pnm','wb'))

    def load(self,filename):
        r'''
        Load an entire simulation from a 'pnm' file.

        Parameters
        ----------
        filename : string
            The file name of the simulation to load.

        Notes
        -----
        This calls the ``clear`` method of the Controller object, so it will
        over write the calling objects information AND remove any references
        to the calling object from existing simulation objects.
        '''
        filename = filename.split('.')[0]
        if self != {}:
            print('Warning: Loading data onto non-empty controller object, existing data will be lost')
            self.clear()
        self = _pickle.load(open(filename+'.pnm','rb'))

    def export(self,filename='',fileformat='VTK'):
        r'''
        Export simulation data to the specified file format.

        Parameters
        ----------
        filename : string, optional
            The file name to save as.  If no name is given then the name of
            suppiled object is used.  If no object is given, the name of the
            Network is used.
        fileformat : string
            The type of file to create.  Options are:

            1. VTK: Suitable for visualizing in VTK capable software such as Paraview
            2. MAT: Suitable for loading data into Matlab for post-processing

        '''
        import OpenPNM.Utilities.IO as io
        if fileformat == 'VTK':
            net = self.network()[0]
            phases = net._phases
            io.VTK.save(filename=filename,network=net,phases=phases)
            return
        if fileformat == 'MAT':
            net = self.network()[0]
            phases = net._phases
            io.MAT.save(filename=filename,network=net,phases=phases)
            return

if __name__ == '__main__':
    sim = Controller()

<|endoftext|>


<commit_before>#!/usr/bin/env python2

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage archivematicaClientScript
# @author Joseph Perry <joseph@artefactual.com>
import re
import os
from lxml import etree as etree
import sys
import traceback
import uuid
import string
from datetime import datetime
from datetime import timedelta
from dateutil.relativedelta import relativedelta

import django
django.setup()
# dashboard
from main.models import RightsStatement, RightsStatementOtherRightsInformation, RightsStatementOtherRightsDocumentationIdentifier, RightsStatementRightsGranted, RightsStatementRightsGrantedNote, RightsStatementRightsGrantedRestriction

# archivematicaCommon
from fileOperations import getFileUUIDLike
import databaseFunctions

while False:
    import time
    time.sleep(10)

transferUUID = sys.argv[1]
transferName = sys.argv[2]
transferPath = sys.argv[3]
date = sys.argv[4]

currentDirectory = ""
exitCode = 0

def callWithException(exception):
    traceback

def getTimedeltaFromRetensionSchedule(RetentionSchedule):
    ret = 0
    rs = ["0"]
    rss = RetentionSchedule.split(".")
    for part in rss:
        entry = "0"
        for c in part:
            if c in string.digits:
                entry = "%s%s" % (entry, c)
        rs.append(entry)
    for entry in rs:
        ret += int (entry)

    ret = relativedelta(years=ret)
    return ret

def getDateTimeFromDateClosed(dateClosed):
    i = 19 #the + or minus for offset (DST + timezone)
    if dateClosed== None:
        return

    dateClosedDT = datetime.strptime(dateClosed[:i], '%Y-%m-%dT%H:%M:%S')
    print dateClosedDT
    offSet = dateClosed[i+1:].split(":")
    offSetTD = timedelta(hours=int(offSet[0]), minutes=int(offSet[1]))

    if dateClosed[i] == "-":
        dateClosedDT = dateClosedDT - offSetTD
    elif  dateClosed[i] == "+":
        dateClosedDT = dateClosedDT + offSetTD
    else:
        print >>sys.stderr,"Error with offset in:", dateClosed
        return dateClosedDT
    return dateClosedDT

for dir in os.listdir(transferPath):
    dirPath = os.path.join(transferPath, dir)
    if not os.path.isdir(dirPath):
        continue

    xmlFilePath = os.path.join(dirPath, "ContainerMetadata.xml")
    try:
        tree = etree.parse(xmlFilePath)
        root = tree.getroot()
    except:
        print >>sys.stderr, "Error parsing: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1)
        exitCode += 1
        continue
    try:
        RetentionSchedule = root.find("Container/RetentionSchedule").text
        DateClosed = root.find("Container/DateClosed").text
    except:
        print >>sys.stderr, "Error retrieving values from: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1)
        exitCode += 1
        continue

    retentionPeriod = getTimedeltaFromRetensionSchedule(RetentionSchedule)
    startTime = getDateTimeFromDateClosed(DateClosed)
    endTime = startTime + retentionPeriod

    # make end time end of year
    endTimeEndOfYearDiff = datetime(endTime.year, 12, 31) - endTime
    endTime = endTime + endTimeEndOfYearDiff

    indexForOnlyDate = 10
    startTime = startTime.__str__()[:indexForOnlyDate]
    endTime = endTime.__str__()[:indexForOnlyDate]

    for file in os.listdir(dirPath):
        filePath = os.path.join(dirPath, file)
        if  file == "ContainerMetadata.xml" or file.endswith("Metadata.xml") or not os.path.isfile(filePath):
            continue

        fileUUID = getFileUUIDLike(filePath, transferPath, transferUUID, "transferUUID", "%transferDirectory%")[filePath.replace(transferPath, "%transferDirectory%", 1)]
        FileMetadataAppliesToType = '7f04d9d4-92c2-44a5-93dc-b7bfdf0c1f17'

        # RightsStatement
        statement = RightsStatement.objects.create(
            metadataappliestotype_id=FileMetadataAppliesToType,
            metadataappliestoidentifier=fileUUID,
            rightsstatementidentifiertype="UUID",
            rightsstatementidentifiervalue=str(uuid.uuid4(),
            rightsholder=1,
            rightsbasis="Other"
        )

        # RightsStatementOtherRightsInformation
        info = RightsStatementOtherRightsInformation.objects.create(
           rightsstatement=statement,
           otherrightsbasis="Policy"
        )

        # RightsStatementOtherRightsDocumentationIdentifier
        identifier = RightsStatementOtherRightsDocumentationIdentifier.objects.create(
            rightsstatementotherrights=info
        )

        # RightsStatementRightsGranted
        granted = RightsStatementRightsGranted.objects.create(
            rightsstatement=statement,
            act="Disseminate",
            startdate=startTime,
            enddate=endTime
        )

        # RightsStatementRightsGrantedNote
        RightsStatementRightsGrantedNote.objects.create(
            rightsgranted=granted,
            rightsgrantednote="Closed until " + endTime
        )

        # RightsStatementRightsGrantedRestriction
        RightsStatementRightsGrantedRestriction.objects.create(
            rightsgranted=granted,
            restriction="Disallow"
        )

quit(exitCode)
<commit_msg>Fix typo in trimCreateRightsEntries<commit_after>#!/usr/bin/env python2

# This file is part of Archivematica.
#
# Copyright 2010-2013 Artefactual Systems Inc. <http://artefactual.com>
#
# Archivematica is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Archivematica is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Archivematica.  If not, see <http://www.gnu.org/licenses/>.

# @package Archivematica
# @subpackage archivematicaClientScript
# @author Joseph Perry <joseph@artefactual.com>
import re
import os
from lxml import etree as etree
import sys
import traceback
import uuid
import string
from datetime import datetime
from datetime import timedelta
from dateutil.relativedelta import relativedelta

import django
django.setup()
# dashboard
from main.models import RightsStatement, RightsStatementOtherRightsInformation, RightsStatementOtherRightsDocumentationIdentifier, RightsStatementRightsGranted, RightsStatementRightsGrantedNote, RightsStatementRightsGrantedRestriction

# archivematicaCommon
from fileOperations import getFileUUIDLike
import databaseFunctions

while False:
    import time
    time.sleep(10)

transferUUID = sys.argv[1]
transferName = sys.argv[2]
transferPath = sys.argv[3]
date = sys.argv[4]

currentDirectory = ""
exitCode = 0

def callWithException(exception):
    traceback

def getTimedeltaFromRetensionSchedule(RetentionSchedule):
    ret = 0
    rs = ["0"]
    rss = RetentionSchedule.split(".")
    for part in rss:
        entry = "0"
        for c in part:
            if c in string.digits:
                entry = "%s%s" % (entry, c)
        rs.append(entry)
    for entry in rs:
        ret += int (entry)

    ret = relativedelta(years=ret)
    return ret

def getDateTimeFromDateClosed(dateClosed):
    i = 19 #the + or minus for offset (DST + timezone)
    if dateClosed== None:
        return

    dateClosedDT = datetime.strptime(dateClosed[:i], '%Y-%m-%dT%H:%M:%S')
    print dateClosedDT
    offSet = dateClosed[i+1:].split(":")
    offSetTD = timedelta(hours=int(offSet[0]), minutes=int(offSet[1]))

    if dateClosed[i] == "-":
        dateClosedDT = dateClosedDT - offSetTD
    elif  dateClosed[i] == "+":
        dateClosedDT = dateClosedDT + offSetTD
    else:
        print >>sys.stderr,"Error with offset in:", dateClosed
        return dateClosedDT
    return dateClosedDT

for dir in os.listdir(transferPath):
    dirPath = os.path.join(transferPath, dir)
    if not os.path.isdir(dirPath):
        continue

    xmlFilePath = os.path.join(dirPath, "ContainerMetadata.xml")
    try:
        tree = etree.parse(xmlFilePath)
        root = tree.getroot()
    except:
        print >>sys.stderr, "Error parsing: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1)
        exitCode += 1
        continue
    try:
        RetentionSchedule = root.find("Container/RetentionSchedule").text
        DateClosed = root.find("Container/DateClosed").text
    except:
        print >>sys.stderr, "Error retrieving values from: ", xmlFilePath.replace(transferPath, "%transferDirectory%", 1)
        exitCode += 1
        continue

    retentionPeriod = getTimedeltaFromRetensionSchedule(RetentionSchedule)
    startTime = getDateTimeFromDateClosed(DateClosed)
    endTime = startTime + retentionPeriod

    # make end time end of year
    endTimeEndOfYearDiff = datetime(endTime.year, 12, 31) - endTime
    endTime = endTime + endTimeEndOfYearDiff

    indexForOnlyDate = 10
    startTime = startTime.__str__()[:indexForOnlyDate]
    endTime = endTime.__str__()[:indexForOnlyDate]

    for file in os.listdir(dirPath):
        filePath = os.path.join(dirPath, file)
        if  file == "ContainerMetadata.xml" or file.endswith("Metadata.xml") or not os.path.isfile(filePath):
            continue

        fileUUID = getFileUUIDLike(filePath, transferPath, transferUUID, "transferUUID", "%transferDirectory%")[filePath.replace(transferPath, "%transferDirectory%", 1)]
        FileMetadataAppliesToType = '7f04d9d4-92c2-44a5-93dc-b7bfdf0c1f17'

        # RightsStatement
        statement = RightsStatement.objects.create(
            metadataappliestotype_id=FileMetadataAppliesToType,
            metadataappliestoidentifier=fileUUID,
            rightsstatementidentifiertype="UUID",
            rightsstatementidentifiervalue=str(uuid.uuid4()),
            rightsholder=1,
            rightsbasis="Other"
        )

        # RightsStatementOtherRightsInformation
        info = RightsStatementOtherRightsInformation.objects.create(
           rightsstatement=statement,
           otherrightsbasis="Policy"
        )

        # RightsStatementOtherRightsDocumentationIdentifier
        identifier = RightsStatementOtherRightsDocumentationIdentifier.objects.create(
            rightsstatementotherrights=info
        )

        # RightsStatementRightsGranted
        granted = RightsStatementRightsGranted.objects.create(
            rightsstatement=statement,
            act="Disseminate",
            startdate=startTime,
            enddate=endTime
        )

        # RightsStatementRightsGrantedNote
        RightsStatementRightsGrantedNote.objects.create(
            rightsgranted=granted,
            rightsgrantednote="Closed until " + endTime
        )

        # RightsStatementRightsGrantedRestriction
        RightsStatementRightsGrantedRestriction.objects.create(
            rightsgranted=granted,
            restriction="Disallow"
        )

quit(exitCode)
<|endoftext|>


<commit_before>VERSION = 50  # Please update this whenever .client or .services changes
# Please update MIN_VERSION_OVERRIDE and MIN_CLIENT_VERSION_OVERRIDE as needed
<commit_msg>client: Bump version 51<commit_after>VERSION = 51  # Please update this whenever .client or .services changes
# Please update MIN_VERSION_OVERRIDE and MIN_CLIENT_VERSION_OVERRIDE as needed
<|endoftext|>


<commit_before>"""Plot what was saved from daily_clifile_editor processing"""
from pyiem.plot import MapPlot
import sys
import datetime
import numpy as np
import os
SOUTH = 36.9
NORTH = 44.9
EAST = -90.0
WEST = -99.2
YS = np.arange(SOUTH, NORTH, 0.01)
XS = np.arange(WEST, EAST, 0.01)

def load_precip(date):
    """Load up our QC'd daily precip dataset """
    fn = date.strftime("/mnt/idep2/data/dailyprecip/%Y/%Y%m%d.npy")
    if not os.path.isfile(fn):
        print("load_precip(%s) failed, no such file" % (date,))
        return
    return np.load(fn)

def do(valid):
    """Do Something"""
    precip = load_precip(valid)

    m = MapPlot(sector='midwest', axisbg='white')
    (lats, lons) = np.meshgrid(YS, XS)
    print np.shape(precip), np.shape(lons), np.shape(lats)
    m.pcolormesh(lons, lats, np.transpose(precip), np.arange(20))
    m.postprocess(view=True)

def main(argv):
    valid = datetime.date(int(argv[1]), int(argv[2]),
                          int(argv[3]))
    do(valid)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>improve diagnostic plot of QC precip<commit_after>"""Plot what was saved from daily_clifile_editor processing"""
from pyiem.plot import MapPlot
import sys
import datetime
import numpy as np
import os
SOUTH = 36.9
NORTH = 44.9
EAST = -90.0
WEST = -99.2
YS = np.arange(SOUTH, NORTH, 0.01)
XS = np.arange(WEST, EAST, 0.01)

def load_precip(date):
    """Load up our QC'd daily precip dataset """
    fn = date.strftime("/mnt/idep2/data/dailyprecip/%Y/%Y%m%d.npy")
    if not os.path.isfile(fn):
        print("load_precip(%s) failed, no such file" % (date,))
        return
    return np.load(fn)

def do(valid):
    """Do Something"""
    precip = load_precip(valid)
    clevs = np.arange(0, 0.25, 0.05)
    clevs = np.append(clevs, np.arange(0.25, 3., 0.25))
    clevs = np.append(clevs, np.arange(3., 10.0, 1))
    clevs[0] = 0.01

    m = MapPlot(sector='iowa', axisbg='white',
                title='24 Jun 2015 DEP Quality Controlled Precip Totals')
    (lats, lons) = np.meshgrid(YS, XS)
    print np.shape(precip), np.shape(lons), np.shape(lats)
    m.pcolormesh(lons, lats, np.transpose(precip) / 25.4, clevs,
                 units='inch')
    m.drawcounties()
    m.postprocess(filename='qc.png')

def main(argv):
    valid = datetime.date(int(argv[1]), int(argv[2]),
                          int(argv[3]))
    do(valid)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import unittest
from core.app import App, AppParameter, AppFile
import shutil

class TestAppParameter(unittest.TestCase):
    """
    test for App Parameters
    """
    def test_check_error_parameter_type(self):
        param = {'type': 'error type'}
        self.assertRaises(TypeError, AppParameter, param)

    def test_number(self):
        num_param = {
            'separator': " ",
            'prefix': "-n",
            'type': 'number',
            'required': True,
            'minvalue': 0,
            'maxvalue': 100,
            'hint': 'number test',
            'default': 6
        }
        param = AppParameter(num_param)
        self.assertEqual(param.__str__(), '-n 6')

    def test_string(self):
        string_param = {
            'separator': " ",
            'prefix': "-db",
            'type': 'string',
            'required': True,
            'quotes': False,
            'hint': 'working space',
            'default': 'nt',
            'quotes': False,
            'hint': 'working space'
        }
        param = AppParameter(string_param)
        self.assertEqual(param.__str__(), '-db nt')
        string_param['quotes'] = True
        param = AppParameter(string_param)
        self.assertEqual(param.__str__(), "-db 'nt'")

    def test_array(self):
        array_param = {
            "separator": '=',
            "prefix": '-taxid',
            "type": 'array',
            "required": True,
            "minitems": 1,
            "maxitems": 100,
            "item":{
                "type": 'string',
                "item_quotes": True,
                "separator": ' ',
                "is_split": True,
                },
            "hint": 'array test',
            "default": ['4', '5', '6', '7']
        }
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid='4' -taxid='5' -taxid='6' -taxid='7'")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (True, False)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4 -taxid=5 -taxid=6 -taxid=7")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (False, True)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid='4' '5' '6' '7'")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (False, False)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4 5 6 7")
        # print param

        array_param['item']['separator'] = ','
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4,5,6,7")
        # print param
        array_param['separator'] = ' '
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid 4,5,6,7")
        # print param

    def test_flag(self):
        flag_param = {
            'separator': ' ',
            'prefix': '-pe',
            'type': 'flag',
            'required': True,
            'hint': 'flag test',
            'default': True
        }
        param = AppParameter(flag_param)
        self.assertEqual(param.__str__(), "-pe")
        flag_param['value']=False
        param = AppParameter(flag_param)
        self.assertEqual(param.__str__(), "")

    def test_boolean(self):
        boolean_param = {
            'separator': ' ',
            'prefix': '-pe',
            'type': 'boolean',
            'required': True,
            'hint': 'boolean test',
            'default': True
        }
        param = AppParameter(boolean_param)
        self.assertEqual(param.__str__(), "-pe True")
        boolean_param['value']=False
        param = AppParameter(boolean_param)
        self.assertEqual(param.__str__(), "-pe False")

class TestAppFile(unittest.TestCase):
    """docstring for TestAppFile"""
    def setUp(self):
        self.file = {
            'type': 'file',
            'required': True,
            'minitems': 1,
            'maxitems': 1,
            'item':{
                'separator': " "
            },
            'formats': ['tgz']
        }

    def test_path(self):
        output = AppFile(self.file)
        print output.path

    def test_enid(self):
        output = AppFile(self.file)
        print output.enid

    def test_format(self):
        self.file['formats'] = 'bam'
        output = AppFile(self.file)
        print output.path

    def test_name(self):
        self.file['name'] = '/path/to/data'
        output = AppFile(self.file)
        print output.path

class TestApp(unittest.TestCase):
    """docstring for TestApp"""
    def setUp(self):
        print "=====setUP====="
        self.app = App('test/test_app')
        self.app.new()

    def tearDown(self):
        print "=====tearDown====="
        shutil.rmtree('test/test_app')

    def test_load(self):
        print "=====load====="
        app = App('test/test_app')
        app.load()
        # print app.config

    def test_new(self):
        print "=====new====="
        # print self.app.config

    def test_setParameters(self):
        print "=====set_default_parameters====="
        self.app.load()
        # print self.app.config['app']['parameters']['workspace']

        self.app.parameters['inputs']['bam']={'data': [{"name":"/path/to/data1"}, {"name":"/path/to/data2"}]}
        self.app.setParameters()
        self.assertEqual(self.app.config['app']['inputs']['bam'][0].path, "/path/to/data1")
        self.assertEqual(self.app.config['app']['outputs']['results'][0].path[0:10], "/var/data/")
        self.assertEqual(self.app.config['app']['outputs']['results'][0].path[-3:], 'tgz')

    def test_newParameters(self):
        self.app.load()
        # self.app.setParameters()
        self.app.newParameters('test/test_app/test_parameter.yaml')
        # print self.app.parameters

if __name__ == '__main__':
    unittest.main()
<commit_msg>more setParameters newParameters test<commit_after>import unittest
from core.app import App, AppParameter, AppFile
import shutil

class TestAppParameter(unittest.TestCase):
    """
    test for App Parameters
    """
    def test_check_error_parameter_type(self):
        param = {'type': 'error type'}
        self.assertRaises(TypeError, AppParameter, param)

    def test_number(self):
        num_param = {
            'separator': " ",
            'prefix': "-n",
            'type': 'number',
            'required': True,
            'minvalue': 0,
            'maxvalue': 100,
            'hint': 'number test',
            'default': 6
        }
        param = AppParameter(num_param)
        self.assertEqual(param.__str__(), '-n 6')

    def test_string(self):
        string_param = {
            'separator': " ",
            'prefix': "-db",
            'type': 'string',
            'required': True,
            'quotes': False,
            'hint': 'working space',
            'default': 'nt',
            'quotes': False,
            'hint': 'working space'
        }
        param = AppParameter(string_param)
        self.assertEqual(param.__str__(), '-db nt')
        string_param['quotes'] = True
        param = AppParameter(string_param)
        self.assertEqual(param.__str__(), "-db 'nt'")

    def test_array(self):
        array_param = {
            "separator": '=',
            "prefix": '-taxid',
            "type": 'array',
            "required": True,
            "minitems": 1,
            "maxitems": 100,
            "item":{
                "type": 'string',
                "item_quotes": True,
                "separator": ' ',
                "is_split": True,
                },
            "hint": 'array test',
            "default": ['4', '5', '6', '7']
        }
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid='4' -taxid='5' -taxid='6' -taxid='7'")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (True, False)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4 -taxid=5 -taxid=6 -taxid=7")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (False, True)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid='4' '5' '6' '7'")
        # print param
        (array_param['item']['is_split'], array_param['item']['item_quotes']) = (False, False)
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4 5 6 7")
        # print param

        array_param['item']['separator'] = ','
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid=4,5,6,7")
        # print param
        array_param['separator'] = ' '
        param = AppParameter(array_param)
        self.assertEqual(param.__str__(), "-taxid 4,5,6,7")
        # print param

    def test_flag(self):
        flag_param = {
            'separator': ' ',
            'prefix': '-pe',
            'type': 'flag',
            'required': True,
            'hint': 'flag test',
            'default': True
        }
        param = AppParameter(flag_param)
        self.assertEqual(param.__str__(), "-pe")
        flag_param['value']=False
        param = AppParameter(flag_param)
        self.assertEqual(param.__str__(), "")

    def test_boolean(self):
        boolean_param = {
            'separator': ' ',
            'prefix': '-pe',
            'type': 'boolean',
            'required': True,
            'hint': 'boolean test',
            'default': True
        }
        param = AppParameter(boolean_param)
        self.assertEqual(param.__str__(), "-pe True")
        boolean_param['value']=False
        param = AppParameter(boolean_param)
        self.assertEqual(param.__str__(), "-pe False")

class TestAppFile(unittest.TestCase):
    """docstring for TestAppFile"""
    def setUp(self):
        self.file = {
            'type': 'file',
            'required': True,
            'minitems': 1,
            'maxitems': 1,
            'item':{
                'separator': " "
            },
            'formats': ['tgz']
        }

    def test_path(self):
        output = AppFile(self.file)
        print output.path

    def test_enid(self):
        output = AppFile(self.file)
        print output.enid

    def test_format(self):
        self.file['formats'] = 'bam'
        output = AppFile(self.file)
        print output.path

    def test_name(self):
        self.file['name'] = '/path/to/data'
        output = AppFile(self.file)
        print output.path

class TestApp(unittest.TestCase):
    """docstring for TestApp"""
    def setUp(self):
        #setUp before each test
        self.app = App('test/test_app')
        self.app.new()
        self.app.load()

    def tearDown(self):
        # tearDown after each test
        shutil.rmtree('test/test_app')

    def test_load(self):
        self.assertEqual(self.app.config['app']['name'], 'app name')
        # print app.config

    def test_new(self):
        #done in setUP
        pass

    def test_setParameters(self):
        self.app.parameters['inputs']['bam']={'data': [{"name":"/path/to/data1"}, {"name":"/path/to/data2"}]}
        self.app.setParameters()
        self.assertEqual(self.app.config['app']['inputs']['bam'][0].path, "/path/to/data1")
        self.assertEqual(self.app.config['app']['outputs']['results'][0].path[0:10], "/var/data/")
        self.assertEqual(self.app.config['app']['outputs']['results'][0].path[-3:], 'tgz')
        #try setParameters ater newParameters
        self.app.newParameters('test/test_app/test_parameter.yaml')
        self.app.parameters['parameters']['workspace']['value'] = '/path/to/data3'
        self.app.setParameters()
        self.assertEqual(self.app.config['app']['parameters']['workspace'].__str__(), '/path/to/data3')

    def test_newParameters(self):
        self.app.newParameters('test/test_app/test_parameter.yaml')
        # print self.app.parameters

    def test_newParameters_after_setParameters(self):
        self.app.setParameters()
        self.app.newParameters('test/test_app/test_parameter.yaml')

    def test_newParameters_before_setParameters(self):
        self.app.newParameters('test/test_app/test_parameter.yaml')
        self.app.setParameters()

    def test_newParameters_and_setParameters_more(self):
        self.app.newParameters('test/test_app/test_parameter.yaml')
        self.app.setParameters()
        self.app.newParameters('test/test_app/test_parameter.yaml')
        self.app.setParameters()

    def test_newParameters_and_setParameters_more2(self):
        self.app.setParameters()
        self.app.newParameters('test/test_app/test_parameter.yaml')
        self.app.setParameters()
        self.app.newParameters('test/test_app/test_parameter.yaml')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before><commit_msg>Run buildifier -mode=fix over the file.<commit_after><|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# a test for container management and recovery. The docker container is managed
# by docker and should be restarted automatically upon failure.

import urllib
import json
from time import sleep, time

import fabric.api

from cosmo_tester.test_suites.test_blueprints import nodecellar_test

class DockerRecoveryTest(nodecellar_test.NodecellarAppTest):

    def test_docker_recovery(self):
        context_before = self.get_provider_context()
        self.init_fabric()
        self.restart_vm()
        down = self._wait_for_management_state(self.env.management_ip, 180,
                                               state=False)
        self.assertTrue(down, 'Management VM {} failed to terminate'
                              .format(self.env.management_ip))
        started = self._wait_for_management_state(self.env.management_ip, 180)
        self.assertTrue(started, 'Cloudify docker service container failed'
                                 ' to start after reboot. Test failed.')
        context_after = self._wait_for_provider_context(180)

        self.assertEqual(json.load(context_before),
                         json.load(context_after),
                         msg='Provider context should be identical to what it '
                             'was prior to reboot.')

    def get_provider_context(self):
        context_url = 'http://{0}/provider/context'\
                      .format(self.env.management_ip)
        return urllib.urlopen(context_url)

    def init_fabric(self):
        manager_keypath = self.env._config_reader.management_key_path
        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': 'ubuntu',
            'key_filename': manager_keypath,
            'host_string': self.env.management_ip,
        })

    def restart_vm(self):
        self.logger.info('Restarting machine with ip {0}'
                         .format(self.env.management_ip))
        return fabric.api.run('sudo shutdown -r now')

    def _wait_for_provider_context(self, timeout):
        end = time() + timeout
        while end - time() >= 0:
            context = self.get_provider_context()
            if context:
                return context
            else:
                self.logger.info('Provider context is empty. sleeping for 2 '
                                 'seconds...')
                sleep(2)

        raise RuntimeError('Failed waiting for provider context. '
                           'waited {0} seconds'.format(timeout))

    def _wait_for_management_state(self, ip, timeout, port=80, state=True):
        """ Wait for management to reach state
            :param ip: the manager IP
            :param timeout: in seconds
            :param port: port used by the rest service.
            :param state: management state, true for running, else false.
            :return: True of False
        """
        validation_url = 'http://{0}:{1}/blueprints'.format(ip, port)

        end = time() + timeout

        while end - time() >= 0:
            try:
                status = urllib.urlopen(validation_url).getcode()
                if status == 200 and state:
                    return True
                if not state:
                    if status == 200:
                        self.logger.info('Manager is accessible. '
                                         'retrying in 2 seconds.')
                    else:
                        return True

            except IOError:
                if not state:
                    return True
                self.logger.info('Manager not accessible. '
                                 'retrying in 2 seconds.')
            sleep(2)

        return False

    def get_inputs(self):

        return {
            'image': self.env.ubuntu_image_id,
            'flavor': self.env.small_flavor_id,
            'agent_user': 'ubuntu'
        }
<commit_msg>stabilize test<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

# a test for container management and recovery. The docker container is managed
# by docker and should be restarted automatically upon failure.

import urllib
from time import sleep, time

import fabric.api
from cloudify_rest_client import exceptions

from cosmo_tester.test_suites.test_blueprints import nodecellar_test

class DockerRecoveryTest(nodecellar_test.NodecellarAppTest):

    def test_docker_recovery(self):
        context_before = self.get_provider_context()
        self.init_fabric()
        self.restart_vm()
        down = self._wait_for_management_state(self.env.management_ip, 180,
                                               state=False)
        self.assertTrue(down, 'Management VM {} failed to terminate'
                              .format(self.env.management_ip))
        started = self._wait_for_management_state(self.env.management_ip, 180)
        self.assertTrue(started, 'Cloudify docker service container failed'
                                 ' to start after reboot. Test failed.')
        context_after = self._wait_for_provider_context(180)

        self.logger.info('provider context before restart is : {0}'
                         .format(context_before))
        self.logger.info('provider context after restart is : {0}'
                         .format(context_after))
        self.assertEqual(context_before,
                         context_after,
                         msg='Provider context is not the same after restart')

    def get_provider_context(self):
        return self.client.manager.get_context()

    def init_fabric(self):
        manager_keypath = self.env._config_reader.management_key_path
        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': 'ubuntu',
            'key_filename': manager_keypath,
            'host_string': self.env.management_ip,
        })

    def restart_vm(self):
        self.logger.info('Restarting machine with ip {0}'
                         .format(self.env.management_ip))
        return fabric.api.run('sudo shutdown -r now')

    def _wait_for_provider_context(self, timeout):
        end = time() + timeout
        while end - time() >= 0:
            try:
                context = self.get_provider_context()
                if context:
                    return context
                self.logger.info('Provider context is empty. sleeping for 2 '
                                 'seconds...')
                sleep(2)
            except exceptions.CloudifyClientError as e:
                # might be an elastic search issue (
                # NoShardAvailableActionException)
                self.logger.warning(str(e))
                sleep(2)

        raise RuntimeError('Failed waiting for provider context. '
                           'waited {0} seconds'.format(timeout))

    def _wait_for_management_state(self, ip, timeout, port=80, state=True):
        """ Wait for management to reach state
            :param ip: the manager IP
            :param timeout: in seconds
            :param port: port used by the rest service.
            :param state: management state, true for running, else false.
            :return: True of False
        """
        validation_url = 'http://{0}:{1}/blueprints'.format(ip, port)

        end = time() + timeout

        while end - time() >= 0:
            try:
                status = urllib.urlopen(validation_url).getcode()
                if status == 200 and state:
                    return True
                if not state:
                    if status == 200:
                        self.logger.info('Manager is accessible. '
                                         'retrying in 2 seconds.')
                    else:
                        return True

            except IOError:
                if not state:
                    return True
                self.logger.info('Manager not accessible. '
                                 'retrying in 2 seconds.')
            sleep(2)

        return False

    def get_inputs(self):

        return {
            'image': self.env.ubuntu_image_id,
            'flavor': self.env.small_flavor_id,
            'agent_user': 'ubuntu'
        }
<|endoftext|>


<commit_before>import os
import numpy as np
import pandas as pd
from ogusa import utils

CUR_PATH = os.path.split(os.path.abspath(__file__))[0]

def get_wealth_data(scf_yrs_list=[2019, 2016, 2013, 2010, 2007], web=True,
                    directory=None):
    '''
    Reads wealth data from the 2007, 2010, 2013, 2016, and 2019 Survey of
    Consumer Finances (SCF) files.

    Args:
        scf_yrs_list (list): list of SCF years to import. Currently the
            largest set of years that will work is
            [2019, 2016, 2013, 2010, 2007]
        web (Boolean): =True if function retrieves data from internet
        directory (string or None): local directory location if data are
            stored on local drive, not use internet (web=False)

    Returns:
        df_scf (Pandas DataFrame): pooled cross-sectional data from SCFs

    '''
    # Hard code cpi list for given years. Index values are annual average index
    # values from monthly FRED Consumer Price Index for All Urban Consumers:
    # All Items Less Food and Energy in U.S. City Average (CPILFESL,
    # https://fred.stlouisfed.org/series/CPILFESL). Base year is 1982-1984=100.
    # Values are [263.209, 247.585, 233.810, 221.336, 210.725].
    # We then reset the base year to 2019 by dividing each annual average by
    # the 2019 annual average and multiply by 100. Base year is 2019=100
    cpi_dict = {'cpi2019': 100.000,
                'cpi2016': 94.06403464,
                'cpi2013': 88.83067929,
                'cpi2010': 84.09125952,
                'cpi2007': 80.05995867}
    if web:
        # Throw an error if the machine is not connected to the internet
        if utils.not_connected():
            err_msg = ('SCF DATA ERROR: The local machine is not ' +
                       'connected to the internet and web=True was ' +
                       'selected.')
            raise RuntimeError(err_msg)

        file_urls = []
        for yr in scf_yrs_list:
            zipfilename = ('https://www.federalreserve.gov/econres/' +
                           'files/scfp' + str(yr) + 's.zip')
            file_urls.append(zipfilename)

        file_paths = utils.fetch_files_from_web(file_urls)

    if not web and directory is None:
        # Thow an error if web=False no source of files is given
        err_msg = ('SCF DATA ERROR: No local directory was ' +
                   'specified as the source for the data.')
        raise ValueError(err_msg)

    elif not web and directory is not None:
        file_paths = []
        full_directory = os.path.expanduser(directory)
        filename_list = []
        for yr in scf_yrs_list:
            filename = 'rscfp' + str(yr) + '.dta'
            filename_list.append(filename)

        for name in filename_list:
            file_paths.append(os.path.join(full_directory, name))
        # Check to make sure the necessary files are present in the
        # local directory
        err_msg = ('hrs_by_age() ERROR: The file %s was not found in ' +
                   'the directory %s')
        for path in file_paths:
            if not os.path.isfile(path):
                raise ValueError(err_msg % (path, full_directory))

    # read in raw SCF data to calculate moments
    scf_dict = {}
    for filename, year in zip(file_paths, scf_yrs_list):
        df_yr = pd.read_stata(filename, columns=['networth', 'wgt'])
        # Add inflation adjusted net worth
        cpi = cpi_dict['cpi' + str(year)]
        df_yr['networth_infadj'] = df_yr['networth'] * cpi
        scf_dict[str(year)] = df_yr

    df_scf = scf_dict[str(scf_yrs_list[0])]
    num_yrs = len(scf_yrs_list)
    if num_yrs >= 2:
        for year in scf_yrs_list[1:]:
            df_scf = df_scf.append(scf_dict[str(year)],
                                   ignore_index=True)

    return df_scf

def compute_wealth_moments(scf, bin_weights):
    '''
    This function computes moments from the distribution of wealth
    using SCF data.

    Args:
        scf (Pandas DataFrame): pooled cross-sectional data from SCFs
        bin_weights (Numpy Array) = ability weights
        J (int) = number of ability groups

    Returns:
        wealth_moments (Numpy Array): array of wealth moments

    '''
    # calculate percentile shares (percentiles based on lambdas input)
    scf.sort_values(by='networth_infadj', ascending=True, inplace=True)
    scf['weight_networth'] = scf['wgt'] * scf['networth_infadj']
    total_weight_wealth = scf.weight_networth.sum()
    cumsum = scf.wgt.cumsum()
    J = bin_weights.shape[0]
    pct_wealth = np.zeros(J)
    top_pct_wealth = np.zeros(J)
    wealth = np.zeros((J,))
    cum_weights = bin_weights.cumsum()
    for i in range(J):
        cutoff = scf.wgt.sum() / (1. / cum_weights[i])
        pct_wealth[i] = scf.networth[cumsum >= cutoff].iloc[0]
        top_pct_wealth[i] = 1 - (
            (scf.weight_networth[cumsum < cutoff].sum()) /
            total_weight_wealth)
        wealth[i] = ((
            scf.weight_networth[cumsum < cutoff].sum()) /
            total_weight_wealth)

    wealth_share = np.zeros(J)
    wealth_share[0] = wealth[0]
    wealth_share[1:] = wealth[1:] - wealth[0:-1]

    # compute gini coeff
    scf.sort_values(by='networth_infadj', ascending=True, inplace=True)
    p = (scf.wgt.cumsum() / scf.wgt.sum()).values
    nu = ((scf.wgt * scf.networth_infadj).cumsum()).values
    nu = nu / nu[-1]
    gini_coeff = (nu[1:] * p[:-1]).sum() - (nu[:-1] * p[1:]).sum()

    # compute variance in logs
    df = scf.drop(scf[scf['networth_infadj'] <= 0.0].index)
    df['ln_networth'] = np.log(df['networth_infadj'])
    df.sort_values(by='ln_networth', ascending=True, inplace=True)
    weight_mean = ((df.ln_networth * df.wgt).sum()) / (df.wgt.sum())
    var_ln_wealth = (((
        df.wgt * ((df.ln_networth - weight_mean) ** 2)).sum()) *
        (1. / (df.wgt.sum() - 1)))

    wealth_moments = np.append(
        [wealth_share], [gini_coeff, var_ln_wealth])

    return wealth_moments
<commit_msg>Commented out unused error-causing objects in wealth.py<commit_after>import os
import numpy as np
import pandas as pd
from ogusa import utils

CUR_PATH = os.path.split(os.path.abspath(__file__))[0]

def get_wealth_data(scf_yrs_list=[2019, 2016, 2013, 2010, 2007], web=True,
                    directory=None):
    '''
    Reads wealth data from the 2007, 2010, 2013, 2016, and 2019 Survey of
    Consumer Finances (SCF) files.

    Args:
        scf_yrs_list (list): list of SCF years to import. Currently the
            largest set of years that will work is
            [2019, 2016, 2013, 2010, 2007]
        web (Boolean): =True if function retrieves data from internet
        directory (string or None): local directory location if data are
            stored on local drive, not use internet (web=False)

    Returns:
        df_scf (Pandas DataFrame): pooled cross-sectional data from SCFs

    '''
    # Hard code cpi list for given years. Index values are annual average index
    # values from monthly FRED Consumer Price Index for All Urban Consumers:
    # All Items Less Food and Energy in U.S. City Average (CPILFESL,
    # https://fred.stlouisfed.org/series/CPILFESL). Base year is 1982-1984=100.
    # Values are [263.209, 247.585, 233.810, 221.336, 210.725].
    # We then reset the base year to 2019 by dividing each annual average by
    # the 2019 annual average and multiply by 100. Base year is 2019=100
    cpi_dict = {'cpi2019': 100.000,
                'cpi2016': 94.06403464,
                'cpi2013': 88.83067929,
                'cpi2010': 84.09125952,
                'cpi2007': 80.05995867}
    if web:
        # Throw an error if the machine is not connected to the internet
        if utils.not_connected():
            err_msg = ('SCF DATA ERROR: The local machine is not ' +
                       'connected to the internet and web=True was ' +
                       'selected.')
            raise RuntimeError(err_msg)

        file_urls = []
        for yr in scf_yrs_list:
            zipfilename = ('https://www.federalreserve.gov/econres/' +
                           'files/scfp' + str(yr) + 's.zip')
            file_urls.append(zipfilename)

        file_paths = utils.fetch_files_from_web(file_urls)

    if not web and directory is None:
        # Thow an error if web=False no source of files is given
        err_msg = ('SCF DATA ERROR: No local directory was ' +
                   'specified as the source for the data.')
        raise ValueError(err_msg)

    elif not web and directory is not None:
        file_paths = []
        full_directory = os.path.expanduser(directory)
        filename_list = []
        for yr in scf_yrs_list:
            filename = 'rscfp' + str(yr) + '.dta'
            filename_list.append(filename)

        for name in filename_list:
            file_paths.append(os.path.join(full_directory, name))
        # Check to make sure the necessary files are present in the
        # local directory
        err_msg = ('hrs_by_age() ERROR: The file %s was not found in ' +
                   'the directory %s')
        for path in file_paths:
            if not os.path.isfile(path):
                raise ValueError(err_msg % (path, full_directory))

    # read in raw SCF data to calculate moments
    scf_dict = {}
    for filename, year in zip(file_paths, scf_yrs_list):
        df_yr = pd.read_stata(filename, columns=['networth', 'wgt'])
        # Add inflation adjusted net worth
        cpi = cpi_dict['cpi' + str(year)]
        df_yr['networth_infadj'] = df_yr['networth'] * cpi
        scf_dict[str(year)] = df_yr

    df_scf = scf_dict[str(scf_yrs_list[0])]
    num_yrs = len(scf_yrs_list)
    if num_yrs >= 2:
        for year in scf_yrs_list[1:]:
            df_scf = df_scf.append(scf_dict[str(year)],
                                   ignore_index=True)

    return df_scf

def compute_wealth_moments(scf, bin_weights):
    '''
    This function computes moments from the distribution of wealth
    using SCF data.

    Args:
        scf (Pandas DataFrame): pooled cross-sectional data from SCFs
        bin_weights (Numpy Array) = ability weights
        J (int) = number of ability groups

    Returns:
        wealth_moments (Numpy Array): array of wealth moments

    '''
    # calculate percentile shares (percentiles based on lambdas input)
    scf.sort_values(by='networth_infadj', ascending=True, inplace=True)
    scf['weight_networth'] = scf['wgt'] * scf['networth_infadj']
    total_weight_wealth = scf.weight_networth.sum()
    cumsum = scf.wgt.cumsum()
    J = bin_weights.shape[0]
    # pct_wealth = np.zeros(J)
    # top_pct_wealth = np.zeros(J)
    wealth = np.zeros((J,))
    cum_weights = bin_weights.cumsum()
    for i in range(J):
        # Get number of individuals at top of percentile bin
        cutoff = scf.wgt.sum() * cum_weights[i]
        # pct_wealth[i] = scf.networth[cumsum >= cutoff].iloc[0]
        # top_pct_wealth[i] = 1 - (
        #     (scf.weight_networth[cumsum < cutoff].sum()) /
        #     total_weight_wealth)
        wealth[i] = ((
            scf.weight_networth[cumsum < cutoff].sum()) /
            total_weight_wealth)

    wealth_share = np.zeros(J)
    wealth_share[0] = wealth[0]
    wealth_share[1:] = wealth[1:] - wealth[0:-1]

    # compute gini coeff
    scf.sort_values(by='networth_infadj', ascending=True, inplace=True)
    p = (scf.wgt.cumsum() / scf.wgt.sum()).values
    nu = ((scf.wgt * scf.networth_infadj).cumsum()).values
    nu = nu / nu[-1]
    gini_coeff = (nu[1:] * p[:-1]).sum() - (nu[:-1] * p[1:]).sum()

    # compute variance in logs
    df = scf.drop(scf[scf['networth_infadj'] <= 0.0].index)
    df['ln_networth'] = np.log(df['networth_infadj'])
    df.sort_values(by='ln_networth', ascending=True, inplace=True)
    weight_mean = ((df.ln_networth * df.wgt).sum()) / (df.wgt.sum())
    var_ln_wealth = (((
        df.wgt * ((df.ln_networth - weight_mean) ** 2)).sum()) *
        (1. / (df.wgt.sum() - 1)))

    wealth_moments = np.append(
        [wealth_share], [gini_coeff, var_ln_wealth])

    return wealth_moments
<|endoftext|>


<commit_before># Copyright 2014 - Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import six

from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.workflow import states

_CAPACITY = 'capacity'
_CONCURRENCY = 'concurrency'
_COUNT = 'count'
_WITH_ITEMS = 'with_items_context'

def _get_context(task_ex):
    return task_ex.runtime_context[_WITH_ITEMS]

def get_count(task_ex):
    return _get_context(task_ex)[_COUNT]

def is_completed(task_ex):
    action_exs = db_api.get_action_executions(
        task_execution_id=task_ex.id,
        accepted=True
    )
    count = get_count(task_ex) or 1

    return count == len(action_exs)

def get_index(task_ex):
    return len(
        list(filter(
            lambda x: x.accepted or states.RUNNING, task_ex.executions
        ))
    )

def get_concurrency(task_ex):
    return task_ex.runtime_context.get(_CONCURRENCY)

def get_final_state(task_ex):
    find_error = lambda x: x.accepted and x.state == states.ERROR

    if filter(find_error, task_ex.executions):
        return states.ERROR
    else:
        return states.SUCCESS

def _get_indices_if_rerun(unaccepted_executions):
    """Returns a list of indices in case of re-running with-items.

    :param unaccepted_executions: List of executions.
    :return: a list of numbers.
    """

    return sorted(
        set([
            ex.runtime_context['with_items_index']
            for ex in unaccepted_executions
        ])
    )

def _get_unaccepted_act_exs(task_ex):
    # Choose only if not accepted but completed.
    return list(filter(
        lambda x: not x.accepted and states.is_completed(x.state),
        task_ex.executions
    ))

def get_indices_for_loop(task_ex):
    capacity = _get_context(task_ex)[_CAPACITY]
    count = get_count(task_ex)

    unaccepted = _get_unaccepted_act_exs(task_ex)

    if unaccepted:
        indices = _get_indices_if_rerun(unaccepted)

        if max(indices) < count - 1:
            indices += list(six.moves.range(max(indices) + 1, count))

        return indices[:capacity] if capacity else indices

    index = get_index(task_ex)

    number_to_execute = capacity if capacity else count - index

    return list(six.moves.range(index, index + number_to_execute))

def decrease_capacity(task_ex, count):
    with_items_context = _get_context(task_ex)

    if with_items_context[_CAPACITY] is not None:
        if with_items_context[_CAPACITY] >= count:
            with_items_context[_CAPACITY] -= count
        else:
            raise exc.WorkflowException(
                "Impossible to apply current with-items concurrency."
            )

    task_ex.runtime_context.update({_WITH_ITEMS: with_items_context})

def increase_capacity(task_ex):
    with_items_context = _get_context(task_ex)
    max_concurrency = get_concurrency(task_ex)

    if max_concurrency and with_items_context[_CAPACITY] < max_concurrency:
        with_items_context[_CAPACITY] += 1
        task_ex.runtime_context.update({_WITH_ITEMS: with_items_context})

def prepare_runtime_context(task_ex, task_spec, input_dicts):
    runtime_context = task_ex.runtime_context
    with_items_spec = task_spec.get_with_items()

    if with_items_spec and not runtime_context.get(_WITH_ITEMS):
        # Prepare current indexes and parallel limitation.
        runtime_context[_WITH_ITEMS] = {
            _CAPACITY: get_concurrency(task_ex),
            _COUNT: len(input_dicts)
        }

def validate_input(with_items_input):
    # Take only mapped values and check them.
    values = list(with_items_input.values())

    if not all([isinstance(v, list) for v in values]):
        raise exc.InputException(
            "Wrong input format for: %s. List type is"
            " expected for each value." % with_items_input
        )

    required_len = len(values[0])

    if not all(len(v) == required_len for v in values):
        raise exc.InputException(
            "Wrong input format for: %s. All arrays must"
            " have the same length." % with_items_input
        )

def has_more_iterations(task_ex):
    # See action executions which have been already
    # accepted or are still running.
    action_exs = list(filter(
        lambda x: x.accepted or x.state == states.RUNNING,
        task_ex.executions
    ))

    return get_count(task_ex) > len(action_exs)
<commit_msg>Filter is converted to list.<commit_after># Copyright 2014 - Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import six

from mistral.db.v2 import api as db_api
from mistral import exceptions as exc
from mistral.workflow import states

_CAPACITY = 'capacity'
_CONCURRENCY = 'concurrency'
_COUNT = 'count'
_WITH_ITEMS = 'with_items_context'

def _get_context(task_ex):
    return task_ex.runtime_context[_WITH_ITEMS]

def get_count(task_ex):
    return _get_context(task_ex)[_COUNT]

def is_completed(task_ex):
    action_exs = db_api.get_action_executions(
        task_execution_id=task_ex.id,
        accepted=True
    )
    count = get_count(task_ex) or 1

    return count == len(action_exs)

def get_index(task_ex):
    return len(
        list(filter(
            lambda x: x.accepted or states.RUNNING, task_ex.executions
        ))
    )

def get_concurrency(task_ex):
    return task_ex.runtime_context.get(_CONCURRENCY)

def get_final_state(task_ex):
    find_error = lambda x: x.accepted and x.state == states.ERROR

    if list(filter(find_error, task_ex.executions)):
        return states.ERROR
    else:
        return states.SUCCESS

def _get_indices_if_rerun(unaccepted_executions):
    """Returns a list of indices in case of re-running with-items.

    :param unaccepted_executions: List of executions.
    :return: a list of numbers.
    """

    return sorted(
        set([
            ex.runtime_context['with_items_index']
            for ex in unaccepted_executions
        ])
    )

def _get_unaccepted_act_exs(task_ex):
    # Choose only if not accepted but completed.
    return list(filter(
        lambda x: not x.accepted and states.is_completed(x.state),
        task_ex.executions
    ))

def get_indices_for_loop(task_ex):
    capacity = _get_context(task_ex)[_CAPACITY]
    count = get_count(task_ex)

    unaccepted = _get_unaccepted_act_exs(task_ex)

    if unaccepted:
        indices = _get_indices_if_rerun(unaccepted)

        if max(indices) < count - 1:
            indices += list(six.moves.range(max(indices) + 1, count))

        return indices[:capacity] if capacity else indices

    index = get_index(task_ex)

    number_to_execute = capacity if capacity else count - index

    return list(six.moves.range(index, index + number_to_execute))

def decrease_capacity(task_ex, count):
    with_items_context = _get_context(task_ex)

    if with_items_context[_CAPACITY] is not None:
        if with_items_context[_CAPACITY] >= count:
            with_items_context[_CAPACITY] -= count
        else:
            raise exc.WorkflowException(
                "Impossible to apply current with-items concurrency."
            )

    task_ex.runtime_context.update({_WITH_ITEMS: with_items_context})

def increase_capacity(task_ex):
    with_items_context = _get_context(task_ex)
    max_concurrency = get_concurrency(task_ex)

    if max_concurrency and with_items_context[_CAPACITY] < max_concurrency:
        with_items_context[_CAPACITY] += 1
        task_ex.runtime_context.update({_WITH_ITEMS: with_items_context})

def prepare_runtime_context(task_ex, task_spec, input_dicts):
    runtime_context = task_ex.runtime_context
    with_items_spec = task_spec.get_with_items()

    if with_items_spec and not runtime_context.get(_WITH_ITEMS):
        # Prepare current indexes and parallel limitation.
        runtime_context[_WITH_ITEMS] = {
            _CAPACITY: get_concurrency(task_ex),
            _COUNT: len(input_dicts)
        }

def validate_input(with_items_input):
    # Take only mapped values and check them.
    values = list(with_items_input.values())

    if not all([isinstance(v, list) for v in values]):
        raise exc.InputException(
            "Wrong input format for: %s. List type is"
            " expected for each value." % with_items_input
        )

    required_len = len(values[0])

    if not all(len(v) == required_len for v in values):
        raise exc.InputException(
            "Wrong input format for: %s. All arrays must"
            " have the same length." % with_items_input
        )

def has_more_iterations(task_ex):
    # See action executions which have been already
    # accepted or are still running.
    action_exs = list(filter(
        lambda x: x.accepted or x.state == states.RUNNING,
        task_ex.executions
    ))

    return get_count(task_ex) > len(action_exs)
<|endoftext|>


<commit_before>import numpy as np
from numpy import pi

class PowerSpectrumEstimator(object):
    """
    Class for estimating the shear power spectrum from gridded shears.

    This class stores all the data used in power spectrum estimation that is fixed with the geometry
    of the problem - the binning and spin weighting factors.

    The only public method is `estimate()`, which is called with 2D g1 and g2 arrays on a square
    grid.  It assumes the flat sky approximation (where ell and k are interchangeable), and rebins
    the observed ell modes into a user-defined number of logarithimic bins in ell.  Given that the
    grid parameters are precomputed and stored when the PowerSpectrumEstimator is initialized,
    computation of the PS for multiple sets of shears corresponding to the same grid setup can
    proceed more rapidly than if everything had to be recomputed each time.

    Some important notes:

    1) Power spectrum estimation requires a weight function which decides how the averaging
    is done across ell within each bin.  By default, that weighting is flat in ell, but
    this is easy to change (in the _bin_power function).  A keyword allows for weighting by the
    power itself, but requires the GalSim software package.

    2) This is the power spectrum of the gridded *data*, not the underlying field - we do not
    account for the effects of the finite grid (basically, ignoring all the reasons why power
    spectrum estimation of the hard).  Users must account for the contribution of noise in g1, g2
    and the masking.

    3) The binning is currently fixed as uniform in log(ell).

    4) The code for this class uses the notation of the GREAT10 handbook (Kitching et al. 2011,
    http://dx.doi.org/10.1214/11-AOAS484), equations 17-21.
    """
    def __init__(self, N=100, sky_size_deg=10., nbin=15):
        """Create a PSE object given some grid parameters.

        The grid parameters are:
        N: the number of pixels along each side;
        sky_size_deg: the total grid width (in one dimension) in degrees; and
        nbin: the number of evenly-spaced logarithmic ell bins to use.

        Each parameter has a reasonable default value.
        """

        # Set up the scales of the sky and pixels
        self.N = N
        self.sky_size = np.radians(sky_size_deg)
        self.dx = self.sky_size / N

        # Define the possible ell range, the bin edges and effective ell values.
        # This is necessary for binning the power spectrum in ell.
        lmin = 2*pi / self.sky_size
        lmax = np.sqrt(2.)*pi / self.dx # in 2 dimensions
        self.bin_edges = np.logspace(np.log10(lmin), np.log10(lmax), nbin+1)
        # By default, report an area-averaged value of ell, which should be fine if there is
        # no weighting (in which case it's recomputed) and if there are many ell modes in
        # each bin.  The latter assumption is most likely to break down at low ell.
        self.ell = (2./3.)*(self.bin_edges[1:]**3-self.bin_edges[:-1]**3) \
                                   / (self.bin_edges[1:]**2-self.bin_edges[:-1]**2)

        # Precompute and store two useful factors, both in the form of 2D grids in Fourier space.
        # These are the lengths of the wavevector |ell| for each point in the space, and the complex
        # valued spin-weighting that takes g_r, g_i -> E,B
        self.l_abs, self.eb_rot = self._generate_eb_rotation()

    def _generate_eb_rotation(self):
        # Set up the Fourier space grid lx, ly.
        ell = 2*pi*np.fft.fftfreq(self.N, self.dx)
        lx = np.vstack([ell for i in xrange(self.N)])
        ly = lx.T

        # Now compute the lengths and angles of the ell vectors.
        l_abs = (lx**2 + ly**2)**0.5
        l_ang = np.arctan2(ly, lx)

        self.lx = lx
        self.ly = ly
        self.l_ang = l_ang

        # Compute the spin-2 weights.
        rot = np.exp(-2j*l_ang)

        return l_abs, rot

    def _bin_power(self, C, ell_weight=None):
        # This little utility function bins a 2D C^{E/B, E/B}_{ell} based on |ell|.  The use of
        # histogram is a little hack, but is quite convenient since it means everything is done in C
        # so it is very fast. The first call to `histogram` just returns an array over the
        # logarithmic ell bins of
        # sum_{|ell| in bin} C_{ell_x,ell_y}
        # and the second call returns
        # sum_{|ell| in bin} 1.
        # Thus, the ratio is just the mean power in the bin.  If `ell_weight` is not None, then some
        # non-flat weighting scheme is used for averaging over the ell values within a bin.
        if ell_weight is not None:
            P,_ = np.histogram(self.l_abs, self.bin_edges, weights=C*ell_weight)
            count,_ = np.histogram(self.l_abs, self.bin_edges, weights=ell_weight)
        else:
            P,_ = np.histogram(self.l_abs, self.bin_edges, weights=C)
            count,_ = np.histogram(self.l_abs, self.bin_edges)
        return P/count

    def estimate(self, g1, g2, weight_EE=False, weight_BB=False, theory_func=None):
        """ Compute the EE, BB, and EB power spectra of two 2D arrays g1 and g2.

        In addition to the obvious arguments (the shear components as 2D NumPy arrays), this method
        can take three optional arguments:
        * weight_EE and weight_BB determine whether the E and B auto-power spectra are re-computed
          weighting by the power within the bin.
        * theory_func is some callable function that can be used to get an idealized value of power
          at each point on the grid, and then see what results it gives for our chosen ell binning.
        """
        # Check for square geometry is what we expect.
        assert g1.shape == g2.shape == (self.N, self.N)

        # Transform g1+j*g2 into Fourier space and rotate into E-B, then separate into E and B.
        EB = np.fft.ifft2(self.eb_rot * np.fft.fft2(g1 + 1j*g2))
        E = np.fft.fft2(EB.real)
        B = np.fft.fft2(EB.imag)

        # Use the internal function above to bin, and account for the normalization of the FFT.
        # Recall that power has units of angle^2, which is the reason why we need a self.dx^2 in the
        # equations below in addition to the standard 1/N^2 coming from the FFTs.
        C_EE = self._bin_power(E*np.conjugate(E))*(self.dx/self.N)**2
        C_BB = self._bin_power(B*np.conjugate(B))*(self.dx/self.N)**2
        C_EB = self._bin_power(E*np.conjugate(B))*(self.dx/self.N)**2

        if theory_func or weight_EE or weight_BB:
            import galsim

        if theory_func is not None:
            # theory_func needs to be a callable function; need to carefully sanity check this
            # later, but for now just assume.
            C_theory_ell = np.zeros_like(self.l_abs)
            C_theory_ell[self.l_abs>0] = theory_func(self.l_abs[self.l_abs>0])
            C_theory = self._bin_power(C_theory_ell)

        if weight_EE or weight_BB:
            # Need to interpolate C_EE to values of self.l_abs.  A bit of kludginess as we go off
            # the end of our final ell grid...
            new_ell = np.zeros(len(self.ell)+2)
            new_ell[1:len(self.ell)+1] = self.ell
            new_ell[len(self.ell)+1] = 10.*max(self.ell)
            if theory_func is not None:
                C_theory = self._bin_power(C_theory_ell, ell_weight=C_theory_ell)

        if weight_EE:
            new_CEE = np.zeros_like(new_ell)
            new_CEE[1:len(self.ell)+1] = np.real(C_EE)
            new_CEE[len(self.ell)+1] = new_CEE[len(self.ell)]
            EE_table = galsim.LookupTable(new_ell, new_CEE)
            ell_weight = EE_table(self.l_abs)
            C_EE = self._bin_power(E*np.conjugate(E), ell_weight=ell_weight)*(self.dx/self.N)**2

        if weight_BB:
            new_CBB = np.zeros_like(new_ell)
            new_CBB[1:len(self.ell)+1] = np.real(C_BB)
            new_CBB[len(self.ell)+1] = new_CBB[len(self.ell)]
            BB_table = galsim.LookupTable(new_ell, new_CBB)
            ell_weight = BB_table(self.l_abs)
            C_BB = self._bin_power(B*np.conjugate(B), ell_weight=ell_weight)*(self.dx/self.N)**2

        # For convenience, return ell (copied in case the user messes with it) and the three power
        # spectra. If the user requested a binned theoretical spectrum, return that as well.
        if theory_func is None:
            return self.ell.copy(), np.real(C_EE), np.real(C_BB), np.real(C_EB)
        else:
            return self.ell.copy(), np.real(C_EE), np.real(C_BB), np.real(C_EB), np.real(C_theory)
<commit_msg>some more unit checking on pse.py (#6)<commit_after>import numpy as np
from numpy import pi

class PowerSpectrumEstimator(object):
    """
    Class for estimating the shear power spectrum from gridded shears.

    This class stores all the data used in power spectrum estimation that is fixed with the geometry
    of the problem - the binning and spin weighting factors.

    The only public method is `estimate()`, which is called with 2D g1 and g2 arrays on a square
    grid.  It assumes the flat sky approximation (where ell and k are interchangeable), and rebins
    the observed ell modes into a user-defined number of logarithimic bins in ell.  Given that the
    grid parameters are precomputed and stored when the PowerSpectrumEstimator is initialized,
    computation of the PS for multiple sets of shears corresponding to the same grid setup can
    proceed more rapidly than if everything had to be recomputed each time.

    Some important notes:

    1) Power spectrum estimation requires a weight function which decides how the averaging
    is done across ell within each bin.  By default, that weighting is flat in ell, but
    this is easy to change (in the _bin_power function).  A keyword allows for weighting by the
    power itself, but requires the GalSim software package.

    2) This is the power spectrum of the gridded *data*, not the underlying field - we do not
    account for the effects of the finite grid (basically, ignoring all the reasons why power
    spectrum estimation of the hard).  Users must account for the contribution of noise in g1, g2
    and the masking.

    3) The binning is currently fixed as uniform in log(ell).

    4) The code for this class uses the notation of the GREAT10 handbook (Kitching et al. 2011,
    http://dx.doi.org/10.1214/11-AOAS484), equations 17-21.
    """
    def __init__(self, N=100, sky_size_deg=10., nbin=15):
        """Create a PSE object given some grid parameters.

        The grid parameters are:
        N: the number of pixels along each side;
        sky_size_deg: the total grid width (in one dimension) in degrees; and
        nbin: the number of evenly-spaced logarithmic ell bins to use.

        Each parameter has a reasonable default value.
        """
        # Set up the scales of the sky and pixels
        self.N = N
        self.sky_size = np.radians(sky_size_deg)
        self.dx = self.sky_size / N

        # Define the possible ell range, the bin edges and effective ell values.
        # This is necessary for binning the power spectrum in ell.
        lmin = 2*pi / self.sky_size
        lmax = np.sqrt(2.)*pi / self.dx # in 2 dimensions
        self.bin_edges = np.logspace(np.log10(lmin), np.log10(lmax), nbin+1)
        # By default, report an area-averaged value of ell, which should be fine if there is
        # no weighting (in which case it's recomputed) and if there are many ell modes in
        # each bin.  The latter assumption is most likely to break down at low ell.
        self.ell = (2./3.)*(self.bin_edges[1:]**3-self.bin_edges[:-1]**3) \
                                   / (self.bin_edges[1:]**2-self.bin_edges[:-1]**2)

        # Precompute and store two useful factors, both in the form of 2D grids in Fourier space.
        # These are the lengths of the wavevector |ell| for each point in the space, and the complex
        # valued spin-weighting that takes g_r, g_i -> E,B
        self.l_abs, self.eb_rot = self._generate_eb_rotation()

    def _generate_eb_rotation(self):
        # Set up the Fourier space grid lx, ly.
        ell = 2*pi*np.fft.fftfreq(self.N, self.dx)
        lx = np.vstack([ell for i in xrange(self.N)])
        ly = lx.T

        # Now compute the lengths and angles of the ell vectors.
        l_abs = (lx**2 + ly**2)**0.5
        l_ang = np.arctan2(ly, lx)

        self.lx = lx
        self.ly = ly
        self.l_ang = l_ang

        # Compute the spin-2 weights.
        rot = np.exp(-2j*l_ang)

        return l_abs, rot

    def _bin_power(self, C, ell_weight=None):
        # This little utility function bins a 2D C^{E/B, E/B}_{ell} based on |ell|.  The use of
        # histogram is a little hack, but is quite convenient since it means everything is done in C
        # so it is very fast. The first call to `histogram` just returns an array over the
        # logarithmic ell bins of
        # sum_{|ell| in bin} C_{ell_x,ell_y}
        # and the second call returns
        # sum_{|ell| in bin} 1.
        # Thus, the ratio is just the mean power in the bin.  If `ell_weight` is not None, then some
        # non-flat weighting scheme is used for averaging over the ell values within a bin.
        if ell_weight is not None:
            P,_ = np.histogram(self.l_abs, self.bin_edges, weights=C*ell_weight)
            count,_ = np.histogram(self.l_abs, self.bin_edges, weights=ell_weight)
        else:
            P,_ = np.histogram(self.l_abs, self.bin_edges, weights=C)
            count,_ = np.histogram(self.l_abs, self.bin_edges)
        return P/count

    def estimate(self, g1, g2, weight_EE=False, weight_BB=False, theory_func=None):
        """ Compute the EE, BB, and EB power spectra of two 2D arrays g1 and g2.

        In addition to the obvious arguments (the shear components as 2D NumPy arrays), this method
        can take three optional arguments:
        * weight_EE and weight_BB determine whether the E and B auto-power spectra are re-computed
          weighting by the power within the bin.
        * theory_func is some callable function that can be used to get an idealized value of power
          at each point on the grid, and then see what results it gives for our chosen ell binning.

        All optional arguments make use of the GalSim software, unlike the rest of the code in this
        file.
        """
        # Check for the expected square geometry consistent with the previously-defined grid size.
        if g1.shape != g2.shape:
            raise ValueError("g1 and g2 grids do not have the same shape!")
        if g1.shape[0] != g1.shape[1]:
            raise ValueError("Input shear arrays are not square!")
        if g1.shape[0] != self.N:
            raise ValueError("Input shear array size is not correct!")

        if not isinstance(weight_EE, bool) or not isinstance(weight_BB, bool):
            raise ValueError("Input weight flags must be bools!")

        # Transform g1+j*g2 into Fourier space and rotate into E-B, then separate into E and B.
        EB = np.fft.ifft2(self.eb_rot * np.fft.fft2(g1 + 1j*g2))
        E = np.fft.fft2(EB.real)
        B = np.fft.fft2(EB.imag)

        # Use the internal function above to bin, and account for the normalization of the FFT.
        # Recall that power has units of angle^2, which is the reason why we need a self.dx^2 in the
        # equations below in addition to the standard 1/N^2 coming from the FFTs.
        C_EE = self._bin_power(E*np.conjugate(E))*(self.dx/self.N)**2
        C_BB = self._bin_power(B*np.conjugate(B))*(self.dx/self.N)**2
        C_EB = self._bin_power(E*np.conjugate(B))*(self.dx/self.N)**2

        if theory_func or weight_EE or weight_BB:
            import galsim

        if theory_func is not None:
            # theory_func needs to be a callable function
            C_theory_ell = np.zeros_like(self.l_abs)
            C_theory_ell[self.l_abs>0] = theory_func(self.l_abs[self.l_abs>0])
            C_theory = self._bin_power(C_theory_ell)

        if weight_EE or weight_BB:
            # Need to interpolate C_EE to values of self.l_abs.  A bit of kludginess as we go off
            # the end of our final ell grid...
            new_ell = np.zeros(len(self.ell)+2)
            new_ell[1:len(self.ell)+1] = self.ell
            new_ell[len(self.ell)+1] = 10.*max(self.ell)
            if theory_func is not None:
                C_theory = self._bin_power(C_theory_ell, ell_weight=C_theory_ell)

        if weight_EE:
            new_CEE = np.zeros_like(new_ell)
            new_CEE[1:len(self.ell)+1] = np.real(C_EE)
            new_CEE[len(self.ell)+1] = new_CEE[len(self.ell)]
            EE_table = galsim.LookupTable(new_ell, new_CEE)
            ell_weight = EE_table(self.l_abs)
            C_EE = self._bin_power(E*np.conjugate(E), ell_weight=ell_weight)*(self.dx/self.N)**2

        if weight_BB:
            new_CBB = np.zeros_like(new_ell)
            new_CBB[1:len(self.ell)+1] = np.real(C_BB)
            new_CBB[len(self.ell)+1] = new_CBB[len(self.ell)]
            BB_table = galsim.LookupTable(new_ell, new_CBB)
            ell_weight = BB_table(self.l_abs)
            C_BB = self._bin_power(B*np.conjugate(B), ell_weight=ell_weight)*(self.dx/self.N)**2

        # For convenience, return ell (copied in case the user messes with it) and the three power
        # spectra. If the user requested a binned theoretical spectrum, return that as well.
        if theory_func is None:
            return self.ell.copy(), np.real(C_EE), np.real(C_BB), np.real(C_EB)
        else:
            return self.ell.copy(), np.real(C_EE), np.real(C_BB), np.real(C_EB), np.real(C_theory)
<|endoftext|>


<commit_before>num = int(input("Enter a number: "))

if num > 1:
   for i in range(2,num):
       if (num % i) == 0:
           print(num,"is not a prime number")
           break
   else:
       print(num,"is a prime number")

else:
   print(num,"is not a prime number")
<commit_msg>Update prime_number.py<commit_after># A Python program in Python to check if the number is prime or not
# input from the user
num = int(input("Enter a number: "))

# prime numbers are greater than 1
if num > 1:
   for i in range(2,num):
       if (num % i) == 0:
           print(num,"is not a prime number")
           break
   else:
       print(num,"is a prime number")

else:
   print(num,"is not a prime number")
<|endoftext|>


<commit_before>from types import AttrDict

class TableSchema(AttrDict):

    def __repr__(self):
        return ', '.join('{} {}'.format(*kv) for kv in self.items())

TABLE_SCHEMAS = AttrDict(
    tasks=TableSchema(
        name='TEXT',
        schedule='TEXT',
        last='TEXT',
        state='TEXT',
    ),
)
<commit_msg>schema: table schema repr code scrub.<commit_after>from types import AttrDict

class TableSchema(AttrDict):

    def __repr__(self):
        return ', '.join(' '.join(kv) for kv in self.items())

TABLE_SCHEMAS = AttrDict(
    tasks=TableSchema(
        name='TEXT',
        schedule='TEXT',
        last='TEXT',
        state='TEXT',
    ),
)
<|endoftext|>