<commit_before>"""
===============================================================================
module __GenericAlgorithm__: Base class to build custom algorithms
==================================================================

This generic class contains the recommended methods for subclassed algorithms.
It inherits from Core, so is Python Dict with the OpenPNM data control methods.

"""
import OpenPNM
import sys
import scipy as sp

class GenericAlgorithm(OpenPNM.Base.Core):
    r"""
    GenericAlgorithm - Base class to execute algorithms

    Parameters
    ----------
    network : OpenPNM Network Object
        The network object to which this algorithm will apply.

    name : string, optional
        Name of this algorithm

    Notes
    -----
    If no network is supplied an empty algorithm object is returned.  This is
    useful for loading in a saved algorithm from memory.

    """

    def __init__(self,network=None,name=None,**kwords):
        r"""
        Initialize
        """
        super(GenericAlgorithm,self).__init__(**kwords)
        self._logger.debug("Construct class")

        if network == None:
            self._net = OpenPNM.Network.GenericNetwork()
        else:
            self._net = network
        self.name = name

        # Initialize label 'all' in the object's own info dictionaries
        self['pore.all'] = self._net['pore.all']
        self['throat.all'] = self._net['throat.all']

    def run(self,**params):
        r"""
        Main run command for the algorithm
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_outer_iteration_stage()

    def _do_outer_iteration_stage(self):
        r"""
        Executes the outer iteration stage
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_one_outer_iteration()

    def _do_one_outer_iteration(self):
        r"""
        One iteration of an outer iteration loop for an algorithm
        (e.g. time or parametric study)
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_inner_iteration_stage()

    def _do_inner_iteration_stage(self):
        r"""
        Executes the inner iteration stage
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_one_inner_iteration()

    def _do_one_inner_iteration(self):
        r"""
        Executes one inner iteration
        """
        self._logger.debug(sys._getframe().f_code.co_name)

    def update_results(self,**kwargs):
        self._logger.debug(sys._getframe().f_code.co_name)

    def set_boundary_conditions(self,component=None,bctype='',bcvalue=[],pores=[],throats=[],mode='merge'):
        r'''
        Apply boundary conditions to specified pores.  This does not support
        throat boundary conditions yet.

        Parameters
        ----------
        bctype : string
            Specifies the type of boundary condition to apply.  Can be one of:

            - 'Dirichlet' : Specify the quantity in each pore
            - 'Neumann' : Specify the flow rate into each pore
            - 'Neumann_group' : Specify the net flow rate into a group of pores
        component : OpenPNM Phase object
            The Phase object to which this BC applies
        bcvalue : array_like
            The boundary value to apply, such as concentration or rate
        pores : array_like
            The pores where the boundary conditions should be applied
        throats : array_like
            The throats where the boundary conditions should be applied
        mode : string, optional
            Controls how the conditions are applied.  Options are:

            - 'merge': Inserts the specified values, leaving existing values elsewhere
            - 'overwrite': Inserts specified values, clearing all other values
            - 'remove': Removes boundary conditions from specified locations

        Notes
        -----
        1. At the moment is it not possible to have multiple boundary conditions
        in the same pore, so when new conditions are applied any existing ones
        are removed from all other boundary types.
        2. It is also not yet possible to apply boundary conditions to throats.
        '''

        BC_default = ['Dirichlet','Neumann','Neumann_group']
        try: self._existing_BC
        except: self._existing_BC = []
        if component==None:
            try:
                component = self._phase
            except:
                raise Exception('For using set_boundary_conditions method, a component/phase should be specified.')
        self._logger.debug('BC applies to the component: '+component.name)

        #If mode is 'remove', also bypass checks
        if mode == 'remove':
            if pores == [] and throats == []:
                if bctype=='':
                    raise Exception('No bctype/pore/throat is specified')
                else:
                    for item in self.labels():
                        if bctype == (item.split('.')[-1]).replace(self._phase.name+'_',"") :
                            element = item.split('.')[0]
                            try:
                                del self[element+'.'+component.name+'_bcval_'+bctype]
                            except: pass
                            del self[item]
                    self._logger.debug('Removing '+bctype+' from all locations in '+self.name)
                    self._existing_BC.remove(bctype)
            else:
                if pores!=[]:
                    if bctype!='':
                        self['pore.'+component.name+'_bcval_'+bctype][pores] = sp.nan
                        self['pore.'+component.name+'_'+bctype][pores] = False
                        self._logger.debug('Removing '+bctype+' from the specified pores in '+self.name)
                    else:   raise Exception('Cannot remove BC from the pores unless bctype is specified')

                if throats!=[]:
                    if bctype!='':
                        self['throat.'+component.name+'_bcval_'+bctype][throats] = sp.nan
                        self['throat.'+component.name+'_'+bctype][throats] = False
                        self._logger.debug('Removing '+bctype+' from the specified throats in '+self.name)
                    else:   raise Exception('Cannot remove BC from the throats unless bctype is specified')

            return
        #Validate bctype
        if bctype == '':
            raise Exception('bctype must be specified')
        #Validate pores/throats
        if pores == [] and throats == []:
            raise Exception('pores/throats must be specified')
        elif pores!=[] and throats!=[]:
            raise Exception('BC for pores and throats must be specified independently')
        elif  pores!=[]:
            element ='pore'
            loc = sp.array(pores,ndmin=1)
            all_length = self.num_pores()
        if throats!=[]:
            element ='throat'
            loc = sp.array(throats,ndmin=1)
            all_length = self.num_throats()
        #Validate bcvalue
        if bcvalue != []:
            #Check bcvalues are compatible with bctypes
            if bctype == 'Neumann_group':  #Only scalars are acceptable
                if sp.size(bcvalue) != 1:
                    raise Exception('When specifying Neumann_group, bcval should be a scalar')
            else: #Only scalars or Np/Nt-long are acceptable
                if sp.size(bcvalue) == 1:
                    bcvalue = sp.ones(sp.shape(loc))*bcvalue
                elif sp.size(bcvalue) != sp.size(loc):
                    raise Exception('The pore/throat list and bcvalue list are different lengths')

        #Confirm that prop and label arrays exist
        if element+'.'+component.name+'_bcval_'+bctype not in self.props():
            self[element+'.'+component.name+'_bcval_'+bctype] = sp.ones((all_length,),dtype=float)*sp.nan
        if element+'.'+component.name+'_'+bctype not in self.labels():
            self[element+'.'+component.name+'_'+bctype] = sp.zeros((all_length,),dtype=bool)

        #Check all BC from specified locations, prior to setting new ones
        for item in self.labels():
            bcname = (item.split('.')[-1]).replace(self._phase.name+'_',"")
            if bcname in self._existing_BC  and item.split('.')[0]==element:
                if mode=='merge':
                    if not (sp.isnan(self[element+'.'+component.name+'_bcval_'+bcname][loc]).all() and sp.sum(self[element+'.'+component.name+'_'+bcname][loc])==0):
                        raise Exception('Because of the existing BCs, the method cannot apply new BC with the merge mode to the specified pore/throat.')
        #Set boundary conditions based on supplied mode
        if mode == 'merge':
            if bcvalue != []:   self[element+'.'+component.name+'_bcval_'+bctype][loc] = bcvalue
            self[element+'.'+component.name+'_'+bctype][loc] = True
            if bctype not in self._existing_BC: self._existing_BC.append(bctype)
        elif mode == 'overwrite':
            self[element+'.'+component.name+'_bcval_'+bctype] = sp.ones((all_length,),dtype=float)*sp.nan
            if bcvalue != []:   self[element+'.'+component.name+'_bcval_'+bctype][loc] = bcvalue
            self[element+'.'+component.name+'_'+bctype] = sp.zeros((all_length,),dtype=bool)
            self[element+'.'+component.name+'_'+bctype][loc] = True
            if bctype not in self._existing_BC: self._existing_BC.append(bctype)

if __name__ == '__main__':
    pn = OpenPNM.Network.TestNet()
    test = OpenPNM.Algorithms.GenericAlgorithm(network=pn,loglevel=10)
    test.run()
<commit_msg>Doc update for generic algorithm<commit_after>"""
===============================================================================
module __GenericAlgorithm__: Base class to build custom algorithms
==================================================================

This generic class contains the recommended methods for subclassed algorithms.
It inherits from Core, so is Python Dict with the OpenPNM data control methods.

"""
import OpenPNM
import sys
import scipy as sp

class GenericAlgorithm(OpenPNM.Base.Core):
    r"""
    GenericAlgorithm - Base class to execute algorithms

    Parameters
    ----------
    network : OpenPNM Network Object
        The network object to which this algorithm will apply.

    name : string, optional
        Name of this algorithm

    Notes
    -----
    If no network is supplied an empty algorithm object is returned.  This is
    useful for loading in a saved algorithm from memory.

    """

    def __init__(self,network=None,name=None,**kwords):
        r"""
        Initialize
        """
        super(GenericAlgorithm,self).__init__(**kwords)
        self._logger.debug("Construct class")

        if network == None:
            self._net = OpenPNM.Network.GenericNetwork()
        else:
            self._net = network
        self.name = name

        # Initialize label 'all' in the object's own info dictionaries
        self['pore.all'] = self._net['pore.all']
        self['throat.all'] = self._net['throat.all']

    def run(self,**params):
        r"""
        Main run command for the algorithm
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_outer_iteration_stage()

    def _do_outer_iteration_stage(self):
        r"""
        Executes the outer iteration stage
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_one_outer_iteration()

    def _do_one_outer_iteration(self):
        r"""
        One iteration of an outer iteration loop for an algorithm
        (e.g. time or parametric study)
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_inner_iteration_stage()

    def _do_inner_iteration_stage(self):
        r"""
        Executes the inner iteration stage
        """
        self._logger.debug(sys._getframe().f_code.co_name)
        self._do_one_inner_iteration()

    def _do_one_inner_iteration(self):
        r"""
        Executes one inner iteration
        """
        self._logger.debug(sys._getframe().f_code.co_name)

    def update_results(self,**kwargs):
        self._logger.debug(sys._getframe().f_code.co_name)

    def set_boundary_conditions(self,component=None,bctype='',bcvalue=[],pores=[],throats=[],mode='merge'):
        r'''
        Apply boundary conditions to specified pores or throats

        Parameters
        ----------
        bctype : string
            Specifies the type or the name of boundary condition to apply.  The types can be one one of the followings:

            - 'Dirichlet' : Specify the quantity in each location
            - 'Neumann' : Specify the flow rate into each location
            - 'Neumann_group' : Specify the net flow rate into a group of pores/throats
        component : OpenPNM Phase object
            The Phase object to which this BC applies
        bcvalue : array_like
            The boundary value to apply, such as concentration or rate
        pores : array_like
            The pores where the boundary conditions should be applied
        throats : array_like
            The throats where the boundary conditions should be applied
        mode : string, optional
            Controls how the conditions are applied.  Options are:

            - 'merge': Inserts the specified values, leaving existing values elsewhere
            - 'overwrite': Inserts specified values, clearing all other values
            - 'remove': Removes boundary conditions from specified locations

        Notes
        -----
        1. It is not possible to have multiple boundary conditions for a specified location in just one algorithm.
        So when new condition is going to be applied to a specific location, any existing one
        should be removed or overwritten..
        '''
        try: self._existing_BC
        except: self._existing_BC = []
        if component==None:
            try:
                component = self._phase
            except:
                raise Exception('For using set_boundary_conditions method, a component/phase should be specified.')
        self._logger.debug('BC applies to the component: '+component.name)

        #If mode is 'remove', also bypass checks
        if mode == 'remove':
            if pores == [] and throats == []:
                if bctype=='':
                    raise Exception('No bctype/pore/throat is specified')
                else:
                    for item in self.labels():
                        if bctype == (item.split('.')[-1]).replace(self._phase.name+'_',"") :
                            element = item.split('.')[0]
                            try:
                                del self[element+'.'+component.name+'_bcval_'+bctype]
                            except: pass
                            del self[item]
                    self._logger.debug('Removing '+bctype+' from all locations in '+self.name)
                    self._existing_BC.remove(bctype)
            else:
                if pores!=[]:
                    if bctype!='':
                        self['pore.'+component.name+'_bcval_'+bctype][pores] = sp.nan
                        self['pore.'+component.name+'_'+bctype][pores] = False
                        self._logger.debug('Removing '+bctype+' from the specified pores in '+self.name)
                    else:   raise Exception('Cannot remove BC from the pores unless bctype is specified')

                if throats!=[]:
                    if bctype!='':
                        self['throat.'+component.name+'_bcval_'+bctype][throats] = sp.nan
                        self['throat.'+component.name+'_'+bctype][throats] = False
                        self._logger.debug('Removing '+bctype+' from the specified throats in '+self.name)
                    else:   raise Exception('Cannot remove BC from the throats unless bctype is specified')

            return
        #Validate bctype
        if bctype == '':
            raise Exception('bctype must be specified')
        #Validate pores/throats
        if pores == [] and throats == []:
            raise Exception('pores/throats must be specified')
        elif pores!=[] and throats!=[]:
            raise Exception('BC for pores and throats must be specified independently')
        elif  pores!=[]:
            element ='pore'
            loc = sp.array(pores,ndmin=1)
            all_length = self.num_pores()
        if throats!=[]:
            element ='throat'
            loc = sp.array(throats,ndmin=1)
            all_length = self.num_throats()
        #Validate bcvalue
        if bcvalue != []:
            #Check bcvalues are compatible with bctypes
            if bctype == 'Neumann_group':  #Only scalars are acceptable
                if sp.size(bcvalue) != 1:
                    raise Exception('When specifying Neumann_group, bcval should be a scalar')
            else: #Only scalars or Np/Nt-long are acceptable
                if sp.size(bcvalue) == 1:
                    bcvalue = sp.ones(sp.shape(loc))*bcvalue
                elif sp.size(bcvalue) != sp.size(loc):
                    raise Exception('The pore/throat list and bcvalue list are different lengths')

        #Confirm that prop and label arrays exist
        if element+'.'+component.name+'_bcval_'+bctype not in self.props():
            self[element+'.'+component.name+'_bcval_'+bctype] = sp.ones((all_length,),dtype=float)*sp.nan
        if element+'.'+component.name+'_'+bctype not in self.labels():
            self[element+'.'+component.name+'_'+bctype] = sp.zeros((all_length,),dtype=bool)

        #Check all BC from specified locations, prior to setting new ones
        for item in self.labels():
            bcname = (item.split('.')[-1]).replace(self._phase.name+'_',"")
            if bcname in self._existing_BC  and item.split('.')[0]==element:
                if mode=='merge':
                    if not (sp.isnan(self[element+'.'+component.name+'_bcval_'+bcname][loc]).all() and sp.sum(self[element+'.'+component.name+'_'+bcname][loc])==0):
                        raise Exception('Because of the existing BCs, the method cannot apply new BC with the merge mode to the specified pore/throat.')
        #Set boundary conditions based on supplied mode
        if mode == 'merge':
            if bcvalue != []:   self[element+'.'+component.name+'_bcval_'+bctype][loc] = bcvalue
            self[element+'.'+component.name+'_'+bctype][loc] = True
            if bctype not in self._existing_BC: self._existing_BC.append(bctype)
        elif mode == 'overwrite':
            self[element+'.'+component.name+'_bcval_'+bctype] = sp.ones((all_length,),dtype=float)*sp.nan
            if bcvalue != []:   self[element+'.'+component.name+'_bcval_'+bctype][loc] = bcvalue
            self[element+'.'+component.name+'_'+bctype] = sp.zeros((all_length,),dtype=bool)
            self[element+'.'+component.name+'_'+bctype][loc] = True
            if bctype not in self._existing_BC: self._existing_BC.append(bctype)

if __name__ == '__main__':
    pn = OpenPNM.Network.TestNet()
    test = OpenPNM.Algorithms.GenericAlgorithm(network=pn,loglevel=10)
    test.run()
<|endoftext|>


<commit_before><commit_msg>Fix Keras 1 GPU fp16 graph tests (#6545)<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008 Jason Davies
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.

from couchdb import Database, Document, ResourceNotFound, Row, Server, \
    ViewResults
import errno
import fuse
import os
import stat
import sys
from time import time
try:
    import simplejson as json
except ImportError:
    import json # Python 2.6
from urllib import quote, unquote

fuse.fuse_python_api = (0, 2)

class CouchStat(fuse.Stat):
    def __init__(self):
        self.st_mode = 0
        self.st_ino = 0
        self.st_dev = 0
        self.st_nlink = 0
        self.st_uid = 0
        self.st_gid = 0
        self.st_size = 4096
        self.st_atime = 0
        self.st_mtime = 0
        self.st_ctime = 0

class CouchFSDocument(fuse.Fuse):
    def __init__(self, mountpoint, uri=None, *args, **kwargs):
        fuse.Fuse.__init__(self, *args, **kwargs)
        db_uri, doc_id = uri.rsplit('/', 1)
        self.doc_id = unquote(doc_id)
        self.db = Database(db_uri)

    def get_dirs(self):
        dirs = {}
        attachments = self.db[self.doc_id].get('_attachments', {}).keys()
        for att in attachments:
            parents = [u'']
            for name in att.split('/'):
                dirs.setdefault(u'/'.join(parents[1:]), set()).add(name)
                parents.append(name)
        return dirs

    def readdir(self, path, offset):
        for r in '.', '..':
            yield fuse.Direntry(r)
        for dirname in self.get_dirs().get(path[1:], []):
            yield fuse.Direntry(dirname.encode('utf-8'))

    def getattr(self, path):
        try:
            st = CouchStat()
            if path == '/' or path[1:] in self.get_dirs().keys():
                st.st_mode = stat.S_IFDIR | 0775
                st.st_nlink = 2
            else:
                att = self.db[self.doc_id].get('_attachments', {})
                data = att[path[1:]]
                st.st_mode = stat.S_IFREG | 0664
                st.st_nlink = 1
                st.st_size = data['length']
            return st
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT

    def open(self, path, flags):
        try:
            #data = self.db.get_attachment(self.db[self.doc_id], path.split('/')[-1])
            #att = self.db[self.doc_id].get('_attachments', {})
            #data = att[path.split('/')[-1]]
            dirname, filename = path.rsplit('/', 1)
            if filename in self.get_dirs()[dirname[1:]]:
                return 0
            return -errno.ENOENT
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT
        #accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
        #if (flags & accmode) != os.O_RDONLY:
        #    return -errno.EACCES

    def read(self, path, size, offset):
        try:
            data = self.db.get_attachment(self.db[self.doc_id], path[1:])
            slen = len(data)
            if offset < slen:
                if offset + size > slen:
                    size = slen - offset
                buf = data[offset:offset+size]
            else:
                buf = ''
            return buf
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def write(self, path, buf, offset):
        try:
            data = self.db.get_attachment(self.db[self.doc_id], path[1:])
            data = data[0:offset] + buf + data[offset+len(buf):]
            self.db.put_attachment(self.db[self.doc_id], data, filename=path[1:])
            return len(buf)
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def mknod(self, path, mode, dev):
        name = path[1:]
        self.db.put_attachment(self.db[self.doc_id], u'', filename=name)
        return 0

    def unlink(self, path):
        name = path[1:]
        self.db.delete_attachment(self.db[self.doc_id], name)
        return 0

    def truncate(self, path, size):
        name = path[1:]
        self.db.put_attachment(self.db[self.doc_id], u'', filename=name)
        return 0

    def utime(self, path, times):
        return 0

    def mkdir(self, path, mode):
        self.db.put_attachment(self.db[self.doc_id], u'', filename=u'%s/%s' % (path[1:], u'.couchdb-fuse-placeholder'))
        return 0

    def rmdir(self, path):
        for filename in self.get_dirs().get(path[1:]):
            self.db.delete_attachment(self.db[self.doc_id], u'%s/%s' % (path[1:], filename))
        return 0

    def rename(self, pathfrom, pathto):
        return 0

    def fsync(self, path, isfsyncfile):
        return 0

    def statfs(self):
        """
        Should return a tuple with the following 6 elements:
            - blocksize - size of file blocks, in bytes
            - totalblocks - total number of blocks in the filesystem
            - freeblocks - number of free blocks
            - availblocks - number of blocks available to non-superuser
            - totalfiles - total number of file inodes
            - freefiles - nunber of free file inodes

        Feel free to set any of the above values to 0, which tells
        the kernel that the info is not available.
        """
        st = fuse.StatVfs()
        block_size = 1024
        blocks = 1024 * 1024
        blocks_free = blocks
        blocks_avail = blocks_free
        files = 0
        files_free = 0
        st.f_bsize = block_size
        st.f_frsize = block_size
        st.f_blocks = blocks
        st.f_bfree = blocks_free
        st.f_bavail = blocks_avail
        st.f_files = files
        st.f_ffree = files_free
        return st

class CouchFS(fuse.Fuse):
    """FUSE interface to a CouchDB database."""

    def __init__(self, mountpoint, uri=None, *args, **kw):
        fuse.Fuse.__init__(self, *args, **kw)
        self.fuse_args.mountpoint = mountpoint
        if uri is not None:
            self.server = Server(uri)
        else:
            self.server = Server()

    def getcouchattrs(self, path):
        attr = self.server
        attrs = [attr]
        parts = [x for x in path[1:].split('/') if x != '']
        i = 0
        for part in parts:
            if isinstance(attr, Database):
                if part == '_view':
                    attr = attr.view('_all_docs')['_design/':'_design/ZZZ']
                    attr.is_view = True
                elif part == '_all_docs':
                    attr = attr.view('_all_docs')
            elif isinstance(attr, ViewResults):
                if getattr(attr, 'is_view', False):
                    attr = list(attr['_design/'+part])[0]
                    attr.is_view = True
                else:
                    if attr.view.name != '_all_docs':
                        part = json.loads(unquote(part))
                    results = list(attr.view()[part])
                    if len(results) == 1:
                        attr = results[0]
                    else:
                        attr = attr.view()[part+'/':part+'/ZZZ']
                        if i + 1 < len(parts):
                            parts[i+1] = '%s/%s' % (part, parts[i+1])
            elif isinstance(attr, Row):
                if getattr(attr, 'is_view', False):
                    db = self.server[parts[0]]
                    attr = db.view('_view/' + '/'.join(parts[i-1:i+1]), group=True)
                elif part == 'value':
                    attr = attr.value
            else:
                attr = attr[part]
            attrs.append(attr)
            i += 1
        return attrs

    def getattr(self, path):
        try:
            st = CouchStat()
            attr = self.getcouchattrs(path)[-1]
            if (isinstance(attr, Server) or isinstance(attr, Database) or
                    isinstance(attr, Document) or isinstance(attr, ViewResults)
                    or isinstance(attr, Row)):
                st.st_mode = stat.S_IFDIR | 0755
                st.st_nlink = 2
            else:
                data = json.dumps(attr)
                st.st_mode = stat.S_IFREG | 0444
                st.st_nlink = 1
                st.st_size = len(data)
            return st
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT

    def readdir(self, path, offset):
        attr = self.getcouchattrs(path)[-1]
        for r in '.', '..':
            yield fuse.Direntry(r)
        if isinstance(attr, Server):
            for db_name in self.server:
                yield fuse.Direntry(db_name.encode('utf-8'))
            return
        if isinstance(attr, Database):
            yield fuse.Direntry('_all_docs')
            yield fuse.Direntry('_view')
            return
        if isinstance(attr, ViewResults):
            is_view = getattr(attr, 'is_view', False)
            for row in list(attr):
                dirname = row.key
                if attr.view.name != '_all_docs':
                    dirname = quote(json.dumps(row.key))
                else:
                    dirname = dirname.split('/')[attr.options.get('startkey', '').count('/')]
                if is_view:
                    dirname = dirname.split('/')[-1]
                yield fuse.Direntry(dirname.encode('utf-8'))
            return
        if isinstance(attr, Row):
            is_view = getattr(attr, 'is_view', False)
            if is_view:
                db = self.getcouchattrs(path)[1]
                for r in db[attr.id]['views']:
                    yield fuse.Direntry(r.encode('utf-8'))
            else:
                yield fuse.Direntry('value')
            return
        for r in attr.keys():
            yield fuse.Direntry(r.encode('utf-8'))

    def open(self, path, flags):
        try:
            attr = self.getcouchattrs(path)[-1]
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT
        accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
        if (flags & accmode) != os.O_RDONLY:
            return -errno.EACCES

    def read(self, path, size, offset):
        try:
            attr = self.getcouchattrs(path)[-1]
            data = json.dumps(attr)
            slen = len(data)
            if offset < slen:
                if offset + size > slen:
                    size = slen - offset
                buf = data[offset:offset+size]
            else:
                buf = ''
            return buf
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def write(self, path, buf, offset):
        pass

    def unlink(self, path):
        attr = self.getcouchattrs(path)[-1]

    def mkdir(self, path, mode):
        server = self.getcouchattrs(path)[0]
        if isinstance(server, Server):
            server.create(path.split('/')[-1])

def main():
    args = sys.argv[1:]
    if len(args) not in (2,):
        print "CouchDB FUSE Connector: Allows you to browse the _attachments of"
        print " any CouchDB document on your own filesystem!"
        print
        print "Remember to URL-encode your <doc_id> appropriately."
        print
        print "Usage: python couchmount.py <http://hostname:port/db/doc_id> <mount-point>"
        sys.exit(-1)

    if len(args) == 1:
        fs = CouchFS(args[0])
    elif len(args) == 2:
        fs = CouchFSDocument(args[1], args[0])

    fs.parse(errex=1)
    fs.main()
<commit_msg>More updates to make directories work.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008 Jason Davies
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.

from couchdb import Database, Document, ResourceNotFound, Row, Server, \
    ViewResults
import errno
import fuse
import os
import stat
import sys
from time import time
try:
    import simplejson as json
except ImportError:
    import json # Python 2.6
from urllib import quote, unquote

fuse.fuse_python_api = (0, 2)

COUCHFS_DIRECTORY_PLACEHOLDER = u'.couchfs-directory-placeholder'

class CouchStat(fuse.Stat):
    def __init__(self):
        self.st_mode = 0
        self.st_ino = 0
        self.st_dev = 0
        self.st_nlink = 0
        self.st_uid = 0
        self.st_gid = 0
        self.st_size = 4096
        self.st_atime = 0
        self.st_mtime = 0
        self.st_ctime = 0

def _normalize_path(path):
    return u'/'.join([part for part in path.split(u'/') if part != u''])

class CouchFSDocument(fuse.Fuse):
    def __init__(self, mountpoint, uri=None, *args, **kwargs):
        fuse.Fuse.__init__(self, *args, **kwargs)
        db_uri, doc_id = uri.rsplit('/', 1)
        self.doc_id = unquote(doc_id)
        self.db = Database(db_uri)

    def get_dirs(self):
        dirs = {}
        attachments = self.db[self.doc_id].get('_attachments', {}).keys()
        for att in attachments:
            parents = [u'']
            for name in att.split('/'):
                filenames = dirs.setdefault(u'/'.join(parents[1:]), set())
                if name != COUCHFS_DIRECTORY_PLACEHOLDER:
                    filenames.add(name)
                    parents.append(name)
        return dirs

    def readdir(self, path, offset):
        path = _normalize_path(path)
        for r in '.', '..':
            yield fuse.Direntry(r)
        for name in self.get_dirs().get(path, []):
            yield fuse.Direntry(name.encode('utf-8'))

    def getattr(self, path):
        path = _normalize_path(path)
        try:
            st = CouchStat()
            if path == '' or path in self.get_dirs().keys():
                st.st_mode = stat.S_IFDIR | 0775
                st.st_nlink = 2
            else:
                att = self.db[self.doc_id].get('_attachments', {})
                data = att[path]
                st.st_mode = stat.S_IFREG | 0664
                st.st_nlink = 1
                st.st_size = data['length']
            return st
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT

    def open(self, path, flags):
        path = _normalize_path(path)
        try:
            #data = self.db.get_attachment(self.db[self.doc_id], path.split('/')[-1])
            #att = self.db[self.doc_id].get('_attachments', {})
            #data = att[path.split('/')[-1]]
            parts = path.rsplit(u'/', 1)
            if len(parts) == 1:
                dirname, filename = u'', parts[0]
            else:
                dirname, filename = parts
            if filename in self.get_dirs()[dirname]:
                return 0
            return -errno.ENOENT
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT
        #accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
        #if (flags & accmode) != os.O_RDONLY:
        #    return -errno.EACCES

    def read(self, path, size, offset):
        path = _normalize_path(path)
        try:
            data = self.db.get_attachment(self.db[self.doc_id], path)
            slen = len(data)
            if offset < slen:
                if offset + size > slen:
                    size = slen - offset
                buf = data[offset:offset+size]
            else:
                buf = ''
            return buf
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def write(self, path, buf, offset):
        path = _normalize_path(path)
        try:
            data = self.db.get_attachment(self.db[self.doc_id], path)
            data = data[0:offset] + buf + data[offset+len(buf):]
            self.db.put_attachment(self.db[self.doc_id], data, filename=path)
            return len(buf)
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def mknod(self, path, mode, dev):
        path = _normalize_path(path)
        self.db.put_attachment(self.db[self.doc_id], u'', filename=path)
        return 0

    def unlink(self, path):
        path = _normalize_path(path)
        parts = path.rsplit(u'/', 1)
        if len(parts) == 1:
            dirname, filename = u'', parts[0]
        else:
            dirname, filename = parts
        self.db.delete_attachment(self.db[self.doc_id], path)
        if filename != COUCHFS_DIRECTORY_PLACEHOLDER and len(self.get_dirs().get(dirname, [])) == 0:
            print "putting to:", u'%s/%s' % (dirname, COUCHFS_DIRECTORY_PLACEHOLDER)
            self.db.put_attachment(self.db[self.doc_id], u'', filename=u'%s/%s' % (dirname, COUCHFS_DIRECTORY_PLACEHOLDER))
        return 0

    def truncate(self, path, size):
        path = _normalize_path(path)
        self.db.put_attachment(self.db[self.doc_id], u'', filename=path)
        return 0

    def utime(self, path, times):
        return 0

    def mkdir(self, path, mode):
        path = _normalize_path(path)
        self.db.put_attachment(self.db[self.doc_id], u'', filename=u'%s/%s' % (path, COUCHFS_DIRECTORY_PLACEHOLDER))
        return 0

    def rmdir(self, path):
        path = _normalize_path(path)
        self.db.delete_attachment(self.db[self.doc_id], u'%s/%s' % (path, COUCHFS_DIRECTORY_PLACEHOLDER))
        return 0

    def rename(self, pathfrom, pathto):
        return 0

    def fsync(self, path, isfsyncfile):
        return 0

    def statfs(self):
        """
        Should return a tuple with the following 6 elements:
            - blocksize - size of file blocks, in bytes
            - totalblocks - total number of blocks in the filesystem
            - freeblocks - number of free blocks
            - availblocks - number of blocks available to non-superuser
            - totalfiles - total number of file inodes
            - freefiles - nunber of free file inodes

        Feel free to set any of the above values to 0, which tells
        the kernel that the info is not available.
        """
        st = fuse.StatVfs()
        block_size = 1024
        blocks = 1024 * 1024
        blocks_free = blocks
        blocks_avail = blocks_free
        files = 0
        files_free = 0
        st.f_bsize = block_size
        st.f_frsize = block_size
        st.f_blocks = blocks
        st.f_bfree = blocks_free
        st.f_bavail = blocks_avail
        st.f_files = files
        st.f_ffree = files_free
        return st

class CouchFS(fuse.Fuse):
    """FUSE interface to a CouchDB database."""

    def __init__(self, mountpoint, uri=None, *args, **kw):
        fuse.Fuse.__init__(self, *args, **kw)
        self.fuse_args.mountpoint = mountpoint
        if uri is not None:
            self.server = Server(uri)
        else:
            self.server = Server()

    def getcouchattrs(self, path):
        attr = self.server
        attrs = [attr]
        parts = [x for x in path[1:].split('/') if x != '']
        i = 0
        for part in parts:
            if isinstance(attr, Database):
                if part == '_view':
                    attr = attr.view('_all_docs')['_design/':'_design/ZZZ']
                    attr.is_view = True
                elif part == '_all_docs':
                    attr = attr.view('_all_docs')
            elif isinstance(attr, ViewResults):
                if getattr(attr, 'is_view', False):
                    attr = list(attr['_design/'+part])[0]
                    attr.is_view = True
                else:
                    if attr.view.name != '_all_docs':
                        part = json.loads(unquote(part))
                    results = list(attr.view()[part])
                    if len(results) == 1:
                        attr = results[0]
                    else:
                        attr = attr.view()[part+'/':part+'/ZZZ']
                        if i + 1 < len(parts):
                            parts[i+1] = '%s/%s' % (part, parts[i+1])
            elif isinstance(attr, Row):
                if getattr(attr, 'is_view', False):
                    db = self.server[parts[0]]
                    attr = db.view('_view/' + '/'.join(parts[i-1:i+1]), group=True)
                elif part == 'value':
                    attr = attr.value
            else:
                attr = attr[part]
            attrs.append(attr)
            i += 1
        return attrs

    def getattr(self, path):
        try:
            st = CouchStat()
            attr = self.getcouchattrs(path)[-1]
            if (isinstance(attr, Server) or isinstance(attr, Database) or
                    isinstance(attr, Document) or isinstance(attr, ViewResults)
                    or isinstance(attr, Row)):
                st.st_mode = stat.S_IFDIR | 0755
                st.st_nlink = 2
            else:
                data = json.dumps(attr)
                st.st_mode = stat.S_IFREG | 0444
                st.st_nlink = 1
                st.st_size = len(data)
            return st
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT

    def readdir(self, path, offset):
        attr = self.getcouchattrs(path)[-1]
        for r in '.', '..':
            yield fuse.Direntry(r)
        if isinstance(attr, Server):
            for db_name in self.server:
                yield fuse.Direntry(db_name.encode('utf-8'))
            return
        if isinstance(attr, Database):
            yield fuse.Direntry('_all_docs')
            yield fuse.Direntry('_view')
            return
        if isinstance(attr, ViewResults):
            is_view = getattr(attr, 'is_view', False)
            for row in list(attr):
                dirname = row.key
                if attr.view.name != '_all_docs':
                    dirname = quote(json.dumps(row.key))
                else:
                    dirname = dirname.split('/')[attr.options.get('startkey', '').count('/')]
                if is_view:
                    dirname = dirname.split('/')[-1]
                yield fuse.Direntry(dirname.encode('utf-8'))
            return
        if isinstance(attr, Row):
            is_view = getattr(attr, 'is_view', False)
            if is_view:
                db = self.getcouchattrs(path)[1]
                for r in db[attr.id]['views']:
                    yield fuse.Direntry(r.encode('utf-8'))
            else:
                yield fuse.Direntry('value')
            return
        for r in attr.keys():
            yield fuse.Direntry(r.encode('utf-8'))

    def open(self, path, flags):
        try:
            attr = self.getcouchattrs(path)[-1]
        except (KeyError, ResourceNotFound):
            return -errno.ENOENT
        accmode = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
        if (flags & accmode) != os.O_RDONLY:
            return -errno.EACCES

    def read(self, path, size, offset):
        try:
            attr = self.getcouchattrs(path)[-1]
            data = json.dumps(attr)
            slen = len(data)
            if offset < slen:
                if offset + size > slen:
                    size = slen - offset
                buf = data[offset:offset+size]
            else:
                buf = ''
            return buf
        except (KeyError, ResourceNotFound):
            pass
        return -errno.ENOENT

    def write(self, path, buf, offset):
        pass

    def unlink(self, path):
        attr = self.getcouchattrs(path)[-1]

    def mkdir(self, path, mode):
        server = self.getcouchattrs(path)[0]
        if isinstance(server, Server):
            server.create(path.split('/')[-1])

def main():
    args = sys.argv[1:]
    if len(args) not in (2, 3):
        print "CouchDB FUSE Connector: Allows you to browse the _attachments of"
        print " any CouchDB document on your own filesystem!"
        print
        print "Remember to URL-encode your <doc_id> appropriately."
        print
        print "Usage: python couchmount.py [-d] <http://hostname:port/db/doc_id> <mount-point>"
        sys.exit(-1)

    if len(args) == 1:
        fs = CouchFS(args[0])
    elif len(args) == 2:
        fs = CouchFSDocument(args[1], args[0])
    elif len(args) == 3:
        fs = CouchFSDocument(args[2], args[1])

    fs.parse(errex=1)
    fs.main()
<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import heapq
import logging
import os
import os.path
import shutil
import subprocess as subprocess
import sys
import tempfile
import time

from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend
from telemetry.util import path
from telemetry.util import support_binaries

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, browser_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path, extensions_to_load):
    super(DesktopBrowserBackend, self).__init__(
        supports_tab_control=not is_content_shell,
        supports_extensions=not is_content_shell,
        browser_options=browser_options,
        output_profile_path=output_profile_path,
        extensions_to_load=extensions_to_load)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    assert not flash_path or os.path.exists(flash_path)
    self._flash_path = flash_path

    self._is_content_shell = is_content_shell

    if len(extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = None
    self._profile_dir = None
    self._tmp_minidump_dir = tempfile.mkdtemp()
    self._crash_service = None

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.browser_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.browser_options.profile_dir
      if profile_dir:
        if self._is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)
    if self.browser_options.use_devtools_active_port:
      # No matter whether we're using an existing profile directory or
      # creating a new one, always delete the well-known file containing
      # the active DevTools port number.
      port_file = self._GetDevToolsActivePortPath()
      if os.path.isfile(port_file):
        try:
          os.remove(port_file)
        except Exception as e:
          logging.critical('Unable to remove DevToolsActivePort file: %s' % e)
          sys.exit(1)

  def _GetDevToolsActivePortPath(self):
    return os.path.join(self.profile_directory, 'DevToolsActivePort')

  def _GetCrashServicePipeName(self):
    # Ensure a unique pipe name by using the name of the temp dir.
    return r'\\.\pipe\%s_service' % os.path.basename(self._tmp_minidump_dir)

  def _StartCrashService(self):
    os_name = self._browser.platform.GetOSName()
    if os_name != 'win':
      return None
    return subprocess.Popen([
        support_binaries.FindPath('crash_service', os_name),
        '--no-window',
        '--dumps-dir=%s' % self._tmp_minidump_dir,
        '--pipe-name=%s' % self._GetCrashServicePipeName()])

  def _GetCdbPath(self):
    possible_paths = (
        'Debugging Tools For Windows',
        'Debugging Tools For Windows (x86)',
        'Debugging Tools For Windows (x64)',
        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x86'),
        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x64'),
        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
                     'x86'),
        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
                     'x64'),
    )
    for possible_path in possible_paths:
      app_path = os.path.join(possible_path, 'cdb.exe')
      app_path = path.FindInstalledWindowsApplication(app_path)
      if app_path:
        return app_path
    return None

  def HasBrowserFinishedLaunching(self):
    # In addition to the functional check performed by the base class, quickly
    # check if the browser process is still alive.
    if not self.IsBrowserRunning():
      raise exceptions.ProcessGoneException(
          "Return code: %d" % self._proc.returncode)
    if self.browser_options.use_devtools_active_port:
      # The Telemetry user selected the new code path to start DevTools on
      # an ephemeral port. Wait for the well-known file containing the port
      # number to exist.
      port_file = self._GetDevToolsActivePortPath()
      if not os.path.isfile(port_file):
        # File isn't ready yet. Return false. Will retry.
        return False
      # Attempt to avoid reading the file until it's populated.
      got_port = False
      try:
        if os.stat(port_file).st_size > 0:
          with open(port_file) as f:
            port_string = f.read()
            self._port = int(port_string)
            logging.info('Discovered ephemeral port %s' % self._port)
            got_port = True
      except Exception:
        # Both stat and open can throw exceptions.
        pass
      if not got_port:
        # File isn't ready yet. Return false. Will retry.
        return False
    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    if self.browser_options.use_devtools_active_port:
      self._port = 0
    else:
      self._port = util.GetUnreservedAvailableLocalPort()
    logging.info('Requested remote debugging port: %d' % self._port)
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    args.append('--use-mock-keychain')
    if not self._is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if not self.browser_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self._is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    assert not self._proc, 'Must call Close() before Start()'

    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    if self.browser_options.startup_url:
      args.append(self.browser_options.startup_url)
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    env['CHROME_BREAKPAD_PIPE_NAME'] = self._GetCrashServicePipeName()
    self._crash_service = self._StartCrashService()
    logging.debug('Starting Chrome %s', args)
    if not self.browser_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
    except:
      self.Close()
      raise

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc and self._proc.poll() == None

  def GetStandardOutput(self):
    if not self._tmp_output_file:
      if self.browser_options.show_stdout:
        # This can happen in the case that loading the Chrome binary fails.
        # We print rather than using logging here, because that makes a
        # recursive call to this function.
        print >> sys.stderr, "Can't get standard output with --show-stdout"
      return ''
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def _GetMostRecentMinidump(self):
    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      return None
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warning('Crash dump is older than 5 minutes. May not be correct.')
    return most_recent_dump

  def _GetStackFromMinidump(self, minidump):
    os_name = self._browser.platform.GetOSName()
    if os_name == 'win':
      cdb = self._GetCdbPath()
      if not cdb:
        logging.warning('cdb.exe not found.')
        return None
      output = subprocess.check_output([cdb, '-y', self._browser_directory,
                                        '-c', '.ecxr;k30;q', '-z', minidump])
      stack_start = output.find('ChildEBP')
      stack_end = output.find('quit:')
      return output[stack_start:stack_end]

    stackwalk = support_binaries.FindPath('minidump_stackwalk', os_name)
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found.')
      return None

    symbols = glob.glob(os.path.join(self._browser_directory, '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found.')
      return None

    with open(minidump, 'rb') as infile:
      minidump += '.stripped'
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
      if not os.path.isfile(symbol):
        continue
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      if os.path.exists(symbol_path):
        continue
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    return subprocess.check_output([stackwalk, minidump, symbols_path],
                                   stderr=open(os.devnull, 'w'))

  def GetStackTrace(self):
    most_recent_dump = self._GetMostRecentMinidump()
    if not most_recent_dump:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()

    stack = self._GetStackFromMinidump(most_recent_dump)
    if not stack:
      logging.warning('Failed to symbolize minidump. Returning browser stdout.')
      return self.GetStandardOutput()

    return stack

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    # First, try to politely shutdown.
    if self.IsBrowserRunning():
      self._proc.terminate()
      try:
        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
        self._proc = None
      except util.TimeoutException:
        logging.warning('Failed to gracefully shutdown. Proceeding to kill.')

    # If it didn't comply, get more aggressive.
    if self.IsBrowserRunning():
      self._proc.kill()

    try:
      util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=10)
    except util.TimeoutException:
      raise Exception('Could not shutdown the browser.')
    finally:
      self._proc = None

    if self._crash_service:
      self._crash_service.kill()
      self._crash_service = None

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None
<commit_msg>[Telemetry] Terminate browsers more aggressively.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import heapq
import logging
import os
import os.path
import shutil
import subprocess as subprocess
import sys
import tempfile
import time

from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend
from telemetry.util import path
from telemetry.util import support_binaries

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, browser_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path, extensions_to_load):
    super(DesktopBrowserBackend, self).__init__(
        supports_tab_control=not is_content_shell,
        supports_extensions=not is_content_shell,
        browser_options=browser_options,
        output_profile_path=output_profile_path,
        extensions_to_load=extensions_to_load)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    assert not flash_path or os.path.exists(flash_path)
    self._flash_path = flash_path

    self._is_content_shell = is_content_shell

    if len(extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = None
    self._profile_dir = None
    self._tmp_minidump_dir = tempfile.mkdtemp()
    self._crash_service = None

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.browser_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.browser_options.profile_dir
      if profile_dir:
        if self._is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)
    if self.browser_options.use_devtools_active_port:
      # No matter whether we're using an existing profile directory or
      # creating a new one, always delete the well-known file containing
      # the active DevTools port number.
      port_file = self._GetDevToolsActivePortPath()
      if os.path.isfile(port_file):
        try:
          os.remove(port_file)
        except Exception as e:
          logging.critical('Unable to remove DevToolsActivePort file: %s' % e)
          sys.exit(1)

  def _GetDevToolsActivePortPath(self):
    return os.path.join(self.profile_directory, 'DevToolsActivePort')

  def _GetCrashServicePipeName(self):
    # Ensure a unique pipe name by using the name of the temp dir.
    return r'\\.\pipe\%s_service' % os.path.basename(self._tmp_minidump_dir)

  def _StartCrashService(self):
    os_name = self._browser.platform.GetOSName()
    if os_name != 'win':
      return None
    return subprocess.Popen([
        support_binaries.FindPath('crash_service', os_name),
        '--no-window',
        '--dumps-dir=%s' % self._tmp_minidump_dir,
        '--pipe-name=%s' % self._GetCrashServicePipeName()])

  def _GetCdbPath(self):
    possible_paths = (
        'Debugging Tools For Windows',
        'Debugging Tools For Windows (x86)',
        'Debugging Tools For Windows (x64)',
        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x86'),
        os.path.join('Windows Kits', '8.0', 'Debuggers', 'x64'),
        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
                     'x86'),
        os.path.join('win_toolchain', 'vs2013_files', 'win8sdk', 'Debuggers',
                     'x64'),
    )
    for possible_path in possible_paths:
      app_path = os.path.join(possible_path, 'cdb.exe')
      app_path = path.FindInstalledWindowsApplication(app_path)
      if app_path:
        return app_path
    return None

  def HasBrowserFinishedLaunching(self):
    # In addition to the functional check performed by the base class, quickly
    # check if the browser process is still alive.
    if not self.IsBrowserRunning():
      raise exceptions.ProcessGoneException(
          "Return code: %d" % self._proc.returncode)
    if self.browser_options.use_devtools_active_port:
      # The Telemetry user selected the new code path to start DevTools on
      # an ephemeral port. Wait for the well-known file containing the port
      # number to exist.
      port_file = self._GetDevToolsActivePortPath()
      if not os.path.isfile(port_file):
        # File isn't ready yet. Return false. Will retry.
        return False
      # Attempt to avoid reading the file until it's populated.
      got_port = False
      try:
        if os.stat(port_file).st_size > 0:
          with open(port_file) as f:
            port_string = f.read()
            self._port = int(port_string)
            logging.info('Discovered ephemeral port %s' % self._port)
            got_port = True
      except Exception:
        # Both stat and open can throw exceptions.
        pass
      if not got_port:
        # File isn't ready yet. Return false. Will retry.
        return False
    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    if self.browser_options.use_devtools_active_port:
      self._port = 0
    else:
      self._port = util.GetUnreservedAvailableLocalPort()
    logging.info('Requested remote debugging port: %d' % self._port)
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    args.append('--use-mock-keychain')
    if not self._is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if not self.browser_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self._is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    assert not self._proc, 'Must call Close() before Start()'

    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    if self.browser_options.startup_url:
      args.append(self.browser_options.startup_url)
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    env['CHROME_BREAKPAD_PIPE_NAME'] = self._GetCrashServicePipeName()
    self._crash_service = self._StartCrashService()
    logging.debug('Starting Chrome %s', args)
    if not self.browser_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
    except:
      self.Close()
      raise

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc and self._proc.poll() == None

  def GetStandardOutput(self):
    if not self._tmp_output_file:
      if self.browser_options.show_stdout:
        # This can happen in the case that loading the Chrome binary fails.
        # We print rather than using logging here, because that makes a
        # recursive call to this function.
        print >> sys.stderr, "Can't get standard output with --show-stdout"
      return ''
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def _GetMostRecentMinidump(self):
    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      return None
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warning('Crash dump is older than 5 minutes. May not be correct.')
    return most_recent_dump

  def _GetStackFromMinidump(self, minidump):
    os_name = self._browser.platform.GetOSName()
    if os_name == 'win':
      cdb = self._GetCdbPath()
      if not cdb:
        logging.warning('cdb.exe not found.')
        return None
      output = subprocess.check_output([cdb, '-y', self._browser_directory,
                                        '-c', '.ecxr;k30;q', '-z', minidump])
      stack_start = output.find('ChildEBP')
      stack_end = output.find('quit:')
      return output[stack_start:stack_end]

    stackwalk = support_binaries.FindPath('minidump_stackwalk', os_name)
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found.')
      return None

    symbols = glob.glob(os.path.join(self._browser_directory, '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found.')
      return None

    with open(minidump, 'rb') as infile:
      minidump += '.stripped'
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
      if not os.path.isfile(symbol):
        continue
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      if os.path.exists(symbol_path):
        continue
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    return subprocess.check_output([stackwalk, minidump, symbols_path],
                                   stderr=open(os.devnull, 'w'))

  def GetStackTrace(self):
    most_recent_dump = self._GetMostRecentMinidump()
    if not most_recent_dump:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()

    stack = self._GetStackFromMinidump(most_recent_dump)
    if not stack:
      logging.warning('Failed to symbolize minidump. Returning browser stdout.')
      return self.GetStandardOutput()

    return stack

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    # Shutdown politely if the profile may be used again.
    if self._output_profile_path and self.IsBrowserRunning():
      self._proc.terminate()
      try:
        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
        self._proc = None
      except util.TimeoutException:
        logging.warning('Failed to gracefully shutdown. Proceeding to kill.')

    # Shutdown aggressively if the above failed or if the profile is temporary.
    if self.IsBrowserRunning():
      self._proc.kill()

    try:
      util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=10)
    except util.TimeoutException:
      raise Exception('Could not shutdown the browser.')
    finally:
      self._proc = None

    if self._crash_service:
      self._crash_service.kill()
      self._crash_service = None

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None
<|endoftext|>


<commit_before>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		UseExceptions()
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))
		os.system("cp %s %s" % (ehdr, "~/projects/dg9/downsamp.hdr") )

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0)

		# ensure decent data is retrieved
		mx = amax(data)
		self.assertTrue(mx != 0)
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<commit_msg>Relaxed value checking in a simple test.<commit_after>'''
Created on 12/09/2012
@author: Ben Davies, ANUSF
		 ben.davies@anu.edu.au
'''

import os, unittest, datetime
from numpy import amin, amax, zeros
from numpy.testing import assert_array_equal

from gdal import Open, UseExceptions

import roipac
import ifgconstants as IFC

class ConversionTests(unittest.TestCase):

	SHORT_HEADER_PATH = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
	FULL_HEADER_PATH  = "../../tests/headers/geo_060619-060828.unw.rsc"
	FULL_HEADER_PATH2 = "../../tests/single/geo_060619-061002.unw.rsc"

	def test_read_short_roipac_header(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs is not None)
		self.assertEqual(hdrs[IFC.WIDTH], 47)
		self.assertEqual(hdrs[IFC.FILE_LENGTH], 72)
		self.assertAlmostEqual(hdrs[IFC.X_FIRST], 150.910)

	def test_read_roipac_header_file(self):
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertEqual(hdrs[IFC.X_STEP], 0.000833333)
		self.assertEqual(hdrs[IFC.Y_FIRST], -34.170000000)
		self.assertEqual(hdrs[IFC.Y_STEP], -0.000833333)
		self.assertEqual(hdrs[IFC.WAVELENGTH], 0.0562356424)

	def test_parse_short_date_pre2000(self):
		dstr = "980416"
		self.assertEqual(datetime.date(1998, 4, 16), roipac.parse_date(dstr))

	def test_parse_short_date_post2000(self):
		dstr = "081006"
		self.assertEqual(datetime.date(2008, 10, 6), roipac.parse_date(dstr))

	def test_parse_date_range(self):
		dstr = "980416-081006"
		exp = (datetime.date(1998, 4, 16), datetime.date(2008, 10, 6))
		self.assertEqual(exp, roipac.parse_date(dstr))

	def test_read_full_roipac_header(self):
		"Tests full original header can be parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)

		# check dates are parsed correctly
		date0 = datetime.date(2006, 6, 19) # from  "DATE 060619" header
		date12 = (date0, datetime.date(2006, 8, 28)) # from DATE12   060619-060828
		self.assertEqual(hdrs[IFC.DATE], date0)
		self.assertEqual(hdrs[IFC.DATE12], date12)

	def test_date_alias(self):
		"""Test header has MASTER and SLAVE dates as keys"""
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.MASTER))
		self.assertTrue(hdrs.has_key(IFC.SLAVE))
		self.assertEqual(hdrs[IFC.DATE], hdrs[IFC.MASTER])
		self.assertEqual(hdrs[IFC.DATE12][-1], hdrs[IFC.SLAVE])

	def test_timespan(self):
		"""Ensures the TIME_SPAN_YEAR element is present after parsing short header"""
		hdrs = roipac.parse_header(self.SHORT_HEADER_PATH)
		self.assertTrue(hdrs.has_key(IFC.TIME_SPAN_YEAR))

		# check time span calc
		master = datetime.date(2006, 06, 19)
		slave = datetime.date(2006, 10, 02)
		diff = (slave - master).days / 365.25
		self.assertEqual(diff, hdrs[IFC.TIME_SPAN_YEAR])

	def test_xylast(self):
		# Test the X_LAST and Y_LAST header elements are added
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertAlmostEqual(hdrs[IFC.X_LAST], 151.8519444445)
		self.assertAlmostEqual(hdrs[IFC.Y_LAST], -34.625)

	def test_read_full_roipac_header2(self):
		"Tests header from cropped original dataset is parsed correctly"
		hdrs = roipac.parse_header(self.FULL_HEADER_PATH)
		self.assertTrue(hdrs is not None)

	def test_filename_pair(self):
		base = "project/run/geo_070709-080310.unw"
		exp_hdr = "project/run/geo_070709-080310.unw.rsc"
		result = roipac.filename_pair(base)
		self.assertEqual( (base, exp_hdr), result)

	def test_roipac_to_ehdr_header(self):
		dest = "/tmp/ehdr.hdr"
		hdr = "../../tests/headers/geo_060619-060828.unw.rsc"
		roipac.to_ehdr_header(hdr, dest)

		with open(dest) as f:
			text = f.read()
			self.assertTrue("ncols 5451" in text)
			self.assertTrue("nrows 4366" in text)
			self.assertTrue("cellsize 0.0002777" in text) # compare to 7 places
			self.assertTrue("xllcorner 150.3377777" in text) # compare to 7 places
			exp_yll = "yllcorner " + str(round(-33.4122222 - (4366 * 0.0002777), 3))
			self.assertTrue(exp_yll in text, "Got " + exp_yll)

		os.remove(dest)

	def test_to_ehdr_header_defaults(self):
		# test default header filename
		base_hdr = os.path.abspath("../../tests/single/geo_060619-061002.unw.rsc")
		hdr = "/tmp/geo_060619-061002.unw.rsc"
		os.symlink(base_hdr, hdr)
		exp_hdr = "/tmp/geo_060619-061002.hdr"

		if os.path.exists(exp_hdr): os.remove(exp_hdr)
		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(exp_hdr))

		# add data to /tmp for GDAL test
		base_data = os.path.abspath("../../tests/single/geo_060619-061002.unw")
		exp_data = "/tmp/geo_060619-061002.unw"
		os.symlink(base_data, exp_data)

		# test GDAL can open the data with the new header & cleanup
		UseExceptions()
		ds = Open(exp_data)
		self.assertTrue(ds is not None)
		bands = ds.GetRasterBand(1), ds.GetRasterBand(1)
		self.assertTrue(all(bands)) # both bands exist?

		del bands, ds
		os.unlink(exp_data)
		os.remove(exp_hdr)
		os.unlink(hdr)

	def test_to_ehdr_header_with_data(self):
		# ensure giving the data file breaks to_ehdr_header()
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		try:
			roipac.to_ehdr_header(src)
			self.fail("Should not be able to accept .unw data file")
		except:
			pass

	def test_to_ehdr_header_gdal(self):
		# test data files can be opened, new headers generated, data is readable
		hdr = "../../tests/sydney_test/obs/geo_060619-061002.unw.rsc"
		ehdr = "../../tests/sydney_test/obs/geo_060619-061002.hdr"
		if os.path.exists(ehdr):
			os.remove(ehdr) # can be left behind if the test fails

		roipac.to_ehdr_header(hdr)
		self.assertTrue(os.path.exists(ehdr))
		os.system("cp %s %s" % (ehdr, "~/projects/dg9/downsamp.hdr") )

		# open with GDAL and ensure there is data
		src = "../../tests/sydney_test/obs/geo_060619-061002.unw"
		ds = Open(src)
		self.assertTrue(ds is not None)

		# check faked amplitude band 1 for 0s
		band = ds.GetRasterBand(1)
		data = band.ReadAsArray()
		shape = data.shape
		assert_array_equal(data, zeros(shape))

		# check phase (band 2) for data
		band = ds.GetRasterBand(2)
		nodata = band.GetNoDataValue()
		self.assertEqual(nodata, 0) # check default ROIPAC NODATA

		# ensure decent data is retrieved
		data = band.ReadAsArray()
		self.assertTrue(amin(data) != 0) # ignore max as NODATA is 0
		self.assertTrue(data.ptp() != 0)

		# cleanup
		os.remove(ehdr)
<|endoftext|>


<commit_before>from multiprocessing.pool import ThreadPool
from unittest import mock

import pytest

from paralleldomain.utilities.lazy_load_cache import CacheEmptyException, LazyLoadCache

class _MockSizeElement:
    def __init__(self, desired_size: int):
        self.desired_size = desired_size

    def __sizeof__(self):
        return self.desired_size

    @staticmethod
    def get_mocked_loader(desired_size: int) -> mock.MagicMock:
        mock_loader = mock.MagicMock()
        element = _MockSizeElement(desired_size=desired_size)
        mock_loader.load.return_value = element
        return mock_loader

class TestLazyLoadCache:
    def test_removes_items_on_exceeding_max_size(self):
        cache = LazyLoadCache(cache_max_size="1200B")

        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)

        cache.get_item(key="key1", loader=mock1_loader.load)
        mock1_loader.load.assert_called_once()
        assert len(cache.keys()) == 1
        assert "key1" in cache
        cache.get_item(key="key2", loader=mock2_loader.load)
        mock2_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" in cache
        assert "key1" in cache

        # key1 and key2 should be present in cache. So no reloading needed
        cache.get_item(key="key1", loader=mock1_loader.load)
        mock1_loader.load.assert_called_once()
        mock2_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" in cache
        assert "key1" in cache

        # loading key3 should remove key2 since its least recently used
        cache.get_item(key="key3", loader=mock3_loader.load)
        mock3_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" not in cache
        assert "key3" in cache
        assert "key1" in cache

        # key1 and key3 should be in cache now so key1 should be removed in favor of key2
        cache.get_item(key="key2", loader=mock2_loader.load)
        assert mock2_loader.load.call_count == 2
        assert len(cache.keys()) == 2
        assert "key1" not in cache
        assert "key3" in cache
        assert "key2" in cache

        # there should only be room for key4 so everything else should be removed to make space
        cache.get_item(key="key4", loader=mock4_loader.load)
        mock4_loader.load.assert_called_once()
        assert len(cache.keys()) == 1
        assert "key1" not in cache
        assert "key3" not in cache
        assert "key2" not in cache
        assert "key4" in cache

    def test_max_change_reduction_drops_values(self):
        cache = LazyLoadCache(cache_max_size="2700B")
        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)
        cache.get_item(key="key1", loader=mock1_loader.load)
        cache.get_item(key="key2", loader=mock2_loader.load)
        cache.get_item(key="key3", loader=mock3_loader.load)
        cache.get_item(key="key4", loader=mock4_loader.load)
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key3 and key4
        cache.maxsize = 1700
        assert len(cache.keys()) == 2
        assert "key1" not in cache
        assert "key3" in cache
        assert "key2" not in cache
        assert "key4" in cache
        # should only leave space for key4
        cache.maxsize = "1100B"
        assert len(cache.keys()) == 1
        assert "key1" not in cache
        assert "key3" not in cache
        assert "key2" not in cache
        assert "key4" in cache

    def test_max_change_increase_keeps_values(self):
        cache = LazyLoadCache(cache_max_size="2700B")
        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)
        cache.get_item(key="key1", loader=mock1_loader.load)
        cache.get_item(key="key2", loader=mock2_loader.load)
        cache.get_item(key="key3", loader=mock3_loader.load)
        cache.get_item(key="key4", loader=mock4_loader.load)
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key3 and key4
        cache.maxsize = 3500
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key4
        cache.maxsize = "1GB"
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache

    def test_thread_safe_delete(self):
        cache = LazyLoadCache(cache_max_size="50KB")
        cache.get_item(key="test", loader=mock.MagicMock)

        cache.pop(key="test", default=None)

        def delete(_):
            with pytest.raises(CacheEmptyException):
                cache.popitem()

        p1 = ThreadPool(140)
        p1.map_async(delete, range(140)).get()
<commit_msg>Byte-string conversion test<commit_after>from multiprocessing.pool import ThreadPool
from unittest import mock

import pytest

from paralleldomain.utilities.lazy_load_cache import CacheEmptyException, LazyLoadCache, byte_str_to_bytes

class _MockSizeElement:
    def __init__(self, desired_size: int):
        self.desired_size = desired_size

    def __sizeof__(self):
        return self.desired_size

    @staticmethod
    def get_mocked_loader(desired_size: int) -> mock.MagicMock:
        mock_loader = mock.MagicMock()
        element = _MockSizeElement(desired_size=desired_size)
        mock_loader.load.return_value = element
        return mock_loader

class TestLazyLoadCache:
    def test_removes_items_on_exceeding_max_size(self):
        cache = LazyLoadCache(cache_max_size="1200B")

        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)

        cache.get_item(key="key1", loader=mock1_loader.load)
        mock1_loader.load.assert_called_once()
        assert len(cache.keys()) == 1
        assert "key1" in cache
        cache.get_item(key="key2", loader=mock2_loader.load)
        mock2_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" in cache
        assert "key1" in cache

        # key1 and key2 should be present in cache. So no reloading needed
        cache.get_item(key="key1", loader=mock1_loader.load)
        mock1_loader.load.assert_called_once()
        mock2_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" in cache
        assert "key1" in cache

        # loading key3 should remove key2 since its least recently used
        cache.get_item(key="key3", loader=mock3_loader.load)
        mock3_loader.load.assert_called_once()
        assert len(cache.keys()) == 2
        assert "key2" not in cache
        assert "key3" in cache
        assert "key1" in cache

        # key1 and key3 should be in cache now so key1 should be removed in favor of key2
        cache.get_item(key="key2", loader=mock2_loader.load)
        assert mock2_loader.load.call_count == 2
        assert len(cache.keys()) == 2
        assert "key1" not in cache
        assert "key3" in cache
        assert "key2" in cache

        # there should only be room for key4 so everything else should be removed to make space
        cache.get_item(key="key4", loader=mock4_loader.load)
        mock4_loader.load.assert_called_once()
        assert len(cache.keys()) == 1
        assert "key1" not in cache
        assert "key3" not in cache
        assert "key2" not in cache
        assert "key4" in cache

    def test_max_change_reduction_drops_values(self):
        cache = LazyLoadCache(cache_max_size="2700B")
        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)
        cache.get_item(key="key1", loader=mock1_loader.load)
        cache.get_item(key="key2", loader=mock2_loader.load)
        cache.get_item(key="key3", loader=mock3_loader.load)
        cache.get_item(key="key4", loader=mock4_loader.load)
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key3 and key4
        cache.maxsize = 1700
        assert len(cache.keys()) == 2
        assert "key1" not in cache
        assert "key3" in cache
        assert "key2" not in cache
        assert "key4" in cache
        # should only leave space for key4
        cache.maxsize = "1100B"
        assert len(cache.keys()) == 1
        assert "key1" not in cache
        assert "key3" not in cache
        assert "key2" not in cache
        assert "key4" in cache

    def test_max_change_increase_keeps_values(self):
        cache = LazyLoadCache(cache_max_size="2700B")
        mock1_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock2_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock3_loader = _MockSizeElement.get_mocked_loader(desired_size=500)
        mock4_loader = _MockSizeElement.get_mocked_loader(desired_size=1000)
        cache.get_item(key="key1", loader=mock1_loader.load)
        cache.get_item(key="key2", loader=mock2_loader.load)
        cache.get_item(key="key3", loader=mock3_loader.load)
        cache.get_item(key="key4", loader=mock4_loader.load)
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key3 and key4
        cache.maxsize = 3500
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache
        # should only leave space for key4
        cache.maxsize = "1GB"
        assert len(cache.keys()) == 4
        assert "key1" in cache
        assert "key3" in cache
        assert "key2" in cache
        assert "key4" in cache

    def test_thread_safe_delete(self):
        cache = LazyLoadCache(cache_max_size="50KB")
        cache.get_item(key="test", loader=mock.MagicMock)

        cache.pop(key="test", default=None)

        def delete(_):
            with pytest.raises(CacheEmptyException):
                cache.popitem()

        p1 = ThreadPool(140)
        p1.map_async(delete, range(140)).get()

    def test_byte_str_to_bytes(self):
        test_values = [
            ("1.048576 MB", "1 MiB", 1_048_576),
            ("2.45366784 MB", "2.34 MiB", 2_453_667),
            ("1 TB", "0.90949470177293 TiB", 1_000_000_000_000),
            ("1 KB", "8 kbit", 1_000),
        ]

        for ga_bytes, bi_bytes, byte_value in test_values:
            ga_bytes_no_whitespace = ga_bytes.replace(" ", "")
            bi_bytes_no_whitespace = ga_bytes.replace(" ", "")

            assert byte_str_to_bytes(byte_str=ga_bytes) == byte_value
            assert byte_str_to_bytes(byte_str=ga_bytes_no_whitespace) == byte_value
            assert byte_str_to_bytes(byte_str=bi_bytes) == byte_value
            assert byte_str_to_bytes(byte_str=bi_bytes_no_whitespace) == byte_value
<|endoftext|>