<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Package-level global flags are defined here, the rest are defined
where they're used.
"""

import getopt
import os
import socket
import sys

import gflags

class FlagValues(gflags.FlagValues):
    """Extension of gflags.FlagValues that allows undefined and runtime flags.

    Unknown flags will be ignored when parsing the command line, but the
    command line will be kept so that it can be replayed if new flags are
    defined after the initial parsing.

    """

    def __init__(self):
        gflags.FlagValues.__init__(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []

    def __call__(self, argv):
        # We're doing some hacky stuff here so that we don't have to copy
        # out all the code of the original verbatim and then tweak a few lines.
        # We're hijacking the output of getopt so we can still return the
        # leftover args at the end
        sneaky_unparsed_args = {"value": None}
        original_argv = list(argv)

        if self.IsGnuGetOpt():
            orig_getopt = getattr(getopt, 'gnu_getopt')
            orig_name = 'gnu_getopt'
        else:
            orig_getopt = getattr(getopt, 'getopt')
            orig_name = 'getopt'

        def _sneaky(*args, **kw):
            optlist, unparsed_args = orig_getopt(*args, **kw)
            sneaky_unparsed_args['value'] = unparsed_args
            return optlist, unparsed_args

        try:
            setattr(getopt, orig_name, _sneaky)
            args = gflags.FlagValues.__call__(self, argv)
        except gflags.UnrecognizedFlagError:
            # Undefined args were found, for now we don't care so just
            # act like everything went well
            # (these three lines are copied pretty much verbatim from the end
            # of the __call__ function we are wrapping)
            unparsed_args = sneaky_unparsed_args['value']
            if unparsed_args:
                if self.IsGnuGetOpt():
                    args = argv[:1] + unparsed_args
                else:
                    args = argv[:1] + original_argv[-len(unparsed_args):]
            else:
                args = argv[:1]
        finally:
            setattr(getopt, orig_name, orig_getopt)

        # Store the arguments for later, we'll need them for new flags
        # added at runtime
        self.__dict__['__stored_argv'] = original_argv
        self.__dict__['__was_already_parsed'] = True
        self.ClearDirty()
        return args

    def SetDirty(self, name):
        """Mark a flag as dirty so that accessing it will case a reparse."""
        self.__dict__['__dirty'].append(name)

    def IsDirty(self, name):
        return name in self.__dict__['__dirty']

    def ClearDirty(self):
        self.__dict__['__is_dirty'] = []

    def WasAlreadyParsed(self):
        return self.__dict__['__was_already_parsed']

    def ParseNewFlags(self):
        if '__stored_argv' not in self.__dict__:
            return
        new_flags = FlagValues()
        for k in self.__dict__['__dirty']:
            new_flags[k] = gflags.FlagValues.__getitem__(self, k)

        new_flags(self.__dict__['__stored_argv'])
        for k in self.__dict__['__dirty']:
            setattr(self, k, getattr(new_flags, k))
        self.ClearDirty()

    def __setitem__(self, name, flag):
        gflags.FlagValues.__setitem__(self, name, flag)
        if self.WasAlreadyParsed():
            self.SetDirty(name)

    def __getitem__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getitem__(self, name)

    def __getattr__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getattr__(self, name)

FLAGS = FlagValues()

def _wrapper(func):
    def _wrapped(*args, **kw):
        kw.setdefault('flag_values', FLAGS)
        func(*args, **kw)
    _wrapped.func_name = func.func_name
    return _wrapped

DEFINE = _wrapper(gflags.DEFINE)
DEFINE_string = _wrapper(gflags.DEFINE_string)
DEFINE_integer = _wrapper(gflags.DEFINE_integer)
DEFINE_bool = _wrapper(gflags.DEFINE_bool)
DEFINE_boolean = _wrapper(gflags.DEFINE_boolean)
DEFINE_float = _wrapper(gflags.DEFINE_float)
DEFINE_enum = _wrapper(gflags.DEFINE_enum)
DEFINE_list = _wrapper(gflags.DEFINE_list)
DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist)
DEFINE_multistring = _wrapper(gflags.DEFINE_multistring)
DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int)

def DECLARE(name, module_string, flag_values=FLAGS):
    if module_string not in sys.modules:
        __import__(module_string, globals(), locals())
    if name not in flag_values:
        raise gflags.UnrecognizedFlag(
                "%s not defined by %s" % (name, module_string))

# __GLOBAL FLAGS ONLY__
# Define any app-specific flags in their own files, docs at:
# http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39

DEFINE_list('region_list',
            [],
            'list of region=url pairs separated by commas')
DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
DEFINE_integer('s3_port', 3333, 's3 port')
DEFINE_string('s3_host', '127.0.0.1', 's3 host')
DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on')
DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on')
DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on')
DEFINE_string('network_topic', 'network', 'the topic network nodes listen on')

DEFINE_bool('verbose', False, 'show debug output')
DEFINE_boolean('fake_rabbit', False, 'use a fake rabbit')
DEFINE_bool('fake_network', False,
            'should we use fake network devices and addresses')
DEFINE_string('rabbit_host', 'localhost', 'rabbit host')
DEFINE_integer('rabbit_port', 5672, 'rabbit port')
DEFINE_string('rabbit_userid', 'guest', 'rabbit userid')
DEFINE_string('rabbit_password', 'guest', 'rabbit password')
DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host')
DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to')
DEFINE_string('cc_host', '127.0.0.1', 'ip of api server')
DEFINE_integer('cc_port', 8773, 'cloud controller port')
DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud',
              'Url to ec2 api server')

DEFINE_string('default_image', 'ami-11111',
              'default image to use, testing only')
DEFINE_string('default_kernel', 'aki-11111',
              'default kernel to use, testing only')
DEFINE_string('default_ramdisk', 'ari-11111',
              'default ramdisk to use, testing only')
DEFINE_string('default_instance_type', 'm1.small',
              'default instance type to use, testing only')

DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server')
DEFINE_string('vpn_key_suffix',
              '-key',
              'Suffix to add to project name for vpn key')

DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger')

DEFINE_string('sql_connection',
              'sqlite:///%s/nova.sqlite' % os.path.abspath("./"),
              'connection string for sql database')

DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager',
              'Manager for compute')
DEFINE_string('network_manager', 'nova.network.manager.VlanManager',
              'Manager for network')
DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager',
              'Manager for volume')
DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager',
              'Manager for scheduler')

DEFINE_string('host', socket.gethostname(),
              'name of this node')

# UNUSED
DEFINE_string('node_availability_zone', 'nova',
              'availability zone of this node')
<commit_msg>Adds BaseImageService and flag to control image service loading.  Adds unit test for local image service.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Package-level global flags are defined here, the rest are defined
where they're used.
"""

import getopt
import os
import socket
import sys

import gflags

class FlagValues(gflags.FlagValues):
    """Extension of gflags.FlagValues that allows undefined and runtime flags.

    Unknown flags will be ignored when parsing the command line, but the
    command line will be kept so that it can be replayed if new flags are
    defined after the initial parsing.

    """

    def __init__(self):
        gflags.FlagValues.__init__(self)
        self.__dict__['__dirty'] = []
        self.__dict__['__was_already_parsed'] = False
        self.__dict__['__stored_argv'] = []

    def __call__(self, argv):
        # We're doing some hacky stuff here so that we don't have to copy
        # out all the code of the original verbatim and then tweak a few lines.
        # We're hijacking the output of getopt so we can still return the
        # leftover args at the end
        sneaky_unparsed_args = {"value": None}
        original_argv = list(argv)

        if self.IsGnuGetOpt():
            orig_getopt = getattr(getopt, 'gnu_getopt')
            orig_name = 'gnu_getopt'
        else:
            orig_getopt = getattr(getopt, 'getopt')
            orig_name = 'getopt'

        def _sneaky(*args, **kw):
            optlist, unparsed_args = orig_getopt(*args, **kw)
            sneaky_unparsed_args['value'] = unparsed_args
            return optlist, unparsed_args

        try:
            setattr(getopt, orig_name, _sneaky)
            args = gflags.FlagValues.__call__(self, argv)
        except gflags.UnrecognizedFlagError:
            # Undefined args were found, for now we don't care so just
            # act like everything went well
            # (these three lines are copied pretty much verbatim from the end
            # of the __call__ function we are wrapping)
            unparsed_args = sneaky_unparsed_args['value']
            if unparsed_args:
                if self.IsGnuGetOpt():
                    args = argv[:1] + unparsed_args
                else:
                    args = argv[:1] + original_argv[-len(unparsed_args):]
            else:
                args = argv[:1]
        finally:
            setattr(getopt, orig_name, orig_getopt)

        # Store the arguments for later, we'll need them for new flags
        # added at runtime
        self.__dict__['__stored_argv'] = original_argv
        self.__dict__['__was_already_parsed'] = True
        self.ClearDirty()
        return args

    def SetDirty(self, name):
        """Mark a flag as dirty so that accessing it will case a reparse."""
        self.__dict__['__dirty'].append(name)

    def IsDirty(self, name):
        return name in self.__dict__['__dirty']

    def ClearDirty(self):
        self.__dict__['__is_dirty'] = []

    def WasAlreadyParsed(self):
        return self.__dict__['__was_already_parsed']

    def ParseNewFlags(self):
        if '__stored_argv' not in self.__dict__:
            return
        new_flags = FlagValues()
        for k in self.__dict__['__dirty']:
            new_flags[k] = gflags.FlagValues.__getitem__(self, k)

        new_flags(self.__dict__['__stored_argv'])
        for k in self.__dict__['__dirty']:
            setattr(self, k, getattr(new_flags, k))
        self.ClearDirty()

    def __setitem__(self, name, flag):
        gflags.FlagValues.__setitem__(self, name, flag)
        if self.WasAlreadyParsed():
            self.SetDirty(name)

    def __getitem__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getitem__(self, name)

    def __getattr__(self, name):
        if self.IsDirty(name):
            self.ParseNewFlags()
        return gflags.FlagValues.__getattr__(self, name)

FLAGS = FlagValues()

def _wrapper(func):
    def _wrapped(*args, **kw):
        kw.setdefault('flag_values', FLAGS)
        func(*args, **kw)
    _wrapped.func_name = func.func_name
    return _wrapped

DEFINE = _wrapper(gflags.DEFINE)
DEFINE_string = _wrapper(gflags.DEFINE_string)
DEFINE_integer = _wrapper(gflags.DEFINE_integer)
DEFINE_bool = _wrapper(gflags.DEFINE_bool)
DEFINE_boolean = _wrapper(gflags.DEFINE_boolean)
DEFINE_float = _wrapper(gflags.DEFINE_float)
DEFINE_enum = _wrapper(gflags.DEFINE_enum)
DEFINE_list = _wrapper(gflags.DEFINE_list)
DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist)
DEFINE_multistring = _wrapper(gflags.DEFINE_multistring)
DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int)

def DECLARE(name, module_string, flag_values=FLAGS):
    if module_string not in sys.modules:
        __import__(module_string, globals(), locals())
    if name not in flag_values:
        raise gflags.UnrecognizedFlag(
                "%s not defined by %s" % (name, module_string))

# __GLOBAL FLAGS ONLY__
# Define any app-specific flags in their own files, docs at:
# http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39

DEFINE_list('region_list',
            [],
            'list of region=url pairs separated by commas')
DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake')
DEFINE_integer('s3_port', 3333, 's3 port')
DEFINE_string('s3_host', '127.0.0.1', 's3 host')
DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on')
DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on')
DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on')
DEFINE_string('network_topic', 'network', 'the topic network nodes listen on')

DEFINE_bool('verbose', False, 'show debug output')
DEFINE_boolean('fake_rabbit', False, 'use a fake rabbit')
DEFINE_bool('fake_network', False,
            'should we use fake network devices and addresses')
DEFINE_string('rabbit_host', 'localhost', 'rabbit host')
DEFINE_integer('rabbit_port', 5672, 'rabbit port')
DEFINE_string('rabbit_userid', 'guest', 'rabbit userid')
DEFINE_string('rabbit_password', 'guest', 'rabbit password')
DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host')
DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to')
DEFINE_string('cc_host', '127.0.0.1', 'ip of api server')
DEFINE_integer('cc_port', 8773, 'cloud controller port')
DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud',
              'Url to ec2 api server')

DEFINE_string('default_image', 'ami-11111',
              'default image to use, testing only')
DEFINE_string('default_kernel', 'aki-11111',
              'default kernel to use, testing only')
DEFINE_string('default_ramdisk', 'ari-11111',
              'default ramdisk to use, testing only')
DEFINE_string('default_instance_type', 'm1.small',
              'default instance type to use, testing only')

DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server')
DEFINE_string('vpn_key_suffix',
              '-key',
              'Suffix to add to project name for vpn key')

DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger')

DEFINE_string('sql_connection',
              'sqlite:///%s/nova.sqlite' % os.path.abspath("./"),
              'connection string for sql database')

DEFINE_string('compute_manager', 'nova.compute.manager.ComputeManager',
              'Manager for compute')
DEFINE_string('network_manager', 'nova.network.manager.VlanManager',
              'Manager for network')
DEFINE_string('volume_manager', 'nova.volume.manager.AOEManager',
              'Manager for volume')
DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager',
              'Manager for scheduler')

# The service to use for image search and retrieval
DEFINE_string('image_service', 'nova.image.service.LocalImageService',
              'The service to use for retrieving and searching for images.')

DEFINE_string('host', socket.gethostname(),
              'name of this node')

# UNUSED
DEFINE_string('node_availability_zone', 'nova',
              'availability zone of this node')
<|endoftext|>


<commit_before>import os
from shutil import rmtree
import gzip
from tempfile import mkdtemp
from glob import iglob, glob

import astropy.io.fits as fits
import numpy as np
import pytest

from .. import image_collection as tff
from ..header_processing.patchers import IRAF_image_type

_n_test = {'files': 0, 'need_object': 0,
           'need_filter': 0, 'bias': 0,
           'compressed': 0, 'light': 0,
           'need_pointing': 0}

_test_dir = ''
_filters = []
_original_dir = ''

def test_fits_summary():
    keywords = ['imagetyp', 'filter']
    image_collection = tff.ImageFileCollection(_test_dir,
                                               keywords=keywords)
    summary = image_collection._fits_summary(header_keywords=keywords)
    print summary['file']
    print summary.keys()
    assert len(summary['file']) == _n_test['files']
    for keyword in keywords:
        assert len(summary[keyword]) == _n_test['files']
    # explicit conversion to array is needed to avoid astropy Table bug in
    # 0.2.4
    print np.array(summary['file'] == 'no_filter_no_object_bias.fit')
    no_filter_no_object_row = np.array(summary['file'] ==
                                       'no_filter_no_object_bias.fit')
    # there should be no filter keyword in the bias file
    assert (summary['filter'][no_filter_no_object_row].mask)

class TestImageFileCollection(object):

    def test_storage_dir_set(self):
        try:
            should_work = tff.ImageFileCollection(location=_test_dir,
                                                  storage_dir=_test_dir)
            assert True
        except OSError:
            assert False

        with pytest.raises(OSError):
            tff.ImageFileCollection(location=_test_dir, storage_dir='/')

        img_collection = tff.ImageFileCollection(
            location=_test_dir, keywords=['imagetyp', 'filter'])
        print img_collection.files_filtered(imagetyp='bias')
        print _n_test
        assert len(img_collection.files_filtered(
            imagetyp='bias')) == _n_test['bias']
        assert len(img_collection.files) == _n_test['files']
        assert ('filter' in img_collection.keywords)
        assert ('flying monkeys' not in img_collection.keywords)
        assert len(img_collection.values('imagetyp', unique=True)) == 2

    def test_files_with_compressed(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        assert len(collection._fits_files_in_directory(
            compressed=True)) == _n_test['files']

    def test_files_with_no_compressed(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        n_files_found = len(
            collection._fits_files_in_directory(compressed=False))
        n_uncompressed = _n_test['files'] - _n_test['compressed']
        assert n_files_found == n_uncompressed

    def test_generator_full_path(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for path, file_name in zip(collection._paths(), collection.files):
            assert path == os.path.join(_test_dir, file_name)

    def test_hdus(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        n_hdus = 0
        for hdu in collection.hdus():
            assert isinstance(hdu, fits.PrimaryHDU)
            data = hdu.data  # must access the data to force scaling
            with pytest.raises(KeyError):
                hdu.header['bzero']
            n_hdus += 1
        assert n_hdus == _n_test['files']

    def test_hdus_masking(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp', 'exposure'])
        old_data = np.array(collection.summary_info)
        for hdu in collection.hdus(imagetyp='bias'):
            pass
        new_data = np.array(collection.summary_info)
        assert (new_data == old_data).all()

    def test_headers(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        n_headers = 0
        for header in collection.headers():
            assert isinstance(header, fits.Header)
            assert ('bzero' in header)
            n_headers += 1
        assert n_headers == _n_test['files']

    def test_headers_save_location(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        destination = mkdtemp()
        for header in collection.headers(save_location=destination):
            pass
        new_collection = \
            tff.ImageFileCollection(location=destination,
                                    keywords=['imagetyp'])
        basenames = lambda paths: set(
            [os.path.basename(file) for file in paths])

        assert (len(basenames(collection._paths()) -
                    basenames(new_collection._paths())) == 0)
                #_n_test['compressed'])
        rmtree(destination)

    def test_headers_with_filter(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='light'):
            assert header['imagetyp'].lower() == 'light'
            cnt += 1
        assert cnt == _n_test['light']

    def test_headers_with_multiple_filters(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='light',
                                         filter='R'):
            assert header['imagetyp'].lower() == 'light'
            assert header['filter'].lower() == 'r'
            cnt += 1
        assert cnt == _n_test['light'] - _n_test['need_filter']

    def test_headers_with_filter_wildcard(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='*'):
            cnt += 1
        assert cnt == _n_test['files']

    def test_headers_with_filter_missing_keyword(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for header in collection.headers(imagetyp='light',
                                         object=''):
            assert header['imagetyp'].lower() == 'light'
            with pytest.raises(KeyError):
                header['object']

    def test_generator_headers_save_with_name(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for header in collection.headers(save_with_name='_new'):
            assert isinstance(header, fits.Header)
        new_collection = tff.ImageFileCollection(location=_test_dir,
                                                 keywords=['imagetyp'])
        assert (len(new_collection._paths()) ==
                2 * (_n_test['files']) - _n_test['compressed'])
        print glob(_test_dir + '/*_new*')
        [os.remove(fil) for fil in iglob(_test_dir + '/*_new*')]
        print glob(_test_dir + '/*_new*')

    def test_generator_data(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for img in collection.data():
            assert isinstance(img, np.ndarray)

    def test_consecutive_fiilters(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp',
                                                       'filter',
                                                       'object'])
        no_files_match = collection.files_filtered(object='fdsafs')
        assert(len(no_files_match) == 0)
        some_files_should_match = collection.files_filtered(object=None,
                                                            imagetyp='light')
        print some_files_should_match
        assert(len(some_files_should_match) == _n_test['need_object'])

    def test_filter_does_not_not_permanently_change_file_mask(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        # ensure all files are originally unmasked
        assert(not collection.summary_info['file'].mask.any())
        # generate list that will match NO files
        collection.files_filtered(imagetyp='foisajfoisaj')
        # if the code works, this should have no permanent effect
        assert(not collection.summary_info['file'].mask.any())

    @pytest.mark.parametrize("new_keywords,collection_keys", [
                            (['imagetyp', 'object'], ['imagetyp', 'filter']),
                            (['imagetyp'], ['imagetyp', 'filter'])])
    def test_keyword_setting(self, new_keywords, collection_keys):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=collection_keys)
        tbl_orig = collection.summary_info
        collection.keywords = new_keywords
        tbl_new = collection.summary_info

        if set(new_keywords).issubset(collection_keys):
            # should just delete columns without rebuilding table
            assert(tbl_orig is tbl_new)
        else:
            # we need new keywords so must rebuild
            assert(tbl_orig is not tbl_new)

        for key in new_keywords:
            assert(key in tbl_new.keys())
        assert (tbl_orig['file'] == tbl_new['file']).all()
        assert (tbl_orig['imagetyp'] == tbl_new['imagetyp']).all()
        assert 'filter' not in tbl_new.keys()
        assert 'object' not in tbl_orig.keys()

    def test_header_and_filename(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        for header, fname in collection.headers(return_fname=True):
            assert (fname in collection._paths())
            assert (isinstance(header, fits.Header))

    def test_dir_with_no_fits_files(self, tmpdir):
        empty_dir = tmpdir.mkdtemp()
        some_file = empty_dir.join('some_file.txt')
        some_file.dump('words')
        print empty_dir.listdir()
        collection = tff.ImageFileCollection(location=empty_dir.strpath)
        assert (collection.summary_info is None)
        for hdr in collection.headers():
            # this statement should not be reached if there are no FITS files
            assert 0

    def test_fits_summary_when_keywords_are_not_subset(self):
        """
        Catch case when there is overlap between keyword list
        passed to the ImageFileCollection and to files_filtered
        but the latter is not a subset of the former.
        """
        ic = tff.ImageFileCollection(_test_dir,
                                     keywords=['imagetyp', 'exptime'])
        n_files = len(ic.files)
        files_missing_this_key = ic.files_filtered(imagetyp='*',
                                                   monkeys=None)
        assert(n_files > 0)
        assert(n_files == len(files_missing_this_key))

    def test_duplicate_keywords_in_setting(self):
        keywords_in = ['imagetyp', 'a', 'a']
        ic = tff.ImageFileCollection(_test_dir,
                                     keywords=keywords_in)
        for key in set(keywords_in):
            assert (key in ic.keywords)

def setup_module():
    global _n_test
    global _test_dir
    global _original_dir

    for key in _n_test.keys():
        _n_test[key] = 0

    _test_dir = mkdtemp()
    _original_dir = os.getcwd()

    os.chdir(_test_dir)
    img = np.uint16(np.arange(100))

    no_filter_no_object = fits.PrimaryHDU(img)
    no_filter_no_object.header['imagetyp'] = IRAF_image_type('light')
    no_filter_no_object.writeto('no_filter_no_object_light.fit')
    _n_test['files'] += 1
    _n_test['need_object'] += 1
    _n_test['need_filter'] += 1
    _n_test['light'] += 1
    _n_test['need_pointing'] += 1

    no_filter_no_object.header['imagetyp'] = IRAF_image_type('bias')
    no_filter_no_object.writeto('no_filter_no_object_bias.fit')
    _n_test['files'] += 1
    _n_test['bias'] += 1

    filter_no_object = fits.PrimaryHDU(img)
    filter_no_object.header['imagetyp'] = IRAF_image_type('light')
    filter_no_object.header['filter'] = 'R'
    filter_no_object.writeto('filter_no_object_light.fit')
    _n_test['files'] += 1
    _n_test['need_object'] += 1
    _n_test['light'] += 1
    _n_test['need_pointing'] += 1

    filter_no_object.header['imagetyp'] = IRAF_image_type('bias')
    filter_no_object.writeto('filter_no_object_bias.fit')
    _n_test['files'] += 1
    _n_test['bias'] += 1

    filter_object = fits.PrimaryHDU(img)
    filter_object.header['imagetyp'] = IRAF_image_type('light')
    filter_object.header['filter'] = 'R'
    filter_object.header['OBJCTRA'] = '00:00:00'
    filter_object.header['OBJCTDEC'] = '00:00:00'
    filter_object.writeto('filter_object_light.fit')
    _n_test['files'] += 1
    _n_test['light'] += 1
    _n_test['need_object'] += 1
    filter_file = open('filter_object_light.fit', 'rb')
    fzipped = gzip.open('filter_object_light.fit.gz', 'wb')
    fzipped.writelines(filter_file)
    fzipped.close()
    _n_test['files'] += 1
    _n_test['compressed'] += 1
    _n_test['light'] += 1
    _n_test['need_object'] += 1

def teardown_module():
    global _n_test

    for key in _n_test.keys():
        _n_test[key] = 0
    rmtree(_test_dir)
    os.chdir(_original_dir)
<commit_msg>Add test for #126 — FITS files being read unnecessarily<commit_after>import os
from shutil import rmtree
import gzip
from tempfile import mkdtemp
from glob import iglob, glob
import logging

import astropy.io.fits as fits
import numpy as np
import pytest

from .. import image_collection as tff
from ..header_processing.patchers import IRAF_image_type

_n_test = {'files': 0, 'need_object': 0,
           'need_filter': 0, 'bias': 0,
           'compressed': 0, 'light': 0,
           'need_pointing': 0}

_test_dir = ''
_filters = []
_original_dir = ''

def test_fits_summary():
    keywords = ['imagetyp', 'filter']
    image_collection = tff.ImageFileCollection(_test_dir,
                                               keywords=keywords)
    summary = image_collection._fits_summary(header_keywords=keywords)
    print summary['file']
    print summary.keys()
    assert len(summary['file']) == _n_test['files']
    for keyword in keywords:
        assert len(summary[keyword]) == _n_test['files']
    # explicit conversion to array is needed to avoid astropy Table bug in
    # 0.2.4
    print np.array(summary['file'] == 'no_filter_no_object_bias.fit')
    no_filter_no_object_row = np.array(summary['file'] ==
                                       'no_filter_no_object_bias.fit')
    # there should be no filter keyword in the bias file
    assert (summary['filter'][no_filter_no_object_row].mask)

class TestImageFileCollection(object):

    def test_storage_dir_set(self):
        try:
            should_work = tff.ImageFileCollection(location=_test_dir,
                                                  storage_dir=_test_dir)
            assert True
        except OSError:
            assert False

        with pytest.raises(OSError):
            tff.ImageFileCollection(location=_test_dir, storage_dir='/')

        img_collection = tff.ImageFileCollection(
            location=_test_dir, keywords=['imagetyp', 'filter'])
        print img_collection.files_filtered(imagetyp='bias')
        print _n_test
        assert len(img_collection.files_filtered(
            imagetyp='bias')) == _n_test['bias']
        assert len(img_collection.files) == _n_test['files']
        assert ('filter' in img_collection.keywords)
        assert ('flying monkeys' not in img_collection.keywords)
        assert len(img_collection.values('imagetyp', unique=True)) == 2

    def test_files_with_compressed(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        assert len(collection._fits_files_in_directory(
            compressed=True)) == _n_test['files']

    def test_files_with_no_compressed(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        n_files_found = len(
            collection._fits_files_in_directory(compressed=False))
        n_uncompressed = _n_test['files'] - _n_test['compressed']
        assert n_files_found == n_uncompressed

    def test_generator_full_path(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for path, file_name in zip(collection._paths(), collection.files):
            assert path == os.path.join(_test_dir, file_name)

    def test_hdus(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        n_hdus = 0
        for hdu in collection.hdus():
            assert isinstance(hdu, fits.PrimaryHDU)
            data = hdu.data  # must access the data to force scaling
            with pytest.raises(KeyError):
                hdu.header['bzero']
            n_hdus += 1
        assert n_hdus == _n_test['files']

    def test_hdus_masking(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp', 'exposure'])
        old_data = np.array(collection.summary_info)
        for hdu in collection.hdus(imagetyp='bias'):
            pass
        new_data = np.array(collection.summary_info)
        assert (new_data == old_data).all()

    def test_headers(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        n_headers = 0
        for header in collection.headers():
            assert isinstance(header, fits.Header)
            assert ('bzero' in header)
            n_headers += 1
        assert n_headers == _n_test['files']

    def test_headers_save_location(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        destination = mkdtemp()
        for header in collection.headers(save_location=destination):
            pass
        new_collection = \
            tff.ImageFileCollection(location=destination,
                                    keywords=['imagetyp'])
        basenames = lambda paths: set(
            [os.path.basename(file) for file in paths])

        assert (len(basenames(collection._paths()) -
                    basenames(new_collection._paths())) == 0)
                #_n_test['compressed'])
        rmtree(destination)

    def test_headers_with_filter(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='light'):
            assert header['imagetyp'].lower() == 'light'
            cnt += 1
        assert cnt == _n_test['light']

    def test_headers_with_multiple_filters(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='light',
                                         filter='R'):
            assert header['imagetyp'].lower() == 'light'
            assert header['filter'].lower() == 'r'
            cnt += 1
        assert cnt == _n_test['light'] - _n_test['need_filter']

    def test_headers_with_filter_wildcard(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        cnt = 0
        for header in collection.headers(imagetyp='*'):
            cnt += 1
        assert cnt == _n_test['files']

    def test_headers_with_filter_missing_keyword(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for header in collection.headers(imagetyp='light',
                                         object=''):
            assert header['imagetyp'].lower() == 'light'
            with pytest.raises(KeyError):
                header['object']

    def test_generator_headers_save_with_name(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for header in collection.headers(save_with_name='_new'):
            assert isinstance(header, fits.Header)
        new_collection = tff.ImageFileCollection(location=_test_dir,
                                                 keywords=['imagetyp'])
        assert (len(new_collection._paths()) ==
                2 * (_n_test['files']) - _n_test['compressed'])
        print glob(_test_dir + '/*_new*')
        [os.remove(fil) for fil in iglob(_test_dir + '/*_new*')]
        print glob(_test_dir + '/*_new*')

    def test_generator_data(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        for img in collection.data():
            assert isinstance(img, np.ndarray)

    def test_consecutive_fiilters(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp',
                                                       'filter',
                                                       'object'])
        no_files_match = collection.files_filtered(object='fdsafs')
        assert(len(no_files_match) == 0)
        some_files_should_match = collection.files_filtered(object=None,
                                                            imagetyp='light')
        print some_files_should_match
        assert(len(some_files_should_match) == _n_test['need_object'])

    def test_filter_does_not_not_permanently_change_file_mask(self):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=['imagetyp'])
        # ensure all files are originally unmasked
        assert(not collection.summary_info['file'].mask.any())
        # generate list that will match NO files
        collection.files_filtered(imagetyp='foisajfoisaj')
        # if the code works, this should have no permanent effect
        assert(not collection.summary_info['file'].mask.any())

    @pytest.mark.parametrize("new_keywords,collection_keys", [
                            (['imagetyp', 'object'], ['imagetyp', 'filter']),
                            (['imagetyp'], ['imagetyp', 'filter'])])
    def test_keyword_setting(self, new_keywords, collection_keys):
        collection = tff.ImageFileCollection(location=_test_dir,
                                             keywords=collection_keys)
        tbl_orig = collection.summary_info
        collection.keywords = new_keywords
        tbl_new = collection.summary_info

        if set(new_keywords).issubset(collection_keys):
            # should just delete columns without rebuilding table
            assert(tbl_orig is tbl_new)
        else:
            # we need new keywords so must rebuild
            assert(tbl_orig is not tbl_new)

        for key in new_keywords:
            assert(key in tbl_new.keys())
        assert (tbl_orig['file'] == tbl_new['file']).all()
        assert (tbl_orig['imagetyp'] == tbl_new['imagetyp']).all()
        assert 'filter' not in tbl_new.keys()
        assert 'object' not in tbl_orig.keys()

    def test_header_and_filename(self):
        collection = tff.ImageFileCollection(location=_test_dir)
        for header, fname in collection.headers(return_fname=True):
            assert (fname in collection._paths())
            assert (isinstance(header, fits.Header))

    def test_dir_with_no_fits_files(self, tmpdir):
        empty_dir = tmpdir.mkdtemp()
        some_file = empty_dir.join('some_file.txt')
        some_file.dump('words')
        print empty_dir.listdir()
        collection = tff.ImageFileCollection(location=empty_dir.strpath)
        assert (collection.summary_info is None)
        for hdr in collection.headers():
            # this statement should not be reached if there are no FITS files
            assert 0

    def test_dir_with_bad_fits_and_no_keys(self, tmpdir, caplog):
        # This test should fail if the FITS files in the directory
        # are actually read.
        bad_dir = tmpdir.mkdtemp()
        not_really_fits = bad_dir.join('not_fits.fit')
        not_really_fits.dump('I am not really a FITS file')
        # make sure an error will be generated if the FITS file is read
        with pytest.raises(IOError):
            fits.getheader(not_really_fits.strpath)
        ic = tff.ImageFileCollection(location=bad_dir.strpath)
        # ImageFileCollection will suppress the IOError but log a warning
        # so check the log for the appropriate warning
        warnings = [rec for rec in caplog.records()
                    if ((rec.levelno == logging.WARN) &
                        ('Unable to get FITS header' in rec.message))]
        assert (len(warnings) == 0)

    def test_fits_summary_when_keywords_are_not_subset(self):
        """
        Catch case when there is overlap between keyword list
        passed to the ImageFileCollection and to files_filtered
        but the latter is not a subset of the former.
        """
        ic = tff.ImageFileCollection(_test_dir,
                                     keywords=['imagetyp', 'exptime'])
        n_files = len(ic.files)
        files_missing_this_key = ic.files_filtered(imagetyp='*',
                                                   monkeys=None)
        assert(n_files > 0)
        assert(n_files == len(files_missing_this_key))

    def test_duplicate_keywords_in_setting(self):
        keywords_in = ['imagetyp', 'a', 'a']
        ic = tff.ImageFileCollection(_test_dir,
                                     keywords=keywords_in)
        for key in set(keywords_in):
            assert (key in ic.keywords)

def setup_module():
    global _n_test
    global _test_dir
    global _original_dir

    for key in _n_test.keys():
        _n_test[key] = 0

    _test_dir = mkdtemp()
    _original_dir = os.getcwd()

    os.chdir(_test_dir)
    img = np.uint16(np.arange(100))

    no_filter_no_object = fits.PrimaryHDU(img)
    no_filter_no_object.header['imagetyp'] = IRAF_image_type('light')
    no_filter_no_object.writeto('no_filter_no_object_light.fit')
    _n_test['files'] += 1
    _n_test['need_object'] += 1
    _n_test['need_filter'] += 1
    _n_test['light'] += 1
    _n_test['need_pointing'] += 1

    no_filter_no_object.header['imagetyp'] = IRAF_image_type('bias')
    no_filter_no_object.writeto('no_filter_no_object_bias.fit')
    _n_test['files'] += 1
    _n_test['bias'] += 1

    filter_no_object = fits.PrimaryHDU(img)
    filter_no_object.header['imagetyp'] = IRAF_image_type('light')
    filter_no_object.header['filter'] = 'R'
    filter_no_object.writeto('filter_no_object_light.fit')
    _n_test['files'] += 1
    _n_test['need_object'] += 1
    _n_test['light'] += 1
    _n_test['need_pointing'] += 1

    filter_no_object.header['imagetyp'] = IRAF_image_type('bias')
    filter_no_object.writeto('filter_no_object_bias.fit')
    _n_test['files'] += 1
    _n_test['bias'] += 1

    filter_object = fits.PrimaryHDU(img)
    filter_object.header['imagetyp'] = IRAF_image_type('light')
    filter_object.header['filter'] = 'R'
    filter_object.header['OBJCTRA'] = '00:00:00'
    filter_object.header['OBJCTDEC'] = '00:00:00'
    filter_object.writeto('filter_object_light.fit')
    _n_test['files'] += 1
    _n_test['light'] += 1
    _n_test['need_object'] += 1
    filter_file = open('filter_object_light.fit', 'rb')
    fzipped = gzip.open('filter_object_light.fit.gz', 'wb')
    fzipped.writelines(filter_file)
    fzipped.close()
    _n_test['files'] += 1
    _n_test['compressed'] += 1
    _n_test['light'] += 1
    _n_test['need_object'] += 1

def teardown_module():
    global _n_test

    for key in _n_test.keys():
        _n_test[key] = 0
    rmtree(_test_dir)
    os.chdir(_original_dir)
<|endoftext|>


<commit_before>from copy import copy

import sqlalchemy as sa
from sqlalchemy_utils.functions import get_declarative_base

from .dialects.postgresql import create_versioning_trigger_listeners
from .model_builder import ModelBuilder
from .relationship_builder import RelationshipBuilder
from .table_builder import TableBuilder

class Builder(object):
    def build_triggers(self):
        """
        Build native database versioning triggers for all versioned models that
        were collected during class instrumentation process.
        """
        processed_tables = set()
        for cls in self.manager.pending_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            if self.manager.option(cls, 'native_versioning'):
                cls.__versioning_manager__ = self.manager

                if cls.__table__ not in processed_tables:
                    create_versioning_trigger_listeners(self.manager, cls)
                    processed_tables.add(cls.__table__)

    def build_tables(self):
        """
        Build tables for version models based on classes that were collected
        during class instrumentation process.
        """
        for cls in self.manager.pending_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            if self.manager.option(cls, 'create_tables'):
                inherited_table = None
                for class_ in self.manager.tables:
                    if (issubclass(cls, class_) and
                            cls.__table__ == class_.__table__):
                        inherited_table = self.manager.tables[class_]
                        break

                builder = TableBuilder(
                    self.manager,
                    cls.__table__,
                    model=cls
                )
                if inherited_table is not None:
                    self.manager.tables[class_] = builder(inherited_table)
                else:
                    table = builder()
                    self.manager.tables[cls] = table

    def closest_matching_table(self, model):
        """
        Returns the closest matching table from the generated tables dictionary
        for given model. First tries to fetch an exact match for given model.
        If no table was found then tries to match given model as a subclass.

        :param model: SQLAlchemy declarative model class.
        """
        if model in self.manager.tables:
            return self.manager.tables[model]
        for cls in self.manager.tables:
            if issubclass(model, cls):
                return self.manager.tables[cls]

    def build_models(self):
        """
        Build declarative version models based on classes that were collected
        during class instrumentation process.
        """
        if self.manager.pending_classes:
            for cls in self.manager.pending_classes:
                if not self.manager.option(cls, 'versioning'):
                    continue

                table = self.closest_matching_table(cls)
                if table is not None:
                    builder = ModelBuilder(self.manager, cls)
                    version_cls = builder(
                        table,
                        self.manager.transaction_cls
                    )

                    self.manager.plugins.after_version_class_built(
                        cls,
                        version_cls
                    )

            self.manager.plugins.after_build_models(self.manager)

    def build_relationships(self, version_classes):
        """
        Builds relationships for all version classes.

        :param version_classes: list of generated version classes
        """
        for cls in version_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            for prop in sa.inspect(cls).iterate_properties:
                if prop.key == 'versions':
                    continue
                builder = RelationshipBuilder(self.manager, cls, prop)
                builder()

    def instrument_versioned_classes(self, mapper, cls):
        """
        Collect versioned class and add it to pending_classes list.

        :mapper mapper: SQLAlchemy mapper object
        :cls cls: SQLAlchemy declarative class
        """
        if not self.manager.options['versioning']:
            return

        if hasattr(cls, '__versioned__'):
            if (not cls.__versioned__.get('class')
                    and cls not in self.manager.pending_classes):
                self.manager.pending_classes.append(cls)
                self.manager.metadata = cls.metadata

        if hasattr(cls, '__version_parent__'):
            parent = cls.__version_parent__
            self.manager.version_class_map[parent] = cls
            self.manager.parent_class_map[cls] = parent
            del cls.__version_parent__

    def build_transaction_class(self):
        if self.manager.pending_classes:
            cls = self.manager.pending_classes[0]
            self.manager.declarative_base = get_declarative_base(cls)
            self.manager.create_transaction_model()
            self.manager.plugins.after_build_tx_class(self.manager)

    def configure_versioned_classes(self):
        """
        Configures all versioned classes that were collected during
        instrumentation process. The configuration has 4 steps:

        1. Build tables for version models.
        2. Build the actual version model declarative classes.
        3. Build relationships between these models.
        4. Empty pending_classes list so that consecutive mapper configuration
           does not create multiple version classes
        5. Assign all versioned attributes to use active history.
        """
        if not self.manager.options['versioning']:
            return

        self.build_triggers()
        self.build_tables()
        self.build_transaction_class()

        if not self.manager.options['create_models']:
            self.manager.pending_classes = []
            return

        self.build_models()

        # Create copy of all pending versioned classes so that we can inspect
        # them later when creating relationships.
        pending_copy = copy(self.manager.pending_classes)
        self.manager.pending_classes = []
        self.build_relationships(pending_copy)

        for cls in pending_copy:
            # set the "active_history" flag
            for prop in sa.inspect(cls).iterate_properties:
                getattr(cls, prop.key).impl.active_history = True
<commit_msg>Fix possible random subclass retrieval on closest_matching_table.<commit_after>from copy import copy
from inspect import getmro

import sqlalchemy as sa
from sqlalchemy_utils.functions import get_declarative_base

from .dialects.postgresql import create_versioning_trigger_listeners
from .model_builder import ModelBuilder
from .relationship_builder import RelationshipBuilder
from .table_builder import TableBuilder

class Builder(object):
    def build_triggers(self):
        """
        Build native database versioning triggers for all versioned models that
        were collected during class instrumentation process.
        """
        processed_tables = set()
        for cls in self.manager.pending_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            if self.manager.option(cls, 'native_versioning'):
                cls.__versioning_manager__ = self.manager

                if cls.__table__ not in processed_tables:
                    create_versioning_trigger_listeners(self.manager, cls)
                    processed_tables.add(cls.__table__)

    def build_tables(self):
        """
        Build tables for version models based on classes that were collected
        during class instrumentation process.
        """
        for cls in self.manager.pending_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            if self.manager.option(cls, 'create_tables'):
                inherited_table = None
                for class_ in self.manager.tables:
                    if (issubclass(cls, class_) and
                            cls.__table__ == class_.__table__):
                        inherited_table = self.manager.tables[class_]
                        break

                builder = TableBuilder(
                    self.manager,
                    cls.__table__,
                    model=cls
                )
                if inherited_table is not None:
                    self.manager.tables[class_] = builder(inherited_table)
                else:
                    table = builder()
                    self.manager.tables[cls] = table

    def closest_matching_table(self, model):
        """
        Returns the closest matching table from the generated tables dictionary
        for given model. First tries to fetch an exact match for given model.
        If no table was found then tries to match given model as a subclass.

        :param model: SQLAlchemy declarative model class.
        """
        if model in self.manager.tables:
            return self.manager.tables[model]
        subclasses = [cls for cls in self.manager.tables if issubclass(model, cls)]
        ordered_subclasses = [cls for cls in getmro(model) if cls in subclasses]
        return self.manager.tables[ordered_subclasses[0]] if ordered_subclasses else None

    def build_models(self):
        """
        Build declarative version models based on classes that were collected
        during class instrumentation process.
        """
        if self.manager.pending_classes:
            for cls in self.manager.pending_classes:
                if not self.manager.option(cls, 'versioning'):
                    continue

                table = self.closest_matching_table(cls)
                if table is not None:
                    builder = ModelBuilder(self.manager, cls)
                    version_cls = builder(
                        table,
                        self.manager.transaction_cls
                    )

                    self.manager.plugins.after_version_class_built(
                        cls,
                        version_cls
                    )

            self.manager.plugins.after_build_models(self.manager)

    def build_relationships(self, version_classes):
        """
        Builds relationships for all version classes.

        :param version_classes: list of generated version classes
        """
        for cls in version_classes:
            if not self.manager.option(cls, 'versioning'):
                continue

            for prop in sa.inspect(cls).iterate_properties:
                if prop.key == 'versions':
                    continue
                builder = RelationshipBuilder(self.manager, cls, prop)
                builder()

    def instrument_versioned_classes(self, mapper, cls):
        """
        Collect versioned class and add it to pending_classes list.

        :mapper mapper: SQLAlchemy mapper object
        :cls cls: SQLAlchemy declarative class
        """
        if not self.manager.options['versioning']:
            return

        if hasattr(cls, '__versioned__'):
            if (not cls.__versioned__.get('class')
                    and cls not in self.manager.pending_classes):
                self.manager.pending_classes.append(cls)
                self.manager.metadata = cls.metadata

        if hasattr(cls, '__version_parent__'):
            parent = cls.__version_parent__
            self.manager.version_class_map[parent] = cls
            self.manager.parent_class_map[cls] = parent
            del cls.__version_parent__

    def build_transaction_class(self):
        if self.manager.pending_classes:
            cls = self.manager.pending_classes[0]
            self.manager.declarative_base = get_declarative_base(cls)
            self.manager.create_transaction_model()
            self.manager.plugins.after_build_tx_class(self.manager)

    def configure_versioned_classes(self):
        """
        Configures all versioned classes that were collected during
        instrumentation process. The configuration has 4 steps:

        1. Build tables for version models.
        2. Build the actual version model declarative classes.
        3. Build relationships between these models.
        4. Empty pending_classes list so that consecutive mapper configuration
           does not create multiple version classes
        5. Assign all versioned attributes to use active history.
        """
        if not self.manager.options['versioning']:
            return

        self.build_triggers()
        self.build_tables()
        self.build_transaction_class()

        if not self.manager.options['create_models']:
            self.manager.pending_classes = []
            return

        self.build_models()

        # Create copy of all pending versioned classes so that we can inspect
        # them later when creating relationships.
        pending_copy = copy(self.manager.pending_classes)
        self.manager.pending_classes = []
        self.build_relationships(pending_copy)

        for cls in pending_copy:
            # set the "active_history" flag
            for prop in sa.inspect(cls).iterate_properties:
                getattr(cls, prop.key).impl.active_history = True
<|endoftext|>


<commit_before># coding: utf-8
from functools import partial
import pandas as pd

from .graph import TransformGraph
from .gravity import (eotvos_correction, latitude_correction,
                      free_air_correction, kinematic_accel)
from .filters import lp_filter
from ..timesync import find_time_delay, shift_frame
from ..etc import align_frames
from .derivatives import taylor_fir, central_difference

def demux(df, col):
    return df[col]

class SyncGravity(TransformGraph):

    # TODO: align_frames only works with this ordering, but should work for either
    def __init__(self, kin_accel, gravity):
        self.transform_graph = {'gravity': gravity,
                                'raw_grav': gravity['gravity'],
                                'kin_accel': kin_accel,
                                'delay': (find_time_delay, 'kin_accel', 'raw_grav'),
                                'shifted_gravity': (shift_frame, 'gravity', 'delay'),
                                }

        super().__init__()

class AirbornePost(TransformGraph):

    # concat = partial(pd.concat, axis=1, join='outer')

    def total_corr(self, *args):
        return pd.Series(sum(*args), name='total_corr')

    def corrected_grav(self, *args):
        return pd.Series(sum(*args), name='corrected_grav')

    # TODO: What if a function takes a string argument? Use partial for now.
    # TODO: Little tricky to debug these graphs. Breakpoints? Print statements?
    def __init__(self, trajectory, gravity, begin_static, end_static):
        self.begin_static = begin_static
        self.end_static = end_static
        self.transform_graph = {'trajectory': trajectory,
                                'gravity': gravity,
                                'synced_gravity': (SyncGravity.run(item='shifted_gravity'), 'kin_accel', 'gravity'),
                                'shifted_trajectory': (partial(align_frames, item='r'), 'synced_gravity', 'trajectory'),
                                'shifted_gravity': (partial(align_frames, item='l'), 'synced_gravity', 'trajectory'),
                                'eotvos_and_accel': (partial(eotvos_correction, differentiator=central_difference), 'trajectory'),
                                'eotvos': (partial(demux, col='eotvos'), 'eotvos_and_accel'),
                                'kin_accel': (partial(demux, col='kin_accel'), 'eotvos_and_accel'),
                                'aligned_eotvos': (partial(align_frames, item='r'), 'shifted_trajectory', 'eotvos'),
                                'aligned_kin_accel': (partial(align_frames, item='r'), 'shifted_trajectory', 'kin_accel'),
                                'lat_corr': (latitude_correction, 'shifted_trajectory'),
                                'fac': (free_air_correction, 'shifted_trajectory'),
                                'total_corr': (self.total_corr, ['aligned_kin_accel', 'aligned_eotvos', 'lat_corr', 'fac']),
                                'abs_grav': (partial(demux, col='gravity'), 'shifted_gravity'),
                                'corrected_grav': (self.corrected_grav, ['total_corr', 'abs_grav']),
                                'filtered_grav': (partial(lp_filter, fs=10), 'corrected_grav')
                                }
        super().__init__()
<commit_msg>Temporarily removed time sync from airborne graph<commit_after># coding: utf-8
from functools import partial
import pandas as pd

from .graph import TransformGraph
from .gravity import (eotvos_correction, latitude_correction,
                      free_air_correction, kinematic_accel)
from .filters import lp_filter
from ..timesync import find_time_delay, shift_frame
from ..etc import align_frames
from .derivatives import taylor_fir, central_difference

def demux(df, col):
    return df[col]

class SyncGravity(TransformGraph):

    # TODO: align_frames only works with this ordering, but should work for either
    def __init__(self, kin_accel, gravity):
        self.transform_graph = {'gravity': gravity,
                                'raw_grav': gravity['gravity'],
                                'kin_accel': kin_accel,
                                'delay': (find_time_delay, 'kin_accel', 'raw_grav'),
                                'shifted_gravity': (shift_frame, 'gravity', 'delay'),
                                }

        super().__init__()

class AirbornePost(TransformGraph):

    # concat = partial(pd.concat, axis=1, join='outer')

    def total_corr(self, *args):
        return pd.Series(sum(*args), name='total_corr')

    def corrected_grav(self, *args):
        return pd.Series(sum(*args), name='corrected_grav')

    # TODO: What if a function takes a string argument? Use partial for now.
    # TODO: Little tricky to debug these graphs. Breakpoints? Print statements?
    def __init__(self, trajectory, gravity, begin_static, end_static):
        self.begin_static = begin_static
        self.end_static = end_static
        self.transform_graph = {'trajectory': trajectory,
                                'gravity': gravity,
                                'eotvos_and_accel': (partial(eotvos_correction, differentiator=central_difference), 'trajectory'),
                                'eotvos': (partial(demux, col='eotvos'), 'eotvos_and_accel'),
                                'kin_accel': (partial(demux, col='kin_accel'), 'eotvos_and_accel'),
                                'aligned_eotvos': (partial(align_frames, item='r'), 'trajectory', 'eotvos'),
                                'aligned_kin_accel': (partial(align_frames, item='r'), 'trajectory', 'kin_accel'),
                                'lat_corr': (latitude_correction, 'trajectory'),
                                'fac': (free_air_correction, 'trajectory'),
                                'total_corr': (self.total_corr, ['aligned_kin_accel', 'aligned_eotvos', 'lat_corr', 'fac']),
                                'abs_grav': (partial(demux, col='gravity'), 'gravity'),
                                'corrected_grav': (self.corrected_grav, ['total_corr', 'abs_grav']),
                                'filtered_grav': (partial(lp_filter, fs=10), 'corrected_grav')
                                }
        super().__init__()

# class ExampleGraph(TransformGraph):
#     inputs = ('trajectory', 'gravity', 'begin_static', 'end_static')
#     graph = {
#         ('gravity', 'trajectory'): (align_frames(item='r'), 'gravity', 'trajectory'),
#         ('eotvos', 'accel'): (eotvos_correction, 'trajectory'),
#         'lat_corr': (latitude_correction, 'trajectory'),
#         'fac': (free_air_correction, 'trajectory'),
#         'total_corr': (sum, ['eotvos', 'accel', 'lat_corr', 'fac']),
#         'corrected_grav': (sum, 'total_corr', demux('gravity', 'gravity')),
#         'filtered_grav': (lp_filter(fs=10), 'corrected_grav')
#     }
#
# @transformgraph
# def detrend(begin_static, end_static, data_in):
#     if hasattr(grav, 'index'):
#         length = len(data_in.index)
#     else:
#         length = len(data_in)
#
#     trend = np.linspace(begin, end, num=length)
#     if hasattr(data_in, 'sub'):
#         trend = pd.Series(trend, index=data_in.index)
#         result = data_in.sub(trend, axis=0)
#     else:
#         result = data_in - trend
#     return result
#
# # TODO: How to deal with keyword args?
# # should result in
# class Detrend(TransformGraph):
#     inputs = ('begin_static', 'end_static', 'data_in')
#     _func = detrend
#     graph = {
#         'result': (detrend, 'begin_static', 'end_static', 'data_in')
#     }<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import datetime
import inspect
import logging
import os
import random
import subprocess
import socket
import sys

from twisted.internet.threads import deferToThread

from nova import exception
from nova import flags
from nova.exception import ProcessExecutionError

FLAGS = flags.FLAGS
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, ValueError, AttributeError):
        raise exception.NotFound('Class %s cannot be found' % class_str)

def import_object(import_str):
    """Returns an object including a module or module and class"""
    try:
        __import__(import_str)
        return sys.modules[import_str]
    except ImportError:
        cls = import_class(import_str)
        return cls()

def fetchfile(url, target):
    logging.debug("Fetching %s" % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl --fail %s -o %s" % (url, target))

def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
    logging.debug("Running cmd: %s", cmd)
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if process_input != None:
        result = obj.communicate(process_input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        logging.debug("Result was %s" % (obj.returncode))
        if check_exit_code and obj.returncode <> 0:
            (stdout, stderr) = result
            raise ProcessExecutionError(exit_code=obj.returncode,
                                        stdout=stdout,
                                        stderr=stderr,
                                        cmd=cmd)
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            sys.argv = sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:]

def debug(arg):
    logging.debug('debug in callback: %s', arg)
    return arg

def runthis(prompt, cmd, check_exit_code = True):
    logging.debug("Running %s" % (cmd))
    exit_code = subprocess.call(cmd.split(" "))
    logging.debug(prompt % (exit_code))
    if check_exit_code and exit_code <> 0:
        raise ProcessExecutionError(exit_code=exit_code,
                                    stdout=None,
                                    stderr=None,
                                    cmd=cmd)

def generate_uid(topic, size=8):
    #TODO(gundlach): we want internal ids to just be ints now.  i just dropped
    #off a topic prefix, so what have I broken?
    return random.randint(0, 2**64-1)

def generate_mac():
    mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff), random.randint(0x00, 0xff)
           ]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def get_my_ip():
    """Returns the actual ip of the local machine."""
    if getattr(FLAGS, 'fake_tests', None):
        return '127.0.0.1'
    try:
        csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        csock.connect(('www.google.com', 80))
        (addr, port) = csock.getsockname()
        csock.close()
        return addr
    except socket.gaierror as ex:
        logging.warn("Couldn't get IP, using 127.0.0.1 %s", ex)
        return "127.0.0.1"

def isotime(at=None):
    if not at:
        at = datetime.datetime.utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    return datetime.datetime.strptime(timestr, TIME_FORMAT)

class LazyPluggable(object):
    """A pluggable backend loaded lazily based on some value."""

    def __init__(self, pivot, **backends):
        self.__backends = backends
        self.__pivot = pivot
        self.__backend = None

    def __get_backend(self):
        if not self.__backend:
            backend_name = self.__pivot.value
            if backend_name not in self.__backends:
                raise exception.Error('Invalid backend: %s' % backend_name)

            backend = self.__backends[backend_name]
            if type(backend) == type(tuple()):
                name = backend[0]
                fromlist = backend[1]
            else:
                name = backend
                fromlist = backend

            self.__backend = __import__(name, None, None, fromlist)
            logging.info('backend %s', self.__backend)
        return self.__backend

    def __getattr__(self, key):
        backend = self.__get_backend()
        return getattr(backend, key)

def deferredToThread(f):
    def g(*args, **kwargs):
        return deferToThread(f, *args, **kwargs)
    return g
<commit_msg>Revert r312<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

import datetime
import inspect
import logging
import os
import random
import subprocess
import socket
import sys

from twisted.internet.threads import deferToThread

from nova import exception
from nova import flags
from nova.exception import ProcessExecutionError

FLAGS = flags.FLAGS
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, ValueError, AttributeError):
        raise exception.NotFound('Class %s cannot be found' % class_str)

def import_object(import_str):
    """Returns an object including a module or module and class"""
    try:
        __import__(import_str)
        return sys.modules[import_str]
    except ImportError:
        cls = import_class(import_str)
        return cls()

def fetchfile(url, target):
    logging.debug("Fetching %s" % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl --fail %s -o %s" % (url, target))

def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
    logging.debug("Running cmd: %s", cmd)
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if process_input != None:
        result = obj.communicate(process_input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        logging.debug("Result was %s" % (obj.returncode))
        if check_exit_code and obj.returncode <> 0:
            (stdout, stderr) = result
            raise ProcessExecutionError(exit_code=obj.returncode,
                                        stdout=stdout,
                                        stderr=stderr,
                                        cmd=cmd)
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            sys.argv = sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:]

def debug(arg):
    logging.debug('debug in callback: %s', arg)
    return arg

def runthis(prompt, cmd, check_exit_code = True):
    logging.debug("Running %s" % (cmd))
    exit_code = subprocess.call(cmd.split(" "))
    logging.debug(prompt % (exit_code))
    if check_exit_code and exit_code <> 0:
        raise ProcessExecutionError(exit_code=exit_code,
                                    stdout=None,
                                    stderr=None,
                                    cmd=cmd)

def generate_uid(topic, size=8):
     return '%s-%s' % (topic, ''.join([random.choice('01234567890abcdefghijklmnopqrstuvwxyz') for x in xrange(size)]))

def generate_mac():
    mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff), random.randint(0x00, 0xff)
           ]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def get_my_ip():
    """Returns the actual ip of the local machine."""
    if getattr(FLAGS, 'fake_tests', None):
        return '127.0.0.1'
    try:
        csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        csock.connect(('www.google.com', 80))
        (addr, port) = csock.getsockname()
        csock.close()
        return addr
    except socket.gaierror as ex:
        logging.warn("Couldn't get IP, using 127.0.0.1 %s", ex)
        return "127.0.0.1"

def isotime(at=None):
    if not at:
        at = datetime.datetime.utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    return datetime.datetime.strptime(timestr, TIME_FORMAT)

class LazyPluggable(object):
    """A pluggable backend loaded lazily based on some value."""

    def __init__(self, pivot, **backends):
        self.__backends = backends
        self.__pivot = pivot
        self.__backend = None

    def __get_backend(self):
        if not self.__backend:
            backend_name = self.__pivot.value
            if backend_name not in self.__backends:
                raise exception.Error('Invalid backend: %s' % backend_name)

            backend = self.__backends[backend_name]
            if type(backend) == type(tuple()):
                name = backend[0]
                fromlist = backend[1]
            else:
                name = backend
                fromlist = backend

            self.__backend = __import__(name, None, None, fromlist)
            logging.info('backend %s', self.__backend)
        return self.__backend

    def __getattr__(self, key):
        backend = self.__get_backend()
        return getattr(backend, key)

def deferredToThread(f):
    def g(*args, **kwargs):
        return deferToThread(f, *args, **kwargs)
    return g
<|endoftext|>