<commit_before>#
#  Copyright (c) 2015 - 2022, Intel Corporation
#  SPDX-License-Identifier: BSD-3-Clause
#

from contextlib import contextmanager
import os
import pipes
import re
import shutil
import sys
import unittest
import subprocess
from io import StringIO
import argparse

from docutils.nodes import literal_block
from docutils.core import publish_doctree

from integration.test import geopm_test_launcher

# the global singleton
class Util:
    _instance = None

    def __init__(self):
        self._do_parse_once = False
        self._skip_launch = False

    @classmethod
    def instance(cls):
        if not cls._instance:
            cls._instance = Util()
            cls._instance.parse_command_line_args()
        return cls._instance

    def parse_command_line_args(self):
        ''' Parses command line to remove geopm integration-specific options.
            New options added here should not conflict with the set
            provided by unittest.main().
            @todo: perhaps a geopm- prefix for options?
        '''
        if self._do_parse_once:
            return

        self._do_parse_once = True
        parser = argparse.ArgumentParser()
        parser.add_argument('--skip-launch', dest='skip_launch',
                            action='store_true', default=False,
                            help='reuse existing data files; do not launch any jobs')
        # workaround so that this help plus base pyunit help text will be printed
        if '-h' in sys.argv or '--help' in sys.argv:
            # TODO: use custom help text; this is confusing
            sys.stdout.write('GEOPM test-specific help:\n')
            sys.stdout.write('-------------------------\n')
            parser.print_help()
            sys.stdout.write('\nCommon pyunit help:\n')
            sys.stdout.write(  '-------------------\n')
            return

        args, remaining = parser.parse_known_args()
        sys.argv = [sys.argv[0]] + remaining
        self._skip_launch = args.skip_launch

    def skip_launch(self):
        ''' Skip parts of test requiring a job launched on compute nodes.
            Ideally the test should only analyze existing report/trace files.
        '''
        return self._skip_launch

# will be created the first time this file is imported
g_util = Util.instance()

def do_launch():
    return not g_util.skip_launch()

def skip_unless_do_launch():
    if not do_launch():
        return unittest.skip("Most tests in this suite require launch; do not set --skip-launch.")
    return lambda func: func

def skip_unless_levelzero_power():
    #try/catch read signal for levelzero
    try:
        power = geopm_test_launcher.geopmread("LEVELZERO::GPU_POWER gpu 0")
    except subprocess.CalledProcessError:
        return unittest.skip("Read of LEVELZERO::GPU_POWER not supported, skipping test.")
    return lambda func: func

def skip_unless_levelzero():
    return skip_unless_config_enable('levelzero');

def skip_unless_platform_bdx():
    fam, mod = geopm_test_launcher.get_platform()
    if fam != 6 or mod not in (45, 47, 79):
        return unittest.skip("Performance test is tuned for BDX server, The family {}, model {} is not supported.".format(fam, mod))
    return lambda func: func

def get_config_log():
    path = os.path.join(
           os.path.dirname(
            os.path.dirname(
             os.path.dirname(
              os.path.realpath(__file__)))),
           'config.log')
    if not os.path.isfile(path):
        path = os.path.join(
               os.path.dirname(path),
                'integration/build/config.log')
    return path

def get_config_value(key):
    """Get the value of an option from the build configuration, returning None
    if no such key is present.
    """
    path = get_config_log()
    with open(path) as config_file:
        for line in config_file:
            line_start = "{}='".format(key)
            line_end = "'\n"
            if line.startswith(line_start) and line.endswith(line_end):
                return line[len(line_start):-len(line_end)]
    return None

def get_exec_path(bin_name):
    """Find test binary

    """
    path_list = []
    int_test_dir = os.path.dirname(os.path.realpath(__file__))
    path_list.append(os.path.join(int_test_dir, '.libs'))
    int_dir = os.path.dirname(int_test_dir)
    # Look for out of place build from using apps/build_func.sh
    path_list.append(os.path.join(int_dir, 'build/integration/test/.libs'))
    bin_list = [os.path.join(pp, bin_name) for pp in path_list]
    for bb in bin_list:
        if os.path.isfile(bb):
            return bb

    sep = '\n    '
    msg = 'Could not find application binary, tried {}{}'.format(sep, sep.join(bin_list))
    raise RuntimeError(msg)

def skip_unless_config_enable(feature):
    if get_config_value('enable_{}'.format(feature)) == '1':
        return lambda func: func
    else:
        return unittest.skip("Feature: {feature} is not enabled, configure with --enable-{feature} to run this test.".format(feature=feature))

def skip_unless_optimized():
    path = get_config_log()
    with open(path) as fid:
        for line in fid.readlines():
            if line.startswith("enable_debug='1'"):
                return unittest.skip("This performance test cannot be run when GEOPM is configured with --enable-debug")
    return lambda func: func

def skip_unless_batch():
    batch_env_vars = ['SLURM_NODELIST', 'COBALT_JOBID']
    if not g_util.skip_launch() and not any(opt in batch_env_vars for opt in os.environ):
        return unittest.skip('Requires batch session.')
    return lambda func: func

def skip_unless_cpufreq():
    if not g_util.skip_launch():
        try:
            test_exec = "dummy -- stat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq"
            dev_null = open('/dev/null', 'w')
            geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
            dev_null.close()
        except subprocess.CalledProcessError:
            return unittest.skip("Could not determine min frequency, enable cpufreq driver to run this test.")
    return lambda func: func

def skip_unless_stressng():
    if not g_util.skip_launch():
        try:
            test_exec = "dummy -- stress-ng -h"
            dev_null = open('/dev/null', 'w')
            geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
            dev_null.close()
        except subprocess.CalledProcessError:
            return unittest.skip("Missing stress-ng.  Please install in the compute image.")
    return lambda func: func

def skip_or_ensure_writable_file(path):
    """Skip the test unless the given file can be created or modified on compute nodes.
    """
    if g_util.skip_launch():
        return unittest.skip("This test requires launch; do not use --skip-launch")

    stdout = StringIO()
    created_directories = list()
    try:
        dirname = os.path.dirname(path)
        with open('/dev/null', 'w') as dev_null:
            # Ensure the parent directory exists
            geopm_test_launcher.allocation_node_test('dummy -- mkdir -vp {}'.format(dirname), stdout, dev_null)
            stdout.seek(0)
            pattern = re.compile(r"mkdir: created directory '([^']+)'")
            for line in stdout:
                match = pattern.search(line)
                if match:
                    created_directories.append(match.group(1))

            run_script_on_compute_nodes("test '(' '!' -e {path} -a -w {dirname} ')' -o -w {path}".format(
                path=pipes.quote(path), dirname=pipes.quote(dirname)), sys.stdout, sys.stdout)
    except subprocess.CalledProcessError as e:
        return unittest.skip("Cannot write to path: {}".format(path))

    def cleanup_decorator(func):
        def cleanup_wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            finally:
                for directory in reversed(created_directories):
                    shutil.rmtree(directory)
        return cleanup_wrapper

    return cleanup_decorator

def skip_unless_library_in_ldconfig(library):
    """Skip the test if the given library is not in ldconfig on the compute nodes.
    """
    if not g_util.skip_launch():
        try:
            with open('/dev/null', 'w') as dev_null:
                stdout = StringIO()
                geopm_test_launcher.allocation_node_test('dummy -- ldconfig --print-cache', stdout, dev_null)
                stdout.seek(0)
                if not any(line.startswith('\t{}'.format(library)) for line in stdout):
                    return unittest.skip("Library '{}' not in ldconfig".format(library))
        except subprocess.CalledProcessError:
            return unittest.skip("Unable to run ldconfig")
    return lambda func: func

def skip_unless_msr_access(msg=None):
    """Skip the test if the user does not have direct access to msr-safe
    """
    if not g_util.skip_launch():
        try_path = '/dev/cpu/msr_batch'
        try:
            test_exec = 'dummy -- test -w {}'.format(try_path)
            with open('/dev/null', 'w') as dev_null:
                geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
        except subprocess.CalledProcessError:
            if msg is None:
                msg = 'No msr-safe access.  Cannot write to ' + try_path
            return unittest.skip(msg)
    return lambda func: func

def run_script_on_compute_nodes(script, stdout, stderr, interpreter='sh'):
    """Run an inline script on compute nodes.
    """
    geopm_test_launcher.allocation_node_test('dummy -- {}'.format(
        '{} -c {}'.format(interpreter, pipes.quote(script))), stdout, stderr)

@contextmanager
def temporarily_remove_compute_node_file(path):
    """Context manager to remove a file from compute nodes if it exists, by renaming
    with a .backup suffix.  When exiting the context, the file is restored by moving
    the backup file to its original path.
    """
    with open('/dev/null', 'w') as dev_null:
        try:
            run_script_on_compute_nodes("test '!' -e {path} || mv -f {path} {path}.backup".format(path=path),
                                        dev_null, dev_null)
            yield path
        finally:
            run_script_on_compute_nodes("rm -f {path} && test '!' -e {path}.backup || mv {path}.backup {path}".format(path=pipes.quote(path)),
                                        dev_null, dev_null)

def remove_file_on_compute_nodes(file_path):
    """Remove a file from compute nodes.
    """
    with open('/dev/null', 'w') as dev_null:
        geopm_test_launcher.allocation_node_test('dummy -- rm {}'.format(file_path), dev_null, dev_null)

def assertNear(self, a, b, epsilon=0.05, msg=''):
    denom = a if a != 0 else 1
    if abs((a - b) / denom) >= epsilon:
        self.fail('The fractional difference between {a} and {b} is greater than {epsilon}.  {msg}'.format(a=a, b=b, epsilon=epsilon, msg=msg))

def assertNotNear(self, a, b, epsilon=0.05, msg=''):
    denom = a if a != 0 else 1
    if abs((a - b) / denom) < epsilon:
        self.fail('The fractional difference between {a} and {b} is less than {epsilon}.  {msg}'.format(a=a, b=b, epsilon=epsilon, msg=msg))

def get_scripts_from_readme(rst_readme_path):
    """Return a list of the sh-formatted scripts that are embedded in a .rst file.
    """
    script_bodies = list()
    with open(rst_readme_path) as f:
        # Syntax highlighting is enabled by default. If the syntax highlighter
        # (Pygments) is not installed, the tree-walk will skip over code blocks
        # and this function will return an empty list. Explicitly disable
        # highlighting so we can run this test regardless of whether Pygments
        # is installed.
        doctree = publish_doctree(f.read(), settings_overrides={'syntax_highlight': 'none'})
        for code_block in doctree.findall(literal_block):
            if 'sh' in code_block['classes']:
                script_bodies.append(code_block.astext())
    return script_bodies
<commit_msg>Added check for workload executable<commit_after>#
#  Copyright (c) 2015 - 2022, Intel Corporation
#  SPDX-License-Identifier: BSD-3-Clause
#

from contextlib import contextmanager
import os
import pipes
import re
import shutil
import sys
import unittest
import subprocess
from io import StringIO
import argparse

from docutils.nodes import literal_block
from docutils.core import publish_doctree

from integration.test import geopm_test_launcher

# the global singleton
class Util:
    _instance = None

    def __init__(self):
        self._do_parse_once = False
        self._skip_launch = False

    @classmethod
    def instance(cls):
        if not cls._instance:
            cls._instance = Util()
            cls._instance.parse_command_line_args()
        return cls._instance

    def parse_command_line_args(self):
        ''' Parses command line to remove geopm integration-specific options.
            New options added here should not conflict with the set
            provided by unittest.main().
            @todo: perhaps a geopm- prefix for options?
        '''
        if self._do_parse_once:
            return

        self._do_parse_once = True
        parser = argparse.ArgumentParser()
        parser.add_argument('--skip-launch', dest='skip_launch',
                            action='store_true', default=False,
                            help='reuse existing data files; do not launch any jobs')
        # workaround so that this help plus base pyunit help text will be printed
        if '-h' in sys.argv or '--help' in sys.argv:
            # TODO: use custom help text; this is confusing
            sys.stdout.write('GEOPM test-specific help:\n')
            sys.stdout.write('-------------------------\n')
            parser.print_help()
            sys.stdout.write('\nCommon pyunit help:\n')
            sys.stdout.write(  '-------------------\n')
            return

        args, remaining = parser.parse_known_args()
        sys.argv = [sys.argv[0]] + remaining
        self._skip_launch = args.skip_launch

    def skip_launch(self):
        ''' Skip parts of test requiring a job launched on compute nodes.
            Ideally the test should only analyze existing report/trace files.
        '''
        return self._skip_launch

# will be created the first time this file is imported
g_util = Util.instance()

def do_launch():
    return not g_util.skip_launch()

def skip_unless_do_launch():
    if not do_launch():
        return unittest.skip("Most tests in this suite require launch; do not set --skip-launch.")
    return lambda func: func

def skip_unless_workload_exists(path):
    path = os.path.join(
            os.path.dirname(
             os.path.dirname(os.path.realpath(__file__))),
            path)
    print("path is {}".format(path))
    print("path exists is {}".format(os.path.exists(path)))
    if not os.path.exists(path):
        return unittest.skip("Could not find workload executable {}, skipping test.".format(path))

    return lambda func: func

def skip_unless_levelzero_power():
    #try/catch read signal for levelzero
    try:
        power = geopm_test_launcher.geopmread("LEVELZERO::GPU_POWER gpu 0")
    except subprocess.CalledProcessError:
        return unittest.skip("Read of LEVELZERO::GPU_POWER not supported, skipping test.")
    return lambda func: func

def skip_unless_levelzero():
    return skip_unless_config_enable('levelzero');

def skip_unless_platform_bdx():
    fam, mod = geopm_test_launcher.get_platform()
    if fam != 6 or mod not in (45, 47, 79):
        return unittest.skip("Performance test is tuned for BDX server, The family {}, model {} is not supported.".format(fam, mod))
    return lambda func: func

def get_config_log():
    path = os.path.join(
           os.path.dirname(
            os.path.dirname(
             os.path.dirname(
              os.path.realpath(__file__)))),
           'config.log')
    if not os.path.isfile(path):
        path = os.path.join(
               os.path.dirname(path),
                'integration/build/config.log')
    return path

def get_config_value(key):
    """Get the value of an option from the build configuration, returning None
    if no such key is present.
    """
    path = get_config_log()
    with open(path) as config_file:
        for line in config_file:
            line_start = "{}='".format(key)
            line_end = "'\n"
            if line.startswith(line_start) and line.endswith(line_end):
                return line[len(line_start):-len(line_end)]
    return None

def get_exec_path(bin_name):
    """Find test binary

    """
    path_list = []
    int_test_dir = os.path.dirname(os.path.realpath(__file__))
    path_list.append(os.path.join(int_test_dir, '.libs'))
    int_dir = os.path.dirname(int_test_dir)
    # Look for out of place build from using apps/build_func.sh
    path_list.append(os.path.join(int_dir, 'build/integration/test/.libs'))
    bin_list = [os.path.join(pp, bin_name) for pp in path_list]
    for bb in bin_list:
        if os.path.isfile(bb):
            return bb

    sep = '\n    '
    msg = 'Could not find application binary, tried {}{}'.format(sep, sep.join(bin_list))
    raise RuntimeError(msg)

def skip_unless_config_enable(feature):
    if get_config_value('enable_{}'.format(feature)) == '1':
        return lambda func: func
    else:
        return unittest.skip("Feature: {feature} is not enabled, configure with --enable-{feature} to run this test.".format(feature=feature))

def skip_unless_optimized():
    path = get_config_log()
    with open(path) as fid:
        for line in fid.readlines():
            if line.startswith("enable_debug='1'"):
                return unittest.skip("This performance test cannot be run when GEOPM is configured with --enable-debug")
    return lambda func: func

def skip_unless_batch():
    batch_env_vars = ['SLURM_NODELIST', 'COBALT_JOBID']
    if not g_util.skip_launch() and not any(opt in batch_env_vars for opt in os.environ):
        return unittest.skip('Requires batch session.')
    return lambda func: func

def skip_unless_cpufreq():
    if not g_util.skip_launch():
        try:
            test_exec = "dummy -- stat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq"
            dev_null = open('/dev/null', 'w')
            geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
            dev_null.close()
        except subprocess.CalledProcessError:
            return unittest.skip("Could not determine min frequency, enable cpufreq driver to run this test.")
    return lambda func: func

def skip_unless_stressng():
    if not g_util.skip_launch():
        try:
            test_exec = "dummy -- stress-ng -h"
            dev_null = open('/dev/null', 'w')
            geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
            dev_null.close()
        except subprocess.CalledProcessError:
            return unittest.skip("Missing stress-ng.  Please install in the compute image.")
    return lambda func: func

def skip_or_ensure_writable_file(path):
    """Skip the test unless the given file can be created or modified on compute nodes.
    """
    if g_util.skip_launch():
        return unittest.skip("This test requires launch; do not use --skip-launch")

    stdout = StringIO()
    created_directories = list()
    try:
        dirname = os.path.dirname(path)
        with open('/dev/null', 'w') as dev_null:
            # Ensure the parent directory exists
            geopm_test_launcher.allocation_node_test('dummy -- mkdir -vp {}'.format(dirname), stdout, dev_null)
            stdout.seek(0)
            pattern = re.compile(r"mkdir: created directory '([^']+)'")
            for line in stdout:
                match = pattern.search(line)
                if match:
                    created_directories.append(match.group(1))

            run_script_on_compute_nodes("test '(' '!' -e {path} -a -w {dirname} ')' -o -w {path}".format(
                path=pipes.quote(path), dirname=pipes.quote(dirname)), sys.stdout, sys.stdout)
    except subprocess.CalledProcessError as e:
        return unittest.skip("Cannot write to path: {}".format(path))

    def cleanup_decorator(func):
        def cleanup_wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            finally:
                for directory in reversed(created_directories):
                    shutil.rmtree(directory)
        return cleanup_wrapper

    return cleanup_decorator

def skip_unless_library_in_ldconfig(library):
    """Skip the test if the given library is not in ldconfig on the compute nodes.
    """
    if not g_util.skip_launch():
        try:
            with open('/dev/null', 'w') as dev_null:
                stdout = StringIO()
                geopm_test_launcher.allocation_node_test('dummy -- ldconfig --print-cache', stdout, dev_null)
                stdout.seek(0)
                if not any(line.startswith('\t{}'.format(library)) for line in stdout):
                    return unittest.skip("Library '{}' not in ldconfig".format(library))
        except subprocess.CalledProcessError:
            return unittest.skip("Unable to run ldconfig")
    return lambda func: func

def skip_unless_msr_access(msg=None):
    """Skip the test if the user does not have direct access to msr-safe
    """
    if not g_util.skip_launch():
        try_path = '/dev/cpu/msr_batch'
        try:
            test_exec = 'dummy -- test -w {}'.format(try_path)
            with open('/dev/null', 'w') as dev_null:
                geopm_test_launcher.allocation_node_test(test_exec, dev_null, dev_null)
        except subprocess.CalledProcessError:
            if msg is None:
                msg = 'No msr-safe access.  Cannot write to ' + try_path
            return unittest.skip(msg)
    return lambda func: func

def run_script_on_compute_nodes(script, stdout, stderr, interpreter='sh'):
    """Run an inline script on compute nodes.
    """
    geopm_test_launcher.allocation_node_test('dummy -- {}'.format(
        '{} -c {}'.format(interpreter, pipes.quote(script))), stdout, stderr)

@contextmanager
def temporarily_remove_compute_node_file(path):
    """Context manager to remove a file from compute nodes if it exists, by renaming
    with a .backup suffix.  When exiting the context, the file is restored by moving
    the backup file to its original path.
    """
    with open('/dev/null', 'w') as dev_null:
        try:
            run_script_on_compute_nodes("test '!' -e {path} || mv -f {path} {path}.backup".format(path=path),
                                        dev_null, dev_null)
            yield path
        finally:
            run_script_on_compute_nodes("rm -f {path} && test '!' -e {path}.backup || mv {path}.backup {path}".format(path=pipes.quote(path)),
                                        dev_null, dev_null)

def remove_file_on_compute_nodes(file_path):
    """Remove a file from compute nodes.
    """
    with open('/dev/null', 'w') as dev_null:
        geopm_test_launcher.allocation_node_test('dummy -- rm {}'.format(file_path), dev_null, dev_null)

def assertNear(self, a, b, epsilon=0.05, msg=''):
    denom = a if a != 0 else 1
    if abs((a - b) / denom) >= epsilon:
        self.fail('The fractional difference between {a} and {b} is greater than {epsilon}.  {msg}'.format(a=a, b=b, epsilon=epsilon, msg=msg))

def assertNotNear(self, a, b, epsilon=0.05, msg=''):
    denom = a if a != 0 else 1
    if abs((a - b) / denom) < epsilon:
        self.fail('The fractional difference between {a} and {b} is less than {epsilon}.  {msg}'.format(a=a, b=b, epsilon=epsilon, msg=msg))

def get_scripts_from_readme(rst_readme_path):
    """Return a list of the sh-formatted scripts that are embedded in a .rst file.
    """
    script_bodies = list()
    with open(rst_readme_path) as f:
        # Syntax highlighting is enabled by default. If the syntax highlighter
        # (Pygments) is not installed, the tree-walk will skip over code blocks
        # and this function will return an empty list. Explicitly disable
        # highlighting so we can run this test regardless of whether Pygments
        # is installed.
        doctree = publish_doctree(f.read(), settings_overrides={'syntax_highlight': 'none'})
        for code_block in doctree.findall(literal_block):
            if 'sh' in code_block['classes']:
                script_bodies.append(code_block.astext())
    return script_bodies
<|endoftext|>


<commit_before>#   This Python module is part of the PyRate software package.
#
#   Copyright 2021 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import resource
from collections import namedtuple
from typing import List, Dict, Tuple, Any
from nptyping import NDArray, Float32, UInt16
import numpy as np

import pyrate.constants as C
from pyrate.core import mpiops
from pyrate.core.shared import Ifg, join_dicts
from pyrate.core.phase_closure.mst_closure import Edge, WeightedLoop
from pyrate.core.logger import pyratelogger as log

IndexedIfg = namedtuple('IndexedIfg', ['index', 'IfgPhase'])

class IfgPhase:
    """
    workaround class to only hold phase data for mpi SwigPyObject pickle error
    """

    def __init__(self, phase_data):
        self.phase_data = phase_data

def __create_ifg_edge_dict(ifg_files: List[str], params: dict) -> Dict[Edge, IndexedIfg]:
    """Returns a dictionary of indexed ifg 'edges'"""
    ifg_files.sort()
    ifgs = [Ifg(i) for i in ifg_files]

    def _func(ifg, index):
        ifg.open()
        ifg.nodata_value = params[C.NO_DATA_VALUE]
        ifg.convert_to_nans()
        ifg.convert_to_radians()
        idx_ifg = IndexedIfg(index, IfgPhase(ifg.phase_data))
        return idx_ifg

    process_ifgs = mpiops.array_split(list(enumerate(ifgs)))
    ret_combined = {}
    for idx, _ifg in process_ifgs:
        ret_combined[Edge(_ifg.first, _ifg.second)] = _func(_ifg, idx)
        _ifg.close()

    ret_combined = join_dicts(mpiops.comm.allgather(ret_combined))
    return ret_combined

def sum_phase_closures(ifg_files: List[str], loops: List[WeightedLoop], params: dict) -> \
        Tuple[NDArray[(Any, Any, Any), Float32], NDArray[(Any, Any, Any), UInt16], NDArray[(Any, ), UInt16]]:
    """
    Compute the closure sum for each pixel in each loop, and count the number of times a pixel
    contributes to a failed closure loop (where the summed closure is above/below the
    LARGE_DEV_THR threshold).
    :param ifg_files: list of ifg files
    :param loops: list of loops
    :param params: params dict
    :return: Tuple of closure, ifgs_breach_count_process, num_occurrences_each_ifg
        closure: summed closure for each loop.
        ifgs_breach_count_process: shape=(ifg.shape, n_ifgs) number of times a pixel in an ifg fails the closure
            check (i.e., has unwrapping error) in all loops under investigation.
        num_occurrences_each_ifg: frequency of ifg appearance in all loops.
    """
    edge_to_indexed_ifgs = __create_ifg_edge_dict(ifg_files, params)
    ifgs = [v.IfgPhase for v in edge_to_indexed_ifgs.values()]
    n_ifgs = len(ifgs)
    closure_dict = {}

    if params[C.PARALLEL]:
        # rets = Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))(
        #     delayed(__compute_ifgs_breach_count)(ifg0, n_ifgs, weighted_loop, edge_to_indexed_ifgs, params)
        #     for weighted_loop in loops
        # )
        # for k, r in enumerate(rets):
        #     closure_dict[k], ifgs_breach_count_dict[k] = r
        # TODO: enable multiprocessing - needs pickle error fix
        ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        for k, weighted_loop in enumerate(loops):
            closure_dict[k], ifgs_breach_count_l = __compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs,
                                                                               params)
            ifgs_breach_count += ifgs_breach_count_l
    else:
        loops_with_index = list(enumerate(loops))
        process_loops = mpiops.array_split(loops_with_index)
        ifgs_breach_count_process = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        for k, weighted_loop in process_loops:
            closure_dict[k], ifgs_breach_count_l = __compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs,
                                                                               params)
            ifgs_breach_count_process += ifgs_breach_count_l  # process
        closure_dict = join_dicts(mpiops.comm.gather(closure_dict, root=0))

        total_gb = mpiops.comm.allreduce(ifgs_breach_count_process.nbytes / 1e9, op=mpiops.MPI.SUM)
        log.info("Memory usage due to ifgs_breach_count_process {:2.4f}GB of data".format(total_gb))
        log.debug(f"shape of ifgs_breach_count_process is {ifgs_breach_count_process.shape}")
        log.debug(f"dtype of ifgs_breach_count_process is {ifgs_breach_count_process.dtype}")
        if mpiops.rank == 0:
            ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        else:
            ifgs_breach_count = None
        if mpiops.MPI_INSTALLED:
            mpiops.comm.Reduce([ifgs_breach_count_process, mpiops.MPI.UINT16_T],
                               [ifgs_breach_count, mpiops.MPI.UINT16_T], op=mpiops.MPI.SUM, root=0)  # global
        else:
            ifgs_breach_count = mpiops.comm.reduce(ifgs_breach_count_process, op=mpiops.sum0_op, root=0)

        log.debug(f"successfully summed phase closure breach array")

    closure, num_occurrences_each_ifg = None, None
    if mpiops.rank == 0:
        num_occurrences_each_ifg = _find_num_occurrences_each_ifg(loops, edge_to_indexed_ifgs, n_ifgs)
        closure = np.dstack([v for k, v in sorted(closure_dict.items(), key=lambda x: x[0])])

    return closure, ifgs_breach_count, num_occurrences_each_ifg

def _find_num_occurrences_each_ifg(loops: List[WeightedLoop],
                                   edge_to_indexed_ifgs: Dict[Edge, IndexedIfg],
                                   n_ifgs: int) -> NDArray[(Any, ), UInt16]:
    """find how many times each ifg appears in total in all loops"""
    num_occurrences_each_ifg = np.zeros(shape=n_ifgs, dtype=np.uint16)
    for weighted_loop in loops:
        for signed_edge in weighted_loop.loop:
            indexed_ifg = edge_to_indexed_ifgs[signed_edge.edge]
            ifg_index = indexed_ifg.index
            num_occurrences_each_ifg[ifg_index] += 1
    return num_occurrences_each_ifg

def __compute_ifgs_breach_count(weighted_loop: WeightedLoop,
                                edge_to_indexed_ifgs: Dict[Edge, IndexedIfg], params: dict) \
        -> Tuple[NDArray[(Any, Any), Float32], NDArray[(Any, Any, Any), UInt16]]:
    """Compute summed `closure` of each loop, and compute `ifgs_breach_count` for each pixel."""
    n_ifgs = len(edge_to_indexed_ifgs)
    indexed_ifg = list(edge_to_indexed_ifgs.values())[0]
    ifg = indexed_ifg.IfgPhase
    large_dev_thr = params[C.LARGE_DEV_THR] * np.pi
    use_median = params[C.SUBTRACT_MEDIAN]

    closure = np.zeros(shape=ifg.phase_data.shape, dtype=np.float32)
    # initiate variable for check of unwrapping issues at the same pixels in all loops
    ifgs_breach_count = np.zeros(shape=(ifg.phase_data.shape + (n_ifgs,)), dtype=np.uint16)

    for signed_edge in weighted_loop.loop:
        indexed_ifg = edge_to_indexed_ifgs[signed_edge.edge]
        ifg = indexed_ifg.IfgPhase
        closure += signed_edge.sign * ifg.phase_data
    if use_median:
        closure -= np.nanmedian(closure)  # may be able to drop median

    # this will deal with nans in `closure`, i.e., nans are not selected in indices_breaching_threshold
    indices_breaching_threshold = np.absolute(closure) > large_dev_thr

    for signed_edge in weighted_loop.loop:
        ifg_index = edge_to_indexed_ifgs[signed_edge.edge].index
        # the variable 'ifgs_breach_count' is increased by 1 for that pixel
        # make sure we are not incrementing the nan positions in the closure
        # as we don't know the phase of these pixels and also they were converted to zero before large_dev_thr check
        # Therefore, we leave them out of ifgs_breach_count, i.e., we don't increment their ifgs_breach_count values
        ifgs_breach_count[indices_breaching_threshold, ifg_index] += 1
    return closure, ifgs_breach_count
<commit_msg>don't use dictionaries to communicate closure result when buffers and Gatherv are fine<commit_after>#   This Python module is part of the PyRate software package.
#
#   Copyright 2021 Geoscience Australia
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import resource
from collections import namedtuple
from typing import List, Dict, Tuple, Any
from nptyping import NDArray, Float32, UInt16
import numpy as np

import pyrate.constants as C
from pyrate.core import mpiops
from pyrate.core.shared import Ifg, join_dicts
from pyrate.core.phase_closure.mst_closure import Edge, WeightedLoop
from pyrate.core.logger import pyratelogger as log

IndexedIfg = namedtuple('IndexedIfg', ['index', 'IfgPhase'])

class IfgPhase:
    """
    workaround class to only hold phase data for mpi SwigPyObject pickle error
    """

    def __init__(self, phase_data):
        self.phase_data = phase_data

def __create_ifg_edge_dict(ifg_files: List[str], params: dict) -> Dict[Edge, IndexedIfg]:
    """Returns a dictionary of indexed ifg 'edges'"""
    ifg_files.sort()
    ifgs = [Ifg(i) for i in ifg_files]

    def _func(ifg, index):
        ifg.open()
        ifg.nodata_value = params[C.NO_DATA_VALUE]
        ifg.convert_to_nans()
        ifg.convert_to_radians()
        idx_ifg = IndexedIfg(index, IfgPhase(ifg.phase_data))
        return idx_ifg

    process_ifgs = mpiops.array_split(list(enumerate(ifgs)))
    ret_combined = {}
    for idx, _ifg in process_ifgs:
        ret_combined[Edge(_ifg.first, _ifg.second)] = _func(_ifg, idx)
        _ifg.close()

    ret_combined = join_dicts(mpiops.comm.allgather(ret_combined))
    return ret_combined

def sum_phase_closures(ifg_files: List[str], loops: List[WeightedLoop], params: dict) -> \
        Tuple[NDArray[(Any, Any, Any), Float32], NDArray[(Any, Any, Any), UInt16], NDArray[(Any, ), UInt16]]:
    """
    Compute the closure sum for each pixel in each loop, and count the number of times a pixel
    contributes to a failed closure loop (where the summed closure is above/below the
    LARGE_DEV_THR threshold).
    :param ifg_files: list of ifg files
    :param loops: list of loops
    :param params: params dict
    :return: Tuple of closure, ifgs_breach_count_process, num_occurrences_each_ifg
        closure: summed closure for each loop.
        ifgs_breach_count_process: shape=(ifg.shape, n_ifgs) number of times a pixel in an ifg fails the closure
            check (i.e., has unwrapping error) in all loops under investigation.
        num_occurrences_each_ifg: frequency of ifg appearance in all loops.
    """
    edge_to_indexed_ifgs = __create_ifg_edge_dict(ifg_files, params)
    ifgs = [v.IfgPhase for v in edge_to_indexed_ifgs.values()]
    n_ifgs = len(ifgs)
    closure = None

    if params[C.PARALLEL]:
        # rets = Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))(
        #     delayed(__compute_ifgs_breach_count)(ifg0, n_ifgs, weighted_loop, edge_to_indexed_ifgs, params)
        #     for weighted_loop in loops
        # )
        # for k, r in enumerate(rets):
        #     closure_dict[k], ifgs_breach_count_dict[k] = r
        # TODO: enable multiprocessing - needs pickle error fix
        ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        closure = np.zeros((len(loops), *ifgs[0].phase_data.shape), dtype=np.float32)
        for k, weighted_loop in enumerate(loops):
            closure[k,:,:], ifgs_breach_count_l = __compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs,
                                                                               params)
            ifgs_breach_count += ifgs_breach_count_l
    else:
        process_loops = mpiops.array_split(loops)
        ifgs_breach_count_process = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        closure_process = np.zeros((len(loops), *ifgs[0].phase_data.shape), dtype=np.float32)
        for k, weighted_loop in enumerate(process_loops):
            closure_process[k,:,:], ifgs_breach_count_l = __compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs,
                                                                               params)
            ifgs_breach_count_process += ifgs_breach_count_l  # process
        if mpiops.rank == 0:
            closure = np.zeros((len(loops), *ifgs[0].phase_data.shape), dtype=np.float32)
        if mpiops.MPI_INSTALLED:
            #this Gatherv setup depends on the behaviour of np.array_split
            #and the default row-major order of numpy arrays.
            counts = ([len(loops) // mpiops.size + 1] * (len(loops) % mpiops.size) +
                      [len(loops) // mpiops.size] * (mpiops.size - (len(loops) % mpiops.size)))
            displs = [0]
            for i in range(1,mpiops.size):
                displs.append(displs[-1] + counts[i])
            ifgres = np.prod(ifgs[0].phase_data.shape)
            counts = [c * ifgres for c in counts]
            displs = [d * ifgres for d in displs]
            mpiops.comm.Gatherv([closure_process, len(process_loops), mpiops.MPI.FLOAT],
                                [closure, counts, displs, mpiops.MPI.FLOAT], root = 0)
            #TODO use Gatherv instead of allocating extra zeros and using Reduce
        else:
            closure = closure_process #if MPI isn't installed there's only one process so this is fine

        total_gb = mpiops.comm.allreduce(ifgs_breach_count_process.nbytes / 1e9, op=mpiops.MPI.SUM)
        log.info("Memory usage due to ifgs_breach_count_process {:2.4f}GB of data".format(total_gb))
        log.debug(f"shape of ifgs_breach_count_process is {ifgs_breach_count_process.shape}")
        log.debug(f"dtype of ifgs_breach_count_process is {ifgs_breach_count_process.dtype}")
        if mpiops.rank == 0:
            ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
        else:
            ifgs_breach_count = None
        if mpiops.MPI_INSTALLED:
            mpiops.comm.Reduce([ifgs_breach_count_process, mpiops.MPI.UINT16_T],
                               [ifgs_breach_count, mpiops.MPI.UINT16_T], op=mpiops.MPI.SUM, root=0)  # global
        else:
            ifgs_breach_count = mpiops.comm.reduce(ifgs_breach_count_process, op=mpiops.sum0_op, root=0)

        log.debug(f"successfully summed phase closure breach array")

    num_occurrences_each_ifg = None
    if mpiops.rank == 0:
        closure = np.moveaxis(closure, 0, -1)
        num_occurrences_each_ifg = _find_num_occurrences_each_ifg(loops, edge_to_indexed_ifgs, n_ifgs)

    return closure, ifgs_breach_count, num_occurrences_each_ifg

def _find_num_occurrences_each_ifg(loops: List[WeightedLoop],
                                   edge_to_indexed_ifgs: Dict[Edge, IndexedIfg],
                                   n_ifgs: int) -> NDArray[(Any, ), UInt16]:
    """find how many times each ifg appears in total in all loops"""
    num_occurrences_each_ifg = np.zeros(shape=n_ifgs, dtype=np.uint16)
    for weighted_loop in loops:
        for signed_edge in weighted_loop.loop:
            indexed_ifg = edge_to_indexed_ifgs[signed_edge.edge]
            ifg_index = indexed_ifg.index
            num_occurrences_each_ifg[ifg_index] += 1
    return num_occurrences_each_ifg

def __compute_ifgs_breach_count(weighted_loop: WeightedLoop,
                                edge_to_indexed_ifgs: Dict[Edge, IndexedIfg], params: dict) \
        -> Tuple[NDArray[(Any, Any), Float32], NDArray[(Any, Any, Any), UInt16]]:
    """Compute summed `closure` of each loop, and compute `ifgs_breach_count` for each pixel."""
    n_ifgs = len(edge_to_indexed_ifgs)
    indexed_ifg = list(edge_to_indexed_ifgs.values())[0]
    ifg = indexed_ifg.IfgPhase
    large_dev_thr = params[C.LARGE_DEV_THR] * np.pi
    use_median = params[C.SUBTRACT_MEDIAN]

    closure = np.zeros(shape=ifg.phase_data.shape, dtype=np.float32)
    # initiate variable for check of unwrapping issues at the same pixels in all loops
    ifgs_breach_count = np.zeros(shape=(ifg.phase_data.shape + (n_ifgs,)), dtype=np.uint16)

    for signed_edge in weighted_loop.loop:
        indexed_ifg = edge_to_indexed_ifgs[signed_edge.edge]
        ifg = indexed_ifg.IfgPhase
        closure += signed_edge.sign * ifg.phase_data
    if use_median:
        closure -= np.nanmedian(closure)  # may be able to drop median

    # this will deal with nans in `closure`, i.e., nans are not selected in indices_breaching_threshold
    indices_breaching_threshold = np.absolute(closure) > large_dev_thr

    for signed_edge in weighted_loop.loop:
        ifg_index = edge_to_indexed_ifgs[signed_edge.edge].index
        # the variable 'ifgs_breach_count' is increased by 1 for that pixel
        # make sure we are not incrementing the nan positions in the closure
        # as we don't know the phase of these pixels and also they were converted to zero before large_dev_thr check
        # Therefore, we leave them out of ifgs_breach_count, i.e., we don't increment their ifgs_breach_count values
        ifgs_breach_count[indices_breaching_threshold, ifg_index] += 1
    return closure, ifgs_breach_count
<|endoftext|>


<commit_before>import clusterpy as _clusterpy
import pysal as ps
import struct

__author__ = "Sergio Rey <sjsrey@gmail.com>"

__ALL__= ['Layer', 'loadArcData', 'importCsvData', 'addRook2Layer', 'addQueen2Layer', 'addArray2Layer' ]

def _importArcData(filename):
    """Creates a new Layer from a shapefile (<file>.shp)

    This function wraps and extends a core clusterPy function to utilize PySAL
    W constructors and dbf readers.

    Parameters
    ==========

    filename: string
              suffix of shapefile (fileName not fileName.shp)

    Returns
    =======
    layer: clusterpy layer instance

    """
    layer = _clusterpy.Layer()
    layer.name = filename.split('/')[-1]
    #print "Loading " + filename + ".dbf"
    dbf = ps.open(filename+".dbf")
    fields = dbf.header
    #data, fields, specs = importDBF(filename + '.dbf')
    data = {}
    #print "Loading " + filename + ".shp"
    if fields[0] != "ID":
        fields = ["ID"] + fields
        for y in range(dbf.n_records):
            data[y] = [y] + dbf.by_row(y)
    else:
        for y in range(dbf.n_records):
            data[y] = dbf.by_row_(y)

    layer.fieldNames = fields
    layer.Y = data
    shpf = filename+".shp"
    layer.shpType = 5
    #print 'pysal reader'
    layer.Wrook = ps.rook_from_shapefile(filename+".shp").neighbors
    layer.Wqueen = ps.queen_from_shapefile(filename+".shp").neighbors
    #print "Done"
    return layer

_clusterpy.importArcData = _importArcData

################# Public functions #######################

def Layer():
    """Provide a clusterpy Layer instance

    Parameters
    ==========

    none

    Returns
    =======

    layer: clusterpy.Layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> l = cp.Layer()
    >>> type(l)
    <type 'instance'>
    >>> l.Wrook
    {}

    """
    return _clusterpy.Layer()

def loadArcData(shapeFileName):
    """
    Handler to use PySAL W and dbf readers in place of clusterpy's

    Parameters
    ==========
    shapeFileName: string
                   filename including .shp extension

    Returns
    =======
    layer: clusterpy layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> import pysal as ps
    >>> shpFile = ps.examples.get_path('columbus.shp')
    >>> columbus = cp.loadArcData(shpFile)
    >>> columbus.Wrook[0]
    [1, 2]
    >>> columbus.Wrook[1]
    [0, 2, 3]
    >>> columbus.fieldNames[0:10]
    ['ID', 'AREA', 'PERIMETER', 'COLUMBUS_', 'COLUMBUS_I', 'POLYID', 'NEIG', 'HOVAL', 'INC', 'CRIME']
    """
    base = shapeFileName.split(".")[0]
    return _clusterpy.importArcData(base)

def importCsvData(filename, layer=None):
    """
    Read a csv file of attributes into a layer

    Parameters
    ==========

    filename: string
              csf file to load

    layer: clusterpy layer instance (default: None)
           if a layer is passed, new attributes, Ws are attached to the layer.
               Otherwise a new layer is created and returned

    Returns
    =======
    layer: clusterpy layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> l = cp.Layer()
    >>> mexico = cp.importCsvData(ps.examples.get_path('mexico.csv'))
    >>> mexico.fieldNames
    ['ID', 'State', 'pcgdp1940', 'pcgdp1950', 'pcgdp1960', 'pcgdp1970', 'pcgdp1980', 'pcgdp1990', 'pcgdp2000', 'hanson03', 'hanson98', 'esquivel99', 'inegi', 'inegi2']

    Notes
    =====

    This assumes the csv file is organized with records on the rows and attributes on the columns

    """

    if not layer:
        layer = Layer()
    csv = ps.open(filename,'r')
    fields = csv.header
    data = {}
    if fields[0] != "ID":
        fields = ["ID"] + fields
        for i, rec in enumerate(csv.data):
            data[i] = [i] + csv.by_row(i)
    else:
        for i, rec in enumerate(csv.data):
            data[i] = csv.by_row(i)
    layer.Y = data
    layer.fieldNames = fields
    return layer

def addGal2Layer(galfile, layer, contiguity='rook'):
    """
    Attach an adjacency object to a layer

    Parameters
    ==========
    galfile: string
             galfile

    layer: clusterpy layer

    contiguity: type of contguity ['rook'|'queen']

    Returns
    =======
    None

    Examples
    ========
    >>> import pysal as ps
    >>> import pysal.contrib.clusterpy as cp
    >>> csvfile = ps.examples.get_path('mexico.csv')
    >>> galfile = ps.examples.get_path('mexico.gal')
    >>> mexico = cp.importCsvData(csvfile)
    >>> cp.addRook2Layer(galfile, mexico)
    >>> mexico.Wrook[0]
    [31, 13]

    """
    gal = ps.open(galfile).read().neighbors
    w = {}
    for key in gal:
        w[int(key)] =  map(int, gal[key])

    if contiguity.upper()== "ROOK":
        layer.Wrook = w
    elif contiguity.upper() == "QUEEN":
        layer.Wqueen = w
    else:
        print 'Unsupported contiguity type: ', contiguity

def addRook2Layer(galfile, layer):
    """
    User function for adding rook to layer

    See addGal2Layer
    """
    addGal2Layer(galfile, layer)

def addQueen2Layer(galfile, layer):
    """
    User function for adding queen to layer

    See addGal2Layer
    """
    addGal2Layer(galfile, layer, contiguity='QUEEN')

def addArray2Layer(array, layer, names=None):
    """
    Add a numpy array to a clusterpy layer

    Parameters
    ==========
    array: nd-array
           nxk with n observations on k attributes

    layer: clusterpy layer object

    names: list
           k strings for attribute names

    Returns
    =======
    None

    Examples
    ========
    # Note this will report as fail since clusterpy prints 'Adding variables
    # for each variable added. But the variables will be correctly added
    >>> #import pysal as ps
    >>> #import pysal.contrib.clusterpy as cp
    >>> #import numpy as np
    >>> #uscsv = ps.examples.get_path("usjoin.csv")
    >>> #f = ps.open(uscsv)
    >>> #pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)]).T
    >>> #usy = cp.Layer()
    >>> #names = ["Y_%d"%v for v in range(1929,2010)]
    >>> #cp.addArray2Layer(pci, usy, names)

    """
    n,k = array.shape
    if not names:
        names = ["X_%d"% v for v in range(k)]

    for j,name in enumerate(names):
        v = {}
        for i in xrange(n):
            v[i] = array[i,j]
        layer.addVariable([name], v)

def addW2Layer(w, layer, contiguity='rook'):
    '''
    Attach a contiguity PySAL W object to a layer

    NOTE: given clusterpy's requirements, this method only extracts the
    `neighbors` dictionary.

    ...

    Parameters
    ----------
    w         : ps.W
                PySAL weights object
    layer     : clusterpy.Layer
                Layer to attach the weights to
    contiguity: str ['rook'|'queen']
                Type of contguity expressed in `w`

    Returns
    -------
    None

    Example
    -------
    '''
    if contiguity.upper()== "ROOK":
        layer.Wrook = w.neighbors
    elif contiguity.upper() == "QUEEN":
        layer.Wqueen = w.neighbors
    else:
        print 'Unsupported contiguity type: ', contiguity
    return None

if __name__ == '__main__':

    import numpy as np

    w = ps.queen_from_shapefile(ps.examples.get_path('columbus.shp'))
    db = ps.open(ps.examples.get_path('columbus.dbf'))
    vars = ['CRIME',  'HOVAL']
    x = np.array([db.by_col(i) for i in vars]).T
    layer = Layer()
    _ = addArray2Layer(x, layer, names=vars)
    _ = addW2Layer(w, layer)
    layer.cluster('arisel', ['CRIME',  'CONSTANT'], 2, dissolve=0, std=0)
    '''
    columbus = loadArcData(ps.examples.get_path('columbus.shp'))
    n = len(columbus.Wqueen)
    columbus.dataOperation("CONSTANT = 1")
    np.random.seed(12345)
    columbus.cluster('maxpTabu', ['CRIME',  'CONSTANT'], threshold=4, dissolve=0, std=0)
    '''
    #np.random.seed(12345)
    #columbus.cluster('arisel', ['CRIME'], 5, wType='rook', inits=10, dissolve=0)

<commit_msg>Adding docstring example for addW2Layer<commit_after>import clusterpy as _clusterpy
import pysal as ps
import struct

__author__ = "Sergio Rey <sjsrey@gmail.com>"

__ALL__= ['Layer', 'loadArcData', 'importCsvData', 'addRook2Layer', 'addQueen2Layer', 'addArray2Layer', 'addW2Layer']

def _importArcData(filename):
    """Creates a new Layer from a shapefile (<file>.shp)

    This function wraps and extends a core clusterPy function to utilize PySAL
    W constructors and dbf readers.

    Parameters
    ==========

    filename: string
              suffix of shapefile (fileName not fileName.shp)

    Returns
    =======
    layer: clusterpy layer instance

    """
    layer = _clusterpy.Layer()
    layer.name = filename.split('/')[-1]
    #print "Loading " + filename + ".dbf"
    dbf = ps.open(filename+".dbf")
    fields = dbf.header
    #data, fields, specs = importDBF(filename + '.dbf')
    data = {}
    #print "Loading " + filename + ".shp"
    if fields[0] != "ID":
        fields = ["ID"] + fields
        for y in range(dbf.n_records):
            data[y] = [y] + dbf.by_row(y)
    else:
        for y in range(dbf.n_records):
            data[y] = dbf.by_row_(y)

    layer.fieldNames = fields
    layer.Y = data
    shpf = filename+".shp"
    layer.shpType = 5
    #print 'pysal reader'
    layer.Wrook = ps.rook_from_shapefile(filename+".shp").neighbors
    layer.Wqueen = ps.queen_from_shapefile(filename+".shp").neighbors
    #print "Done"
    return layer

_clusterpy.importArcData = _importArcData

################# Public functions #######################

def Layer():
    """Provide a clusterpy Layer instance

    Parameters
    ==========

    none

    Returns
    =======

    layer: clusterpy.Layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> l = cp.Layer()
    >>> type(l)
    <type 'instance'>
    >>> l.Wrook
    {}

    """
    return _clusterpy.Layer()

def loadArcData(shapeFileName):
    """
    Handler to use PySAL W and dbf readers in place of clusterpy's

    Parameters
    ==========
    shapeFileName: string
                   filename including .shp extension

    Returns
    =======
    layer: clusterpy layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> import pysal as ps
    >>> shpFile = ps.examples.get_path('columbus.shp')
    >>> columbus = cp.loadArcData(shpFile)
    >>> columbus.Wrook[0]
    [1, 2]
    >>> columbus.Wrook[1]
    [0, 2, 3]
    >>> columbus.fieldNames[0:10]
    ['ID', 'AREA', 'PERIMETER', 'COLUMBUS_', 'COLUMBUS_I', 'POLYID', 'NEIG', 'HOVAL', 'INC', 'CRIME']
    """
    base = shapeFileName.split(".")[0]
    return _clusterpy.importArcData(base)

def importCsvData(filename, layer=None):
    """
    Read a csv file of attributes into a layer

    Parameters
    ==========

    filename: string
              csf file to load

    layer: clusterpy layer instance (default: None)
           if a layer is passed, new attributes, Ws are attached to the layer.
               Otherwise a new layer is created and returned

    Returns
    =======
    layer: clusterpy layer instance

    Examples
    ========
    >>> import pysal.contrib.clusterpy as cp
    >>> l = cp.Layer()
    >>> mexico = cp.importCsvData(ps.examples.get_path('mexico.csv'))
    >>> mexico.fieldNames
    ['ID', 'State', 'pcgdp1940', 'pcgdp1950', 'pcgdp1960', 'pcgdp1970', 'pcgdp1980', 'pcgdp1990', 'pcgdp2000', 'hanson03', 'hanson98', 'esquivel99', 'inegi', 'inegi2']

    Notes
    =====

    This assumes the csv file is organized with records on the rows and attributes on the columns

    """

    if not layer:
        layer = Layer()
    csv = ps.open(filename,'r')
    fields = csv.header
    data = {}
    if fields[0] != "ID":
        fields = ["ID"] + fields
        for i, rec in enumerate(csv.data):
            data[i] = [i] + csv.by_row(i)
    else:
        for i, rec in enumerate(csv.data):
            data[i] = csv.by_row(i)
    layer.Y = data
    layer.fieldNames = fields
    return layer

def addGal2Layer(galfile, layer, contiguity='rook'):
    """
    Attach an adjacency object to a layer

    Parameters
    ==========
    galfile: string
             galfile

    layer: clusterpy layer

    contiguity: type of contguity ['rook'|'queen']

    Returns
    =======
    None

    Examples
    ========
    >>> import pysal as ps
    >>> import pysal.contrib.clusterpy as cp
    >>> csvfile = ps.examples.get_path('mexico.csv')
    >>> galfile = ps.examples.get_path('mexico.gal')
    >>> mexico = cp.importCsvData(csvfile)
    >>> cp.addRook2Layer(galfile, mexico)
    >>> mexico.Wrook[0]
    [31, 13]

    """
    gal = ps.open(galfile).read().neighbors
    w = {}
    for key in gal:
        w[int(key)] =  map(int, gal[key])

    if contiguity.upper()== "ROOK":
        layer.Wrook = w
    elif contiguity.upper() == "QUEEN":
        layer.Wqueen = w
    else:
        print 'Unsupported contiguity type: ', contiguity

def addRook2Layer(galfile, layer):
    """
    User function for adding rook to layer

    See addGal2Layer
    """
    addGal2Layer(galfile, layer)

def addQueen2Layer(galfile, layer):
    """
    User function for adding queen to layer

    See addGal2Layer
    """
    addGal2Layer(galfile, layer, contiguity='QUEEN')

def addArray2Layer(array, layer, names=None):
    """
    Add a numpy array to a clusterpy layer

    Parameters
    ==========
    array: nd-array
           nxk with n observations on k attributes

    layer: clusterpy layer object

    names: list
           k strings for attribute names

    Returns
    =======
    None

    Examples
    ========
    # Note this will report as fail since clusterpy prints 'Adding variables
    # for each variable added. But the variables will be correctly added
    >>> #import pysal as ps
    >>> #import pysal.contrib.clusterpy as cp
    >>> #import numpy as np
    >>> #uscsv = ps.examples.get_path("usjoin.csv")
    >>> #f = ps.open(uscsv)
    >>> #pci = np.array([f.by_col[str(y)] for y in range(1929, 2010)]).T
    >>> #usy = cp.Layer()
    >>> #names = ["Y_%d"%v for v in range(1929,2010)]
    >>> #cp.addArray2Layer(pci, usy, names)

    """
    n,k = array.shape
    if not names:
        names = ["X_%d"% v for v in range(k)]

    for j,name in enumerate(names):
        v = {}
        for i in xrange(n):
            v[i] = array[i,j]
        layer.addVariable([name], v)

def addW2Layer(w, layer, contiguity='rook'):
    '''
    Attach a contiguity PySAL W object to a layer

    NOTE: given clusterpy's requirements, this method only extracts the
    `neighbors` dictionary.

    ...

    Parameters
    ----------
    w         : ps.W
                PySAL weights object
    layer     : clusterpy.Layer
                Layer to attach the weights to
    contiguity: str ['rook'|'queen']
                Type of contguity expressed in `w`

    Returns
    -------
    None

    Example
    -------
    >>> import pysal as ps
    >>> import pysal.contrib.clusterpy as cp
    >>> w = ps.queen_from_shapefile(ps.examples.get_path('columbus.shp'))
    >>> layer = cp.Layer()
    >>> cp.addW2Layer(w, layer, contiguity='queen')
    >>> layer.Wqueen[0]
    [1, 2]
    '''
    if contiguity.upper()== "ROOK":
        layer.Wrook = w.neighbors
    elif contiguity.upper() == "QUEEN":
        layer.Wqueen = w.neighbors
    else:
        print 'Unsupported contiguity type: ', contiguity
    return None

if __name__ == '__main__':

    import numpy as np

    w = ps.queen_from_shapefile(ps.examples.get_path('columbus.shp'))
    db = ps.open(ps.examples.get_path('columbus.dbf'))
    vars = ['CRIME',  'HOVAL']
    x = np.array([db.by_col(i) for i in vars]).T
    layer = Layer()
    _ = addArray2Layer(x, layer, names=vars)
    _ = addW2Layer(w, layer)
    layer.cluster('arisel', ['CRIME',  'CONSTANT'], 2, dissolve=0, std=0)
    '''
    columbus = loadArcData(ps.examples.get_path('columbus.shp'))
    n = len(columbus.Wqueen)
    columbus.dataOperation("CONSTANT = 1")
    np.random.seed(12345)
    columbus.cluster('maxpTabu', ['CRIME',  'CONSTANT'], threshold=4, dissolve=0, std=0)
    '''
    #np.random.seed(12345)
    #columbus.cluster('arisel', ['CRIME'], 5, wType='rook', inits=10, dissolve=0)

<|endoftext|>


<commit_before>from django.db import models
import datetime
from django.utils import timezone

# Create your models here.

class Inventory(models.Model):
	modelNum = models.CharField(max_length=200)
	def __unicode__(self):
		return self.modelNum

class Serial(models.Model):
	inventory = models.ForeignKey(Inventory)
	serialNum = models.CharField(max_length=200)
	def __unicode__(self):
		return self.serialNum

class Description(models.Model):
	inventory = models.ForeignKey(Inventory)
	descript = models.CharField(max_length=255)
	def __unicode__(self):
		return self.descript

class Location(models.Model):
	inventory = models.ForeignKey(Inventory)
	room = models.CharField(max_length=3)
	building = models.CharField(max_length=2)
	def __unicode__(self):
		return self.building + self.room<commit_msg>Heading in the right direction<commit_after>from django.db import models
import datetime
from django.utils import timezone

# Create your models here.

class Equipment(models.Model):
	model_num = models.IntegerField()
	serial_num = models.IntegerField()
	equip_type = models.ForeignField()
	location = models.ForeignField()
	description = models.TextField()

	def __unicode__(self):
		return unicode(model_num)

class Location(models.Model):
	room = models.CharField(max_length=4)
	building = models.CharField(max_length=2)
	def __unicode__(self):
		return unicode(self.building + self.room)
<|endoftext|>


<commit_before>##########################################################################
#
#  Copyright (c) 2011-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import unittest
import imath

import IECore
import Gaffer
import GafferTest

class ApplicationRootTest( GafferTest.TestCase ) :

	__defaultPreferencesFile = os.path.expanduser( "~/gaffer/startup/testApp/preferences.py" )

	class testApp( Gaffer.Application ) :

		def __init__( self ) :

			Gaffer.Application.__init__( self )

	def testPreferences( self ) :

		application = ApplicationRootTest.testApp()
		applicationRoot = application.root()

		p = applicationRoot["preferences"]
		self.assertIsInstance( p, Gaffer.Preferences )

		p["category1"] = Gaffer.Plug()
		p["category1"]["i"] = Gaffer.IntPlug( defaultValue = 2 )

		p["category2"] = Gaffer.Plug()
		p["category2"]["s"] = Gaffer.StringPlug( defaultValue = "apples" )
		p["category2"]["v"] = Gaffer.V3fPlug( defaultValue = imath.V3f( 1, 0, 0 ) )

		p["category1"]["i"].setValue( 10 )
		p["category2"]["s"].setValue( "oranges" )
		p["category2"]["v"].setValue( imath.V3f( 2, 3, 4 ) )

		self.assertFalse( os.path.exists( self.__defaultPreferencesFile ) )
		applicationRoot.savePreferences()
		self.assertTrue( os.path.exists( self.__defaultPreferencesFile ) )

		fileName = os.path.join( self.temporaryDirectory(), "testPreferences.gfr" )

		self.assertFalse( os.path.exists( fileName ) )
		applicationRoot.savePreferences( fileName )
		self.assertTrue( os.path.exists( fileName ) )

		p["category1"]["i"].setValue( 1 )
		p["category2"]["s"].setValue( "beef" )
		p["category2"]["v"].setValue( imath.V3f( -10 ) )

		executionContext = { "application" : application }
		exec(
			compile( open( fileName ).read(), fileName, "exec" ),
			executionContext, executionContext
		)

		self.assertEqual( p["category1"]["i"].getValue(), 10 )
		self.assertEqual( p["category2"]["s"].getValue(), "oranges" )
		self.assertEqual( p["category2"]["v"].getValue(), imath.V3f( 2, 3, 4 ) )

	def testPreferencesPermissionsErrors( self ) :

		a = Gaffer.ApplicationRoot( "testApp" )

		fileName = os.path.join( self.temporaryDirectory(), "testPreferences.gfr" )
		a.savePreferences( fileName )
		os.chmod( fileName, 0 )
		self.assertRaises( RuntimeError, a.savePreferences, fileName )

	def testPreferencesLocation( self ) :

		a = Gaffer.ApplicationRoot( "testApp" )

		self.assertEqual( a.preferencesLocation(), os.path.dirname( self.__defaultPreferencesFile ) )
		self.assertTrue( os.path.isdir( a.preferencesLocation() ) )

	def testClipboard( self ) :

		a = Gaffer.ApplicationRoot()

		d = []
		def f( app ) :

			self.assertTrue( app.isSame( a ) )
			d.append( app.getClipboardContents() )

		c = a.clipboardContentsChangedSignal().connect( f, scoped = True )

		self.assertEqual( len( d ), 0 )
		self.assertEqual( a.getClipboardContents(), None )

		a.setClipboardContents( IECore.IntData( 10 ) )
		self.assertEqual( len( d ), 1 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 10 ) )

		a.setClipboardContents( IECore.IntData( 20 ) )
		self.assertEqual( len( d ), 2 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 20 ) )

		a.setClipboardContents( IECore.IntData( 20 ) )
		self.assertEqual( len( d ), 2 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 20 ) )

	def testScriptContainer( self ) :

		a = Gaffer.ApplicationRoot()
		self.assertIsInstance( a["scripts"], Gaffer.ScriptContainer )

	def tearDown( self ) :

		GafferTest.TestCase.tearDown( self )

		if os.path.exists( self.__defaultPreferencesFile ) :
			os.remove( self.__defaultPreferencesFile )

if __name__ == "__main__":
	unittest.main()
<commit_msg>ApplicationRootTest : Windows needs write access to delete the preference file.<commit_after>##########################################################################
#
#  Copyright (c) 2011-2013, Image Engine Design Inc. All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions are
#  met:
#
#      * Redistributions of source code must retain the above
#        copyright notice, this list of conditions and the following
#        disclaimer.
#
#      * Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided with
#        the distribution.
#
#      * Neither the name of John Haddon nor the names of
#        any other contributors to this software may be used to endorse or
#        promote products derived from this software without specific prior
#        written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
#  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
#  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
#  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
#  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
#  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##########################################################################

import os
import stat
import unittest
import imath

import IECore
import Gaffer
import GafferTest

class ApplicationRootTest( GafferTest.TestCase ) :

	__defaultPreferencesFile = os.path.expanduser( "~/gaffer/startup/testApp/preferences.py" )

	class testApp( Gaffer.Application ) :

		def __init__( self ) :

			Gaffer.Application.__init__( self )

	def testPreferences( self ) :

		application = ApplicationRootTest.testApp()
		applicationRoot = application.root()

		p = applicationRoot["preferences"]
		self.assertIsInstance( p, Gaffer.Preferences )

		p["category1"] = Gaffer.Plug()
		p["category1"]["i"] = Gaffer.IntPlug( defaultValue = 2 )

		p["category2"] = Gaffer.Plug()
		p["category2"]["s"] = Gaffer.StringPlug( defaultValue = "apples" )
		p["category2"]["v"] = Gaffer.V3fPlug( defaultValue = imath.V3f( 1, 0, 0 ) )

		p["category1"]["i"].setValue( 10 )
		p["category2"]["s"].setValue( "oranges" )
		p["category2"]["v"].setValue( imath.V3f( 2, 3, 4 ) )

		self.assertFalse( os.path.exists( self.__defaultPreferencesFile ) )
		applicationRoot.savePreferences()
		self.assertTrue( os.path.exists( self.__defaultPreferencesFile ) )

		fileName = os.path.join( self.temporaryDirectory(), "testPreferences.gfr" )

		self.assertFalse( os.path.exists( fileName ) )
		applicationRoot.savePreferences( fileName )
		self.assertTrue( os.path.exists( fileName ) )

		p["category1"]["i"].setValue( 1 )
		p["category2"]["s"].setValue( "beef" )
		p["category2"]["v"].setValue( imath.V3f( -10 ) )

		executionContext = { "application" : application }
		exec(
			compile( open( fileName ).read(), fileName, "exec" ),
			executionContext, executionContext
		)

		self.assertEqual( p["category1"]["i"].getValue(), 10 )
		self.assertEqual( p["category2"]["s"].getValue(), "oranges" )
		self.assertEqual( p["category2"]["v"].getValue(), imath.V3f( 2, 3, 4 ) )

	def testPreferencesPermissionsErrors( self ) :

		a = Gaffer.ApplicationRoot( "testApp" )

		fileName = os.path.join( self.temporaryDirectory(), "testPreferences.gfr" )
		a.savePreferences( fileName )
		os.chmod( fileName, 0 )
		self.assertRaises( RuntimeError, a.savePreferences, fileName )
		os.chmod( fileName, stat.S_IWRITE)

	def testPreferencesLocation( self ) :

		a = Gaffer.ApplicationRoot( "testApp" )

		self.assertEqual( a.preferencesLocation(), os.path.dirname( self.__defaultPreferencesFile ) )
		self.assertTrue( os.path.isdir( a.preferencesLocation() ) )

	def testClipboard( self ) :

		a = Gaffer.ApplicationRoot()

		d = []
		def f( app ) :

			self.assertTrue( app.isSame( a ) )
			d.append( app.getClipboardContents() )

		c = a.clipboardContentsChangedSignal().connect( f, scoped = True )

		self.assertEqual( len( d ), 0 )
		self.assertEqual( a.getClipboardContents(), None )

		a.setClipboardContents( IECore.IntData( 10 ) )
		self.assertEqual( len( d ), 1 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 10 ) )

		a.setClipboardContents( IECore.IntData( 20 ) )
		self.assertEqual( len( d ), 2 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 20 ) )

		a.setClipboardContents( IECore.IntData( 20 ) )
		self.assertEqual( len( d ), 2 )
		self.assertEqual( a.getClipboardContents(), IECore.IntData( 20 ) )

	def testScriptContainer( self ) :

		a = Gaffer.ApplicationRoot()
		self.assertIsInstance( a["scripts"], Gaffer.ScriptContainer )

	def tearDown( self ) :

		GafferTest.TestCase.tearDown( self )

		if os.path.exists( self.__defaultPreferencesFile ) :
			os.remove( self.__defaultPreferencesFile )

if __name__ == "__main__":
	unittest.main()
<|endoftext|>


<commit_before>'''

INCOMPLETE APPROACH

See this issue: https://github.com/codefordc/housing-insights/issues/152
for some comments on latest status and next steps.

The core method will need to be renamed to `get_data` and should have the same
method signature as others. the current opendata.py file is a good model to
look at to provide consistent approach.

'''

import os
import sys

sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             os.pardir,os.pardir)))

from housinginsights.sources.base import BaseApiConn

# secrets.json, yo
census_key = os.environ.get("CENSUS_KEY")

class CensusApiConn(BaseApiConn):
    """
    Census API connector, confined to ACS5 2015 for now.

    """
    BASEURL = 'http://api.census.gov/data'

    def __init__(self, arg):
        super(CensusApiConn, self).__init__(CensusApiConn.BASEURL)
        self.arg = arg

    def getacs5(self):
        params = {'key': census_key, 'get': 'B01003_001E,B25057_001E,B25058_001E,B25059_001E', 'for': 'tract:*', 'in': 'state:11'}
        result = self.get('/2015/acs5', params=params)
        if result.status_code != 200:
            err = "An error occurred during request: status {0}"
            raise Exception(err.format(result.status_code))
        else:
            data = result
            print(data.text)

CensusApiConn('fakearg').getacs5()
<commit_msg>152 - census data api file<commit_after>import sys, os
import json

#This first if __name__ is needed if this file is test-run from the command line.
# path.append needs to happen before the first from import statement
if __name__ == '__main__':
    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),os.pardir,os.pardir)))

import logging
from housinginsights.sources.base import BaseApiConn

class CensusApiConn(BaseApiConn):
    """

    """
    def __init__(self, proxies=None):
        #baseurl not actually used since we need the _urls property to hold many urls.
        #Needed to get call to super() to work correctly. TODO refactor so this is optional.
        baseurl = 'http://api.census.gov/data/'
        super(CensusApiConn, self).__init__(baseurl, proxies=None)
        secretsFileName =  os.path.abspath(os.path.join(os.path.dirname(__file__),
                                               os.pardir,"secrets.json"))
        self.api_keys = json.loads(open(secretsFileName).read())
        self.census_api_key = self.api_keys['census']['api_key']

        self._available_unique_data_ids = [
                        "acs5_2015",
                        "acs5_2015_moe"
                        ]

        self._urls =	{
                        "acs5_2015"     :"2015/acs5",
                        "acs5_2015_moe" :"2015/acs5"
                    	}
        self._fields = {
                        "acs5_2015"     : "NAME,B01003_001E,B25057_001E,B25058_001E,B25059_001E",
                        "acs5_2015_moe" : "NAME,B01003_001M,B25057_001M,B25058_001M,B25059_001M"
                       }

    def get_data(self, unique_data_ids=None, sample=False, output_type = 'csv', **kwargs):
        '''
        Gets all data sources associated with this ApiConn class.

        sample does not apply to this apiconn object because api doesn't let us request only some data.
        '''
        if unique_data_ids == None:
            unique_data_ids = self._available_unique_data_ids

        for u in unique_data_ids:
            if (u not in self._available_unique_data_ids):
                #TODO Change this error type to a more specific one that should be handled by calling function inproduction
                #We will want the calling function to continue with other data sources instead of erroring out.
                logging.info("  The unique_data_id '{}' is not supported by the CensusApiConn".format(u))
            else:
                result = self.get(self._urls[u], params={'key':self.census_api_key, 'get':self._fields[u], 'for': 'tract:*', 'in': 'state:11'})

                if result.status_code != 200:
                    err = "An error occurred during request: status {0}"
                    #TODO change this error type to be handleable by caller
                    raise Exception(err.format(result.status_code))

                content = result.text

                if output_type == 'stdout':
                    print(content)

                elif output_type == 'csv':
                    self.directly_to_file(content, self.output_paths[u])

                #Can't yield content if we get multiple sources at once
                if len(unique_data_ids) == 1:
                    return content

if __name__ == '__main__':

    # Pushes everything from the logger to the command line output as well.
    log_path = os.path.abspath(os.path.join(os.path.dirname(__file__),os.pardir,os.pardir,"logs","sources.log"))
    logging.basicConfig(filename=log_path, level=logging.DEBUG)
    logging.getLogger().addHandler(logging.StreamHandler())

    api_conn = CensusApiConn()
    unique_data_ids = None #['crime_2013']
    api_conn.get_data(unique_data_ids)
<|endoftext|>


<commit_before>from numpy import arange, ndarray, argmax, unravel_index, asarray

from thunder.rdds.images import Images
from thunder.utils.common import checkparams

class Register(object):

    def __new__(cls, method="crosscorr"):

        checkparams(method, ["crosscorr"])

        if method == "crosscorr":
            return super(Register, cls).__new__(CrossCorr)
        else:
            raise Exception('Registration method not recognized')

    def get_transform(self, im, ref):
        raise NotImplementedError

    def apply_transform(self, im, transform):
        raise NotImplementedError

    def setFilter(self, filter='median', param=2):
        """
        Set a filter to apply to images before registration.

        The filtering will be applied to both the reference and
        image to compute the transformation parameters, but the filtering
        will not be applied to the images themselves.

        Parameters
        ----------

        filter : str, optional, default = 'median'
            Which filter to use (options are 'median' and 'gaussian')

        param : int, optional, default = 2
            Parameter to provide to filtering function (e.g. size for median filter)

        See also
        --------
        Images.medianFilter : apply median filter to images
        Images.gaussianFilter : apply gaussian filter to images

        """

        checkparams(filter, ['median', 'gaussian'])

        if filter == 'median':
            from scipy.ndimage.filters import median_filter
            self._filter = lambda x: median_filter(x, param)

        if filter == 'gaussian':
            from scipy.ndimage.filters import gaussian_filter
            self._filter = lambda x: gaussian_filter(x, param)

        return self

    def filter(self, im):
        """
        Apply filtering, and if not set, return image unchanged
        """

        if hasattr(self, '_filter'):
            return self._filter(im)
        else:
            return im

    @staticmethod
    def reference(images, method='mean', startidx=None, stopidx=None):
        """
        Compute a reference image for use in registration.

        Parameters
        ----------
        method : str, optional, default = 'mean'
            How to compute the reference

        startidx : int, optional, default = None
            Starting index if computing a mean over a specified range

        stopidx : int, optional, default = None
            Stopping index if computing a mean over a specified range

        """

        # TODO easy option for using the mean of the middle n images
        # TODO fix inclusive behavior to match e.g. image loading

        checkparams(method, ['mean'])

        if method == 'mean':
            if startidx is not None and stopidx is not None:
                range = lambda x: startidx <= x < stopidx
                n = stopidx - startidx
                ref = images.filterOnKeys(range)
            else:
                ref = images
                n = images.nimages
            refval = ref.sum() / (1.0 * n)
            return refval.astype(images.dtype)

    @staticmethod
    def _apply_vol(vol, func):
        """
        Apply a function to an image, or a volume (plane-by-plane).
        """

        if vol.ndim == 2:
            return func(vol)
        else:
            vol.setflags(write=True)
            for z in arange(0, vol.shape[2]):
                vol[:, :, z] = func(vol[:, :, z])
            return vol

    @staticmethod
    def _check_reference(images, reference):
        """
        Check the dimensions and type of a reference (relative to an Images object),
        as well as check that the images / volumes themselves are either 2D or 3D
        """

        if isinstance(reference, ndarray):
            if reference.shape != images.dims.count:
                raise Exception('Dimensions of reference %s do not match dimensions of data %s' %
                                (reference.shape, images.dims.count))
            if len(images.dims.count) not in set([2, 3]):
                raise Exception('Number of image dimensions %s must be 2 or 3' % (len(images.dims.count)))
        else:
            raise Exception('Reference must be an array')

    def estimate(self, images, reference):
        """
        Estimate registration parameters on a collection of images / volumes.

        Will return a list of registration parameters to the driver, rather
        than the registered images / volumes themselves.

        Parameters
        ----------
        images : Images
            An Images object containing the images / volumes to estimate registration for

        reference : ndarray
            The reference image / volume to estimate registration against

        Returns
        -------
        params : list
            Registration parameters, one per image. Will be returned as a list of key-value pairs,
            where the key is the same key used to identify each image / volume in the Images object,
            and the value is a list of registration parameters (in whatever format provided by
            the registration function; e.g. for CrossCorr will return a list of deltas in x and y)

        See also
        --------
        Register.transform : apply transformations
        """

        if not (isinstance(images, Images)):
            raise Exception('Input data must be Images or a subclass')

        self._check_reference(images, reference)

        # apply filtering to reference if defined
        if hasattr(self, '_filter'):
            reference = self._apply_vol(reference.copy(), self.filter)

        # broadcast the reference (a potentially very large array)
        reference_bc = images.rdd.context.broadcast(reference)

        # estimate the transform parameters on an image / volume
        def params(im, ref):
            if im.ndim == 2:
                return self.get_transform(self.filter(im), ref.value)
            else:
                t = []
                for z in arange(0, im.shape[2]):
                    t.append(self.get_transform(self.filter(im[:, :, z]), ref.value[:, :, z]))
            return t

        from thunder import Series
        return Series(images.rdd.mapValues(lambda x: params(x, reference_bc)))

    def transform(self, images, reference):
        """
        Apply registration to a collection of images / volumes.

        Parameters
        ----------
        images : Images
            An Images object containing the images / volumes to apply registration to

        reference : ndarray
            The reference image / volume to register against
        """

        if not (isinstance(images, Images)):
            raise Exception('Input data must be Images or a subclass')

        self._check_reference(images, reference)

        # apply filtering to reference if defined
        if hasattr(self, '_filter'):
            reference = self._apply_vol(reference, self.filter)

        # broadcast the reference (a potentially very large array)
        reference_bc = images.rdd.context.broadcast(reference)

        # compute and apply transformation on an image / volume
        def register(im, ref):
            if im.ndim == 2:
                t = self.get_transform(self.filter(im), ref.value)
                return self.apply_transform(im, t)
            else:
                im.setflags(write=True)
                for z in arange(0, im.shape[2]):
                    t = self.get_transform(self.filter(im[:, :, z]), ref.value[:, :, z])
                    im[:, :, z] = self.apply_transform(im[:, :, z], t)
                return im

        # return the transformed volumes
        newrdd = images.rdd.mapValues(lambda x: register(x, reference_bc))
        return Images(newrdd).__finalize__(images)

class CrossCorr(Register):
    """
    Perform affine (translation) registration using cross-correlation
    """

    def get_transform(self, im, ref):

        from numpy.fft import fft2, ifft2

        fref = fft2(ref)
        fim = fft2(im)
        c = abs(ifft2((fim * fref.conjugate())))
        d0, d1 = unravel_index(argmax(c), c.shape)
        if d0 > im.shape[0] // 2:
            d0 -= im.shape[0]
        if d1 > im.shape[1] // 2:
            d1 -= im.shape[1]

        return [d0, d1]

    def apply_transform(self, im, transform):

        from scipy.ndimage.interpolation import shift
        return shift(im, map(lambda x: -x, transform), mode='nearest')<commit_msg>Significant refactoring of registration<commit_after>from numpy import ndarray

from thunder.rdds.images import Images
from thunder.utils.common import checkparams

class Register(object):
    """
    Class for constructing registration algorithms.

    Construct a registration algorthm by specifying its name,
    and passing any additional keyword arguments. The algorithm
    can then be used to fit registration parameters on an Images object.

    Parameters
    ----------
    method : string
        A registration method, options include 'crosscorr' and 'planarcrosscorr'
    """

    def __new__(cls, method, **kwargs):

        from thunder.imgprocessing.regmethods.crosscorr import CrossCorr, PlanarCrossCorr

        REGMETHODS = {
            'crosscorr': CrossCorr,
            'planarcrosscorr': PlanarCrossCorr
        }

        checkparams(method, REGMETHODS.keys())

        return REGMETHODS[method](kwargs)

    @staticmethod
    def load(file):
        """
        Load a registration model from a file specified using JSON.

        See also
        --------
        RegisterModel.save : specification for saving registration models
        RegisterModel.load : specifications for loading registration models
        """

        return RegisterModel.load(file)

class RegisterMethod(object):
    """
    Base class for registration methods
    """

    def __init__(self, *args, **kwargs):
        pass

    def getTransform(self, im, ref):
        raise NotImplementedError

    @staticmethod
    def reference(images, startidx=None, stopidx=None):
        """
        Compute a reference image /volume for use in registration.

        This default method computes the reference as the mean of a
        subset of frames. Alternative registration algorithms
        may wish to override this method.

        Parameters
        ----------
        startidx : int, optional, default = None
            Starting index if computing a mean over a specified range

        stopidx : int, optional, default = None
            Stopping index if computing a mean over a specified range

        Returns
        -------
        refval : ndarray
            The reference image / volume

        """

        # TODO easy option for using the mean of the middle n images
        # TODO fix inclusive behavior to match e.g. image loading

        if startidx is not None and stopidx is not None:
            range = lambda x: startidx <= x < stopidx
            n = stopidx - startidx
            ref = images.filterOnKeys(range)
        else:
            ref = images
            n = images.nimages
        refval = ref.sum() / (1.0 * n)
        return refval.astype(images.dtype)

    @staticmethod
    def checkReference(images, reference):
        """
        Check the dimensions and type of a reference relative to an Images object.

        This default method ensures that the reference and images have the same dimensions.
        Alternative registration procedures with more complex references may
        wish to override this method.

        Parameters
        ----------
        images : Images
            An Images object containg the image / volumes to check reference against

        reference : ndarray
            The reference image / volume
        """

        if isinstance(reference, ndarray):
            if reference.shape != images.dims.count:
                raise Exception('Dimensions of reference %s do not match dimensions of data %s' %
                                (reference.shape, images.dims.count))
        else:
            raise Exception('Reference must be an array')

    def fit(self, images, ref=None):
        """
        Compute registration parameters on a collection of images / volumes.

        Will return the estimated registration parameters to the driver in the form
        of a RegisterModel, which can then be used to transform Images data.

        Parameters
        ----------
        images : Images
            An Images object with the images / volumes to estimate registration for.

        ref : ndarray, optional, default = None
            The reference image / volume to estimate registration against. Can safely pass None
            for methods that do not require or use a reference.

        Returns
        -------
        model : RegisterModel
            Registration params as a model that can be used to transform an Images object.

        See also
        --------
        RegisterModel : model for applying transformations
        """

        if not (isinstance(images, Images)):
            raise Exception('Input data must be Images or a subclass')

        if len(images.dims.count) not in set([2, 3]):
                raise Exception('Number of image dimensions %s must be 2 or 3' % (len(images.dims.count)))

        if ref is not None:
            self.checkReference(images, ref)

        # broadcast the reference
        ref_bc = images.rdd.context.broadcast(ref)

        # compute the transformations
        transformations = images.rdd.mapValues(lambda im: self.getTransform(im, ref_bc.value)).collectAsMap()

        # construct the model
        regmethod = self.__class__.__name__
        transclass = transformations.itervalues().next().__class__.__name__
        model = RegisterModel(transformations, regmethod=regmethod, transclass=transclass)
        return model

    def run(self, images, ref=None):
        """
        Compute and implement registration on a collection of images / volumes.

        This is a lazy operation that combines the estimation of registration
        with its implementaiton. It returns a new Images object with transformed
        images, and does not expose the registration parameters directly, see the
        'fit' method to obtain parameters directly.

        Parameters
        ----------
        images : Images
            An Images object with the images / volumes to apply registration to.

        ref : ndarray, optional, default = None
            Reference image / volume to estimate registration against. Can safely pass
            None for methods that do not require or use a reference.

        Return
        ------
        Images object with registered images / volumes
        """

        if not (isinstance(images, Images)):
            raise Exception('Input data must be Images or a subclass')

        if len(images.dims.count) not in set([2, 3]):
            raise Exception('Number of image dimensions %s must be 2 or 3' % (len(images.dims.count)))

        if ref is not None:
            self.checkReference(images, ref)

        # broadcast the reference
        ref_bc = images.rdd.context.broadcast(ref)

        def fitandtransform(im, ref):
            t = self.getTransform(im, ref.value)
            return t.apply(im)

        newrdd = images.rdd.mapValues(lambda im: fitandtransform(im, ref_bc))

        return Images(newrdd).__finalize__(images)

class RegisterModel(object):

    def __init__(self, transformations, regmethod=None, transclass=None):
        self.transformations = transformations
        self.regmethod = regmethod
        self.transclass = transclass

    def transform(self, images):
        """
        Apply the transformation to an Images object.

        Will apply the underlying dictionary of transformations to
        the images or volumes of the Images object. The dictionary acts as a lookup
        table specifying which transformation should be applied to which record of the
        Images object based on the key. Because transformations are small,
        we broadcast the transformations rather than using a join.

        See also
        --------
        Register : construct registration algorithms
        """

        from thunder.rdds.images import Images

        # broadcast the transformations
        transformations_bc = images.rdd.context.broadcast(self.transformations)

        # apply the transformations
        newrdd = images.rdd.map(lambda (k, im): (k, transformations_bc.value[k].apply(im)))
        return Images(newrdd).__finalize__(images)

    def save(self, file):
        """
        Serialize registration model to a file as text using JSON.

        Format is a dictionary, with keys '_regmethod' and '_transtype' specifying
        the registration method used and the transformation type (as strings),
        and '_transformations' containing the transformations. The exact format of the transformations
        will vary by type, but will always be a dictionary, with keys indexing into an Images object,
        and values containing the transformation parameters.

        Parameters
        ----------
        file : filename or file handle
            The file to write to

        """

        import json

        if hasattr(file, 'write'):
            f = file
        else:
            f = open(file, 'w')
        output = json.dumps(self, default=lambda v: v.__dict__)
        f.write(output)
        f.close()

    @staticmethod
    def load(file):
        """
        Deserialize registration model from a file containing JSON.

        Assumes a JSON formatted registration model, with keys '_regmethod' and '_transclass' specifying
        the registration method used and the transformation type as strings, and '_transformations'
        containing the transformations. The format of the transformations will depend on the type,
        but it should be a dictionary of key value pairs, where the keys are keys of the target
        Images object, and the values are arguments for reconstructing each transformation object.

        Parameters
        ----------
        file : str
            Name of a file to read from

        Returns
        -------
        model : RegisterModel
            Instance of a registration model
        """

        import json
        import importlib

        f = open(file, 'r')
        input = json.loads(f.read())

        # import the appropriate transformation class
        regmethod = str(input['regmethod'])
        classname = str(input['transclass'])
        transclass = getattr(importlib.import_module('thunder.imgprocessing.transformation'), classname)

        # instantiate the transformations and construct the model
        transformations = {int(k): transclass(**v) for k, v in input['transformations'].iteritems()}
        model = RegisterModel(transformations, regmethod=regmethod, transclass=classname)
        return model

    def __repr__(self):
        out = "ResgisterModel(method='%s', transtype='%s', transformations=%s)" % \
              (self.regmethod, self.transclass, self.transformations)
        return out[0:120] + " ..."

<|endoftext|>