<commit_before>from __future__ import print_function

import inspect
import timeit
import sys

import logging
from types import FunctionType
from math import log10

if sys.version[0] == '3':
    import io as StringIO
else:
    import StringIO
__version__ = '1.5'
logging.getLogger().setLevel(logging.INFO)

_import_tag = '#!'

def enable():
    """
    Enable all benchmarking.
    """
    Benchmark.enable = True
    ComparisonBenchmark.enable = True
    BenchmarkedFunction.enable = True
    BenchmarkedClass.enable = True

def disable():
    """
    Disable all benchmarking.
    """
    Benchmark.enable = False
    ComparisonBenchmark.enable = False
    BenchmarkedFunction.enable = False
    BenchmarkedClass.enable = False

def set_import_tag(tag):
    if type(tag) == str:
        globals()['_import_tag'] = tag

def convert_time_units(t):
    """ Convert time in seconds into reasonable time units. """
    order = log10(t)
    if -9 < order < -6:
        time_units = 'ns'
        factor = 1000000000
    elif -6 < order < -3:
        time_units = 'us'
        factor = 1000000
    elif -3 <= order < -1:
        time_units = 'ms'
        factor = 1000.
    elif -1 <= order:
        time_units = 's'
        factor = 1
    return "{:.3f} {}".format(factor * t, time_units)

def globalize_indentation(src):
    """ Strip the indentation level so the code runs in the global scope. """
    lines = src.splitlines()
    indent = len(lines[0]) - len(lines[0].strip(' '))
    func_src = ''
    for ii, l in enumerate(src.splitlines()):
        line = l[indent:]
        func_src += line + '\n'
    return func_src

def remove_decorators(src):
    """ Remove decorators from the source code """
    src = src.strip()
    src_lines = src.splitlines()
    multi_line = False
    n_deleted = 0
    for n in xrange(len(src_lines)):
        line = src_lines[n - n_deleted].strip()
        if 'Benchmark' in line or multi_line:
            del src_lines[n - n_deleted]
            n_deleted += 1
            if line.endswith(')'):
                multi_line = False
            else:
                multi_line = True
    setup_src = '\n'.join(src_lines)
    return setup_src

def get_tagged_imports(fp, tag):
    with open(fp, 'r') as f:
        imports = [l[:l.index(tag)] for l in f if tag in l and (l.startswith('import') or l.startswith('from'))]
    src = '\n'.join(imports)
    return src

def generate_call_statement(func, is_class_method, *args, **kwargs):
    # Create the call statement
    if is_class_method:
        stmt = 'instance.' + func.__name__ + '('
    else:
        stmt = func.__name__ + '('
    for arg in args:
        stmt += arg.__repr__() + ', '
    for kw, val in kwargs.iteritems():
        stmt += '{0}={1}, '.format(kw, val.__repr__())
    stmt = stmt.strip(', ')
    stmt += ')'
    return stmt

class ValidationError(Exception):
    pass

class Benchmark(object):
    enable = True

    def __init__(self, setup=None, classname=None, timeit_repeat=3, timeit_number=1000, largs=None, kwargs=None):
        self.setup = setup
        self.timeit_repeat = timeit_repeat
        self.timeit_number = timeit_number
        self.classname = classname
        self.group = None
        self.is_class_method = None
        if largs is not None and type(largs) is tuple:
            self._args = largs[:]
        else:
            self._args = ()
        self._kwargs = kwargs.copy() if kwargs is not None else {}
        self.setup_src = ''
        self.callable = None
        self._is_function = None
        self.log = StringIO.StringIO()
        self.time_average_seconds = None

    def __call__(self, caller):
        if self.enable:
            self.callable = caller
            self._is_function = isinstance(caller, FunctionType)

            fp = inspect.getfile(caller)
            imports = get_tagged_imports(fp, _import_tag)
            func_src = remove_decorators(globalize_indentation(inspect.getsource(caller)))

            # Determine if the function is bound. If it is, keep track of it so we can run the benchmark after the class
            # benchmark has been initialized.
            src_lines = func_src.splitlines()
            self.is_class_method = 'def' in src_lines[0] and 'self' in src_lines[0]
            if self.is_class_method and self.classname:
                try:
                    BenchmarkedClass.bound_functions[self.classname].append(self)
                except KeyError:
                    BenchmarkedClass.bound_functions[self.classname] = [self]

            if callable(self.setup):
                setup_func = inspect.getsource(self.setup)
                setup_src = globalize_indentation(setup_func[setup_func.index('\n') + 1:])
            elif type(self.setup) == str:
                setup_src = self.setup
            else:
                setup_src = ''

            src = '\n'.join([imports, setup_src, func_src])
            self.setup_src = remove_decorators(src) + '\n'
            self.log.write(self.setup_src)

            self.stmt = generate_call_statement(caller, self.is_class_method, *self._args, **self._kwargs)

        return caller

    def write_log(self, fs=None):
        """
        Write the results of the benchmark to a log file.
        :param fs: file-like object.
        """
        log = StringIO.StringIO()
        log.write(self.setup_src)

        # If the function is not bound, write the test score to the log
        if not self.is_class_method:
            time_avg = convert_time_units(self.time_average_seconds)
            log.write("\nAverage time: {0} \n".format(time_avg))

        if fs:
            with open(fs, 'w') as _f:
                _f.write(log.getvalue())

    def run_timeit(self, stmt, setup):
        """ Create the function call statement as a string used for timeit. """
        _timer = timeit.Timer(stmt=stmt, setup=setup)
        trials = _timer.repeat(self.timeit_repeat, self.timeit_number)
        self.time_average_seconds = sum(trials) / len(trials) / self.timeit_number
        # Convert into reasonable time units
        time_avg = convert_time_units(self.time_average_seconds)
        return time_avg

class BenchmarkedClass(Benchmark):
    bound_functions = {}

    def __init__(self, setup=None, cls_args=None, cls_kwargs=None, *args, **kwargs):
        super(BenchmarkedClass, self).__init__(setup, largs=cls_args, kwargs=cls_kwargs, *args, **kwargs)

    def __call__(self, cls):
        if self.enable:
            super(BenchmarkedClass, self).__call__(cls)
            setup_src = self.setup_src
            setup_src += '\ninstance = {}'.format(self.stmt)

            groups = set()
            for p in self.bound_functions[cls.__name__]:
                stmt = p.stmt
                p.run_timeit(stmt, setup_src)
                p.write_log()
                if isinstance(p, BenchmarkedFunction):
                    print("{} \t {}".format(p.callable.__name__, convert_time_units(p.time_average_seconds)))
                if hasattr(p, 'result_validation') and p.result_validation and p.group not in groups:
                    self.validate(p.groups[p.group])
                groups.add(p.group)

        return cls

    def validate(self, benchmarks):
        # Execute the code once to get it's results (to be used in function validation)
        class_code = self.setup_src
        instance_creation = '\ninstance = {}'.format(self.stmt)
        for i, benchmark in enumerate(benchmarks):
            if not benchmark.result_validation:
                break

            validation_code = class_code + instance_creation + '\nvalidation_result = ' + benchmark.stmt
            validation_scope = {}
            exec (validation_code, validation_scope)
            # Store the result in the first function in the group.
            if i == 0:
                compare_against_function = benchmarks[0].callable.__name__
                compare_against_result = validation_scope['validation_result']
            else:
                if compare_against_result == validation_scope['validation_result']:
                    logging.info('Validating {} against {}......PASSED!'.format(benchmark.callable.__name__,
                                                                                compare_against_function))
                else:
                    error = 'Results of functions {0} and {1} are not equivalent.\n{0}:\t {2}\n{1}:\t{3}'
                    raise ValidationError(error.format(compare_against_function, benchmark.callable.__name__,
                                              compare_against_result, validation_scope['validation_result']))

class BenchmarkedFunction(Benchmark):
    def __call__(self, caller):
        if self.enable:
            super(BenchmarkedFunction, self).__call__(caller)
            if not self.is_class_method:
                self.run_timeit(self.stmt, self.setup_src)
                print("{} \t {}".format(caller.__name__, convert_time_units(self.time_average_seconds)))
        return caller

class ComparisonBenchmark(Benchmark):
    groups = {}

    def __init__(self, group, classname=None, setup=None, validation=False, *largs, **kwargs):
        super(ComparisonBenchmark, self).__init__(setup=setup, *largs, **kwargs)
        self.group = group
        self.classname = classname
        self.result_validation = validation
        self.result = None
        if group not in self.groups:
            self.groups[group] = []

    def __call__(self, caller):
        if self.enable:
            super(ComparisonBenchmark, self).__call__(caller)
            self.groups[self.group].append(self)
            # Bound functions are tested in ClassBenchmark.__call__
            # Just store a reference to the ComparisonBenchmark if the function is bound, otherwise, run the test
            if not self.is_class_method:
                # Run the test
                self.run_timeit(self.stmt, self.setup_src)
                if self.result_validation:
                    self.validate()
        return caller

    def validate(self):
        # Execute the code once to get it's results (to be used in function validation)
        validation_code = self.setup_src + '\nvalidation_result = ' + self.stmt
        validation_scope = {}
        exec (validation_code, validation_scope)
        # Store the result in the first function in the group.
        if len(self.groups[self.group]) == 1:
            self.result = validation_scope['validation_result']
        else:
            compare_against = self.groups[self.group][0]
            test = [benchmark.result_validation for benchmark in self.groups[self.group]]
            if not all(test):
                raise ValueError('All functions within a group must have the same validation flag.')
            compare_result = compare_against.result
            if compare_result == validation_scope['validation_result']:
                logging.info('Validating {}......PASSED!'.format(benchmark.callable.__name__))
            else:
                error = 'Results of functions {0} and {1} are not equivalent.\n{0}:\t {2}\n{1}:\t{3}'
                raise ValidationError(error.format(compare_against.callable.__name__, self.callable.__name__,
                                          compare_result, validation_scope['validation_result']))

    @staticmethod
    def summarize(group, fs=None):
        """
        Tabulate and write the results of ComparisonBenchmarks to a file or standard out.
        :param str group: name of the comparison group.
        :param fs: file-like object (Optional)
        """
        _line_break = '{0:-<100}\n'.format('')
        tests = sorted(ComparisonBenchmark.groups[group], key=lambda t: getattr(t, 'time_average_seconds'))
        log = StringIO.StringIO()
        log.write('Call statement:\n\n')
        log.write('\t' + tests[0].stmt)
        log.write('\n\n\n')
        fmt = "{0: <35} {1: <12} {2: <15} {3: <15} {4: <14}\n"
        log.write(fmt.format('Function Name', 'Time', '% of Fastest', 'timeit_repeat', 'timeit_number'))
        log.write(_line_break)
        log.write('\n')

        for t in tests:
            func_name = "{}.{}".format(t.classname, t.callable.__name__) if t.classname else t.callable.__name__
            log.write(fmt.format(func_name,
                                 convert_time_units(t.time_average_seconds),
                                 "{:.1f}".format(tests[0].time_average_seconds / t.time_average_seconds * 100),
                                 t.timeit_repeat,
                                 t.timeit_number))
        log.write(_line_break)

        log.write('\n\n\nSource Code:\n')
        log.write(_line_break)
        for test in tests:
            log.write(test.log.getvalue())
            log.write(_line_break)

        if isinstance(fs, str):
            with open(fs, 'w') as f:
                f.write(log.getvalue())

        elif fs is None:
            print(log.getvalue())
        else:
            try:
                fs.write(log.getvalue())
            except AttributeError as e:
                print(e)

<commit_msg>Check for '@' in remove_demorators() so that the function does not fail when coming across the text 'Benchmark' in another context<commit_after>from __future__ import print_function

import inspect
import timeit
import sys

import logging
from types import FunctionType
from math import log10

if sys.version[0] == '3':
    import io as StringIO
else:
    import StringIO
__version__ = '1.5'
logging.getLogger().setLevel(logging.INFO)

_import_tag = '#!'

def enable():
    """
    Enable all benchmarking.
    """
    Benchmark.enable = True
    ComparisonBenchmark.enable = True
    BenchmarkedFunction.enable = True
    BenchmarkedClass.enable = True

def disable():
    """
    Disable all benchmarking.
    """
    Benchmark.enable = False
    ComparisonBenchmark.enable = False
    BenchmarkedFunction.enable = False
    BenchmarkedClass.enable = False

def set_import_tag(tag):
    if type(tag) == str:
        globals()['_import_tag'] = tag

def convert_time_units(t):
    """ Convert time in seconds into reasonable time units. """
    order = log10(t)
    if -9 < order < -6:
        time_units = 'ns'
        factor = 1000000000
    elif -6 < order < -3:
        time_units = 'us'
        factor = 1000000
    elif -3 <= order < -1:
        time_units = 'ms'
        factor = 1000.
    elif -1 <= order:
        time_units = 's'
        factor = 1
    return "{:.3f} {}".format(factor * t, time_units)

def globalize_indentation(src):
    """ Strip the indentation level so the code runs in the global scope. """
    lines = src.splitlines()
    indent = len(lines[0]) - len(lines[0].strip(' '))
    func_src = ''
    for ii, l in enumerate(src.splitlines()):
        line = l[indent:]
        func_src += line + '\n'
    return func_src

def remove_decorators(src):
    """ Remove decorators from the source code """
    src = src.strip()
    src_lines = src.splitlines()
    multi_line = False
    n_deleted = 0
    for n in xrange(len(src_lines)):
        line = src_lines[n - n_deleted].strip()
        if (line.startswith('@') and 'Benchmark' in line) or multi_line:
            del src_lines[n - n_deleted]
            n_deleted += 1
            if line.endswith(')'):
                multi_line = False
            else:
                multi_line = True
    setup_src = '\n'.join(src_lines)
    return setup_src

def get_tagged_imports(fp, tag):
    with open(fp, 'r') as f:
        imports = [l[:l.index(tag)] for l in f if tag in l and (l.startswith('import') or l.startswith('from'))]
    src = '\n'.join(imports)
    return src

def generate_call_statement(func, is_class_method, *args, **kwargs):
    # Create the call statement
    if is_class_method:
        stmt = 'instance.' + func.__name__ + '('
    else:
        stmt = func.__name__ + '('
    for arg in args:
        stmt += arg.__repr__() + ', '
    for kw, val in kwargs.iteritems():
        stmt += '{0}={1}, '.format(kw, val.__repr__())
    stmt = stmt.strip(', ')
    stmt += ')'
    return stmt

class ValidationError(Exception):
    pass

class Benchmark(object):
    enable = True

    def __init__(self, setup=None, classname=None, timeit_repeat=3, timeit_number=1000, largs=None, kwargs=None):
        self.setup = setup
        self.timeit_repeat = timeit_repeat
        self.timeit_number = timeit_number
        self.classname = classname
        self.group = None
        self.is_class_method = None
        if largs is not None and type(largs) is tuple:
            self._args = largs[:]
        else:
            self._args = ()
        self._kwargs = kwargs.copy() if kwargs is not None else {}
        self.setup_src = ''
        self.callable = None
        self._is_function = None
        self.log = StringIO.StringIO()
        self.time_average_seconds = None

    def __call__(self, caller):
        if self.enable:
            self.callable = caller
            self._is_function = isinstance(caller, FunctionType)

            fp = inspect.getfile(caller)
            imports = get_tagged_imports(fp, _import_tag)
            func_src = remove_decorators(globalize_indentation(inspect.getsource(caller)))

            # Determine if the function is bound. If it is, keep track of it so we can run the benchmark after the class
            # benchmark has been initialized.
            src_lines = func_src.splitlines()
            self.is_class_method = 'def' in src_lines[0] and 'self' in src_lines[0]
            if self.is_class_method and self.classname:
                try:
                    BenchmarkedClass.bound_functions[self.classname].append(self)
                except KeyError:
                    BenchmarkedClass.bound_functions[self.classname] = [self]

            if callable(self.setup):
                setup_func = inspect.getsource(self.setup)
                setup_src = globalize_indentation(setup_func[setup_func.index('\n') + 1:])
            elif type(self.setup) == str:
                setup_src = self.setup
            else:
                setup_src = ''

            src = '\n'.join([imports, setup_src, func_src])
            self.setup_src = src + '\n'
            self.log.write(self.setup_src)

            self.stmt = generate_call_statement(caller, self.is_class_method, *self._args, **self._kwargs)

        return caller

    def write_log(self, fs=None):
        """
        Write the results of the benchmark to a log file.
        :param fs: file-like object.
        """
        log = StringIO.StringIO()
        log.write(self.setup_src)

        # If the function is not bound, write the test score to the log
        if not self.is_class_method:
            time_avg = convert_time_units(self.time_average_seconds)
            log.write("\nAverage time: {0} \n".format(time_avg))

        if fs:
            with open(fs, 'w') as _f:
                _f.write(log.getvalue())

    def run_timeit(self, stmt, setup):
        """ Create the function call statement as a string used for timeit. """
        _timer = timeit.Timer(stmt=stmt, setup=setup)
        trials = _timer.repeat(self.timeit_repeat, self.timeit_number)
        self.time_average_seconds = sum(trials) / len(trials) / self.timeit_number
        # Convert into reasonable time units
        time_avg = convert_time_units(self.time_average_seconds)
        return time_avg

class BenchmarkedClass(Benchmark):
    bound_functions = {}

    def __init__(self, setup=None, cls_args=None, cls_kwargs=None, *args, **kwargs):
        super(BenchmarkedClass, self).__init__(setup, largs=cls_args, kwargs=cls_kwargs, *args, **kwargs)

    def __call__(self, cls):
        if self.enable:
            super(BenchmarkedClass, self).__call__(cls)
            setup_src = self.setup_src
            setup_src += '\ninstance = {}'.format(self.stmt)

            groups = set()
            for p in self.bound_functions[cls.__name__]:
                stmt = p.stmt
                p.run_timeit(stmt, setup_src)
                p.write_log()
                if isinstance(p, BenchmarkedFunction):
                    print("{} \t {}".format(p.callable.__name__, convert_time_units(p.time_average_seconds)))
                if hasattr(p, 'result_validation') and p.result_validation and p.group not in groups:
                    self.validate(p.groups[p.group])
                groups.add(p.group)

        return cls

    def validate(self, benchmarks):
        # Execute the code once to get it's results (to be used in function validation)
        class_code = self.setup_src
        instance_creation = '\ninstance = {}'.format(self.stmt)
        for i, benchmark in enumerate(benchmarks):
            if not benchmark.result_validation:
                break

            validation_code = class_code + instance_creation + '\nvalidation_result = ' + benchmark.stmt
            validation_scope = {}
            exec (validation_code, validation_scope)
            # Store the result in the first function in the group.
            if i == 0:
                compare_against_function = benchmarks[0].callable.__name__
                compare_against_result = validation_scope['validation_result']
            else:
                if compare_against_result == validation_scope['validation_result']:
                    logging.info('Validating {} against {}......PASSED!'.format(benchmark.callable.__name__,
                                                                                compare_against_function))
                else:
                    error = 'Results of functions {0} and {1} are not equivalent.\n{0}:\t {2}\n{1}:\t{3}'
                    raise ValidationError(error.format(compare_against_function, benchmark.callable.__name__,
                                              compare_against_result, validation_scope['validation_result']))

class BenchmarkedFunction(Benchmark):
    def __call__(self, caller):
        if self.enable:
            super(BenchmarkedFunction, self).__call__(caller)
            if not self.is_class_method:
                self.run_timeit(self.stmt, self.setup_src)
                print("{} \t {}".format(caller.__name__, convert_time_units(self.time_average_seconds)))
        return caller

class ComparisonBenchmark(Benchmark):
    groups = {}

    def __init__(self, group, classname=None, setup=None, validation=False, *largs, **kwargs):
        super(ComparisonBenchmark, self).__init__(setup=setup, *largs, **kwargs)
        self.group = group
        self.classname = classname
        self.result_validation = validation
        self.result = None
        if group not in self.groups:
            self.groups[group] = []

    def __call__(self, caller):
        if self.enable:
            super(ComparisonBenchmark, self).__call__(caller)
            self.groups[self.group].append(self)
            # Bound functions are tested in ClassBenchmark.__call__
            # Just store a reference to the ComparisonBenchmark if the function is bound, otherwise, run the test
            if not self.is_class_method:
                # Run the test
                self.run_timeit(self.stmt, self.setup_src)
                if self.result_validation:
                    self.validate()
        return caller

    def validate(self):
        # Execute the code once to get it's results (to be used in function validation)
        validation_code = self.setup_src + '\nvalidation_result = ' + self.stmt
        validation_scope = {}
        exec (validation_code, validation_scope)
        # Store the result in the first function in the group.
        if len(self.groups[self.group]) == 1:
            self.result = validation_scope['validation_result']
        else:
            compare_against = self.groups[self.group][0]
            test = [benchmark.result_validation for benchmark in self.groups[self.group]]
            if not all(test):
                raise ValueError('All functions within a group must have the same validation flag.')
            compare_result = compare_against.result
            if compare_result == validation_scope['validation_result']:
                logging.info('Validating {}......PASSED!'.format(benchmark.callable.__name__))
            else:
                error = 'Results of functions {0} and {1} are not equivalent.\n{0}:\t {2}\n{1}:\t{3}'
                raise ValidationError(error.format(compare_against.callable.__name__, self.callable.__name__,
                                          compare_result, validation_scope['validation_result']))

    @staticmethod
    def summarize(group, fs=None):
        """
        Tabulate and write the results of ComparisonBenchmarks to a file or standard out.
        :param str group: name of the comparison group.
        :param fs: file-like object (Optional)
        """
        _line_break = '{0:-<100}\n'.format('')
        tests = sorted(ComparisonBenchmark.groups[group], key=lambda t: getattr(t, 'time_average_seconds'))
        log = StringIO.StringIO()
        log.write('Call statement:\n\n')
        log.write('\t' + tests[0].stmt)
        log.write('\n\n\n')
        fmt = "{0: <35} {1: <12} {2: <15} {3: <15} {4: <14}\n"
        log.write(fmt.format('Function Name', 'Time', '% of Fastest', 'timeit_repeat', 'timeit_number'))
        log.write(_line_break)
        log.write('\n')

        for t in tests:
            func_name = "{}.{}".format(t.classname, t.callable.__name__) if t.classname else t.callable.__name__
            log.write(fmt.format(func_name,
                                 convert_time_units(t.time_average_seconds),
                                 "{:.1f}".format(tests[0].time_average_seconds / t.time_average_seconds * 100),
                                 t.timeit_repeat,
                                 t.timeit_number))
        log.write(_line_break)

        log.write('\n\n\nSource Code:\n')
        log.write(_line_break)
        for test in tests:
            log.write(test.log.getvalue())
            log.write(_line_break)

        if isinstance(fs, str):
            with open(fs, 'w') as f:
                f.write(log.getvalue())

        elif fs is None:
            print(log.getvalue())
        else:
            try:
                fs.write(log.getvalue())
            except AttributeError as e:
                print(e)

<|endoftext|>


<commit_before>from pyquante2.basis.basisset import basisset
from pyquante2.basis.cgbf import cgbf
from pyquante2.basis.pgbf import pgbf
from pyquante2.geo.molecule import molecule
from pyquante2.geo.samples import *
from pyquante2.graphics.vtkplot import vtk_orbs
from pyquante2.grid.grid import grid
from pyquante2.ints.one import S,T,V
from pyquante2.pt.mp2 import mp2
from pyquante2.cc.ccsd import ccsd
from pyquante2.scf.hamiltonians import rhf, uhf, rohf

try:
    import matplotlib
    from pyquante2.graphics.lineplot import lineplot_orbs,line
    from pyquante2.graphics.contourplot import contourplot
except:
    pass
<commit_msg>add cuhf<commit_after>from pyquante2.basis.basisset import basisset
from pyquante2.basis.cgbf import cgbf
from pyquante2.basis.pgbf import pgbf
from pyquante2.geo.molecule import molecule
from pyquante2.geo.samples import *
from pyquante2.graphics.vtkplot import vtk_orbs
from pyquante2.grid.grid import grid
from pyquante2.ints.one import S,T,V
from pyquante2.pt.mp2 import mp2
from pyquante2.cc.ccsd import ccsd
from pyquante2.scf.hamiltonians import rhf, uhf, rohf, cuhf

try:
    import matplotlib
    from pyquante2.graphics.lineplot import lineplot_orbs,line
    from pyquante2.graphics.contourplot import contourplot
except:
    pass
<|endoftext|>


<commit_before>import base64
import json
import struct

from six import integer_types
import numpy as np

from pyquil.wavefunction import Wavefunction

OCTETS_PER_DOUBLE_FLOAT = 8
OCTETS_PER_COMPLEX_DOUBLE = 2 * OCTETS_PER_DOUBLE_FLOAT

class JobResult(object):
    def __init__(self, qpu, success, result=None, payload=None):
        """
        :param QPUConnection qpu:
        :param bool success:
        :param dict result: JSON dictionary of the result message
        """
        self.qpu = qpu
        self.success = success
        self.result = result
        self.payload = payload

    def job_id(self):
        return self.result['jobId']

    def get(self):
        return self.qpu.get_job(self)

    def update(self, success, result):
        self.success = success
        self.result = result
        return self

    def decode(self):
        return json.loads(self.result['result'])

    def __str__(self):
        return str(self.result)

    @classmethod
    def load_res(cls, qpu, response):
        result = json.loads(response.content.decode("utf-8"))
        return cls(qpu, response.ok, result)

class WavefunctionResult(JobResult):

    def decode(self):
        string_result = self.result['result']
        coef_string = base64.b64decode(string_result.encode("utf-8"))
        wvf, classical_mem = recover_complexes(coef_string=coef_string,
                                               classical_addresses=self.payload['addresses'])
        return wvf, classical_mem

def recover_complexes(coef_string, classical_addresses):
    num_octets = len(coef_string)
    num_addresses = len(classical_addresses)
    num_memory_octets = _round_to_next_multiple(num_addresses, 8) / 8
    num_wavefunction_octets = num_octets - num_memory_octets

    # Parse the classical memory
    mem = []
    for i in range(num_memory_octets):
        octet = struct.unpack('B', coef_string[i])[0]
        mem.extend(_octet_bits(octet))

    mem = mem[0:num_addresses]

    # Parse the wavefunction
    wf = np.zeros(num_wavefunction_octets / OCTETS_PER_COMPLEX_DOUBLE, dtype=np.cfloat)
    for i, p in enumerate(range(num_memory_octets, num_octets, OCTETS_PER_COMPLEX_DOUBLE)):
        re_be = coef_string[p: p + OCTETS_PER_DOUBLE_FLOAT]
        im_be = coef_string[p + OCTETS_PER_DOUBLE_FLOAT: p + OCTETS_PER_COMPLEX_DOUBLE]
        re = struct.unpack('>d', re_be)[0]
        im = struct.unpack('>d', im_be)[0]
        wf[i] = complex(re, im)

    return Wavefunction(wf), mem

class RamseyResult(JobResult):
    pass

class RabiResult(JobResult):
    pass

class T1Result(JobResult):
    pass

def _round_to_next_multiple(n, m):
    """
    Round up the the next multiple.

    :param n: The number to round up.
    :param m: The multiple.
    :return: The rounded number
    """
    return n if n % m == 0 else n + m - n % m

def _octet_bits(o):
    """
    Get the bits of an octet.

    :param o: The octets.
    :return: The bits as a list in LSB-to-MSB order.
    :rtype: list
    """
    if not isinstance(o, integer_types):
        raise TypeError("o should be an int")
    if not (0 <= o <= 255):
        raise ValueError("o should be between 0 and 255 inclusive")
    bits = [0] * 8
    for i in range(8):
        if 1 == o & 1:
            bits[i] = 1
        o = o >> 1
    return bits<commit_msg>add wait for job<commit_after>import base64
import json
import struct
import time

from six import integer_types
import numpy as np

from pyquil.wavefunction import Wavefunction

OCTETS_PER_DOUBLE_FLOAT = 8
OCTETS_PER_COMPLEX_DOUBLE = 2 * OCTETS_PER_DOUBLE_FLOAT

def wait_for_job(res, ping_time=0.5):
    while not res.is_done():
        res.get()
        time.sleep(ping_time)
    return res

class JobResult(object):
    def __init__(self, qpu, success, result=None, payload=None):
        """
        :param QPUConnection qpu:
        :param bool success:
        :param dict result: JSON dictionary of the result message
        """
        self.qpu = qpu
        self.success = success
        self.result = result
        self.payload = payload

    def is_done(self):
        return 'result' in self.result

    def job_id(self):
        return self.result['jobId']

    def get(self):
        return self.qpu.get_job(self)

    def update(self, success, result):
        self.success = success
        self.result = result
        return self

    def decode(self):
        return json.loads(self.result['result'])

    def __str__(self):
        return str(self.result)

    @classmethod
    def load_res(cls, qpu, response):
        result = json.loads(response.content.decode("utf-8"))
        return cls(qpu, response.ok, result)

class WavefunctionResult(JobResult):

    def decode(self):
        string_result = self.result['result']
        coef_string = base64.b64decode(string_result.encode("utf-8"))
        wvf, classical_mem = recover_complexes(coef_string=coef_string,
                                               classical_addresses=self.payload['addresses'])
        return wvf, classical_mem

def recover_complexes(coef_string, classical_addresses):
    num_octets = len(coef_string)
    num_addresses = len(classical_addresses)
    num_memory_octets = _round_to_next_multiple(num_addresses, 8) / 8
    num_wavefunction_octets = num_octets - num_memory_octets

    # Parse the classical memory
    mem = []
    for i in range(num_memory_octets):
        octet = struct.unpack('B', coef_string[i])[0]
        mem.extend(_octet_bits(octet))

    mem = mem[0:num_addresses]

    # Parse the wavefunction
    wf = np.zeros(num_wavefunction_octets / OCTETS_PER_COMPLEX_DOUBLE, dtype=np.cfloat)
    for i, p in enumerate(range(num_memory_octets, num_octets, OCTETS_PER_COMPLEX_DOUBLE)):
        re_be = coef_string[p: p + OCTETS_PER_DOUBLE_FLOAT]
        im_be = coef_string[p + OCTETS_PER_DOUBLE_FLOAT: p + OCTETS_PER_COMPLEX_DOUBLE]
        re = struct.unpack('>d', re_be)[0]
        im = struct.unpack('>d', im_be)[0]
        wf[i] = complex(re, im)

    return Wavefunction(wf), mem

class RamseyResult(JobResult):
    pass

class RabiResult(JobResult):
    pass

class T1Result(JobResult):
    pass

def _round_to_next_multiple(n, m):
    """
    Round up the the next multiple.

    :param n: The number to round up.
    :param m: The multiple.
    :return: The rounded number
    """
    return n if n % m == 0 else n + m - n % m

def _octet_bits(o):
    """
    Get the bits of an octet.

    :param o: The octets.
    :return: The bits as a list in LSB-to-MSB order.
    :rtype: list
    """
    if not isinstance(o, integer_types):
        raise TypeError("o should be an int")
    if not (0 <= o <= 255):
        raise ValueError("o should be between 0 and 255 inclusive")
    bits = [0] * 8
    for i in range(8):
        if 1 == o & 1:
            bits[i] = 1
        o = o >> 1
    return bits<|endoftext|>


<commit_before>import sys

if sys.version_info >= (3, 0):
    from io import StringIO
    from urllib.request import urlopen
    from urllib.error import HTTPError
else:
    from cStringIO import StringIO
    from urllib2 import urlopen, HTTPError

import os
import re
import shutil
import subprocess as sp
import xml.etree.ElementTree as ET
import zipfile as zf
from ftplib import FTP
from time import strftime, gmtime
from xml.dom import minidom
import warnings
from os.path import expanduser

import pyroSAR
from pyroSAR import identify
from pyroSAR.ancillary import dissolve, finder
from pyroSAR.spatial import gdal_translate
from .._dev_config import LOOKUP, ExamineExe

def parse_recipe(name):
    name = name if name.endswith('.xml') else name + '.xml'
    absname = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'recipes', name)
    with open(absname, 'r') as workflow:
        tree = ET.fromstring(workflow.read())
    return tree

def parse_node(name):
    name = name if name.endswith('.xml') else name + '.xml'
    absname = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'recipes', 'nodes', name)
    with open(absname, 'r') as workflow:
        tree = ET.fromstring(workflow.read())
    return tree

def parse_suffix(workflow):
    nodes = workflow.findall('node')
    suffix = '_'.join(filter(None, [LOOKUP.snap.suffix[x] for x in [y.attrib['id'] for y in nodes]]))
    return suffix

def insert_node(workflow, predecessor_id, node):
    predecessor = workflow.find('.//node[@id="{}"]'.format(predecessor_id))
    position = list(workflow).index(predecessor) + 1
    workflow.insert(position, node)
    newnode = workflow[position]
    newnode.find('.//sources/sourceProduct').attrib['refid'] = predecessor.attrib['id']
    workflow[position + 1].find('.//sources/sourceProduct').attrib['refid'] = newnode.attrib['id']

def write_recipe(recipe, outfile):
    outfile = outfile if outfile.endswith('.xml') else outfile + '.xml'
    rough_string = ET.tostring(recipe, 'utf-8')
    reparsed = minidom.parseString(rough_string)
    with open(outfile, 'w') as out:
        out.write(reparsed.toprettyxml(indent='\t', newl=''))

def getOrbitContentVersions(contentVersion):
    return dict(
        [re.split('\s*=\s*', x.strip('\r')) for x in contentVersion.read().split('\n') if re.search('^[0-9]{4}', x)])

class GetAuxdata:
    def __init__(self, datasets, scenes):
        self.datasets = datasets
        self.scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes]
        self.sensors = list(set([scene.sensor for scene in scenes]))

        try:
            self.auxDataPath = os.path.join(os.environ['HOME'], '.snap/auxdata')
        except KeyError:
            self.auxDataPath = os.path.join(os.environ['USERPROFILE'], '.snap/auxdata')

    def srtm_1sec_hgt(self):
        pass
        # Wird nicht benutzt?

#        for dataset in self.datasets:
#            files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in
#                     list(set(dissolve([scene.getHGT() for scene in self.scenes])))]

def getAuxdata(datasets, scenes):
    auxDataPath = os.path.join(expanduser("~"), '.snap/auxdata')

    scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes]
    sensors = list(set([scene.sensor for scene in scenes]))
    for dataset in datasets:
        if dataset == 'SRTM 1Sec HGT':
            files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in
                     list(set(dissolve([scene.getHGT() for scene in scenes])))]
            for file in files:
                infile = os.path.join('http://step.esa.int/auxdata/dem/SRTMGL1', file)
                outfile = os.path.join(auxDataPath, 'dem/SRTM 1Sec HGT', file)
                if not os.path.isfile(outfile):
                    print(infile)
                    try:
                        input = urlopen(infile)
                    except HTTPError:
                        print('-> not available')
                        continue
                    with open(outfile, 'wb') as output:
                        output.write(input.read())
                    input.close()
        elif dataset == 'POEORB':
            for sensor in sensors:
                if re.search('S1[AB]', sensor):

                    dates = [(scene.start[:4], scene.start[4:6]) for scene in scenes]
                    years = list(set([x[0] for x in dates]))

                    remote_contentVersion = urlopen(
                        'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/remote_contentVersion.txt')
                    versions_remote = getOrbitContentVersions(remote_contentVersion)

                    for year in years:
                        dir_orb = os.path.join(auxDataPath, 'Orbits/Sentinel-1/POEORB', year)

                        if not os.path.isdir(dir_orb):
                            os.makedirs(dir_orb)
                        contentVersionFile = os.path.join(dir_orb, 'contentVersion.txt')

                        if os.path.isfile(contentVersionFile):
                            contentVersion = open(contentVersionFile, 'r+')
                            versions_local = getOrbitContentVersions(contentVersion)
                        else:
                            contentVersion = open(contentVersionFile, 'w')
                            versions_local = {}

                        combine = dict(set(versions_local.items()) & set(versions_remote.items()))

                        dates_select = [x for x in dates if x[0] == year]
                        months = list(set([x[1] for x in dates_select]))

                        orb_ids = sorted(
                            [x for x in ['{}-{}.zip'.format(year, month) for month in months] if not x in combine])

                        if len(orb_ids) > 0:
                            contentVersion.write('#\n#{}\n'.format(strftime('%a %b %d %H:%M:%S %Z %Y', gmtime())))

                            for orb_id in orb_ids:
                                orb_remote = urlopen(
                                    'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/{}'.format(orb_id))
                                orb_remote_stream = zf.ZipFile(StringIO(orb_remote.read()), 'r')
                                orb_remote.close()

                                targets = [x for x in orb_remote_stream.namelist() if
                                           not os.path.isfile(os.path.join(dir_orb, x))]
                                orb_remote_stream.extractall(dir_orb, targets)
                                orb_remote_stream.close()

                                versions_local[orb_id] = versions_remote[orb_id]

                                for key, val in versions_local.iteritems():
                                    contentVersion.write('{}={}\n'.format(key, val))

                        contentVersion.close()
                    remote_contentVersion.close()
                else:
                    print('not implemented yet')
        elif dataset == 'Delft Precise Orbits':
            path_server = 'dutlru2.lr.tudelft.nl'
            subdirs = {'ASAR:': 'ODR.ENVISAT1/eigen-cg03c', 'ERS1': 'ODR.ERS-1/dgm-e04', 'ERS2': 'ODR.ERS-2/dgm-e04'}
            ftp = FTP(path_server)
            ftp.login()
            for sensor in sensors:
                if sensor in subdirs.keys():
                    path_target = os.path.join('pub/orbits', subdirs[sensor])
                    path_local = os.path.join(auxDataPath, 'Orbits/Delft Precise Orbits', subdirs[sensor])
                    ftp.cwd(path_target)
                    for item in ftp.nlst():
                        ftp.retrbinary('RETR ' + item, open(os.path.join(path_local, item), 'wb').write)
            ftp.quit()
        else:
            print('not implemented yet')

def gpt(xmlfile):
    """
    wrapper for ESA SNAP Graph Processing Tool GPT
    input is a readily formatted workflow xml file as created by function geocode in module snap.util
    """

    with open(xmlfile, 'r') as infile:
        workflow = ET.fromstring(infile.read())
    write = workflow.find('.//node[@id="Write"]')
    outname = write.find('.//parameters/file').text
    outdir = os.path.dirname(outname)
    format = write.find('.//parameters/formatName').text
    infile = workflow.find('.//node[@id="Read"]/parameters/file').text

    if format == 'GeoTiff-BigTIFF':
        cmd = ['gpt',
               # '-Dsnap.dataio.reader.tileWidth=*',
               # '-Dsnap.dataio.reader.tileHeight=1',
               '-Dsnap.dataio.bigtiff.tiling.width=256',
               '-Dsnap.dataio.bigtiff.tiling.height=256',
               # '-Dsnap.dataio.bigtiff.compression.type=LZW',
               # '-Dsnap.dataio.bigtiff.compression.quality=0.75',
               xmlfile]
    else:
        cmd = ['gpt', xmlfile]

    proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    out, err = proc.communicate()
    if proc.returncode != 0:
        print('failed: ', os.path.basename(infile))
        if os.path.isfile(outname + '.tif'):
            os.remove(outname + '.tif')
        elif os.path.isdir(outname):
            shutil.rmtree(outname)
        raise RuntimeError(err)

    if format == 'ENVI':
        id = pyroSAR.identify(infile)
        suffix = parse_suffix(workflow)
        for item in finder(outname, ['*.img']):
            pol = re.search('[HV]{2}', item).group()
            name_new = os.path.join(outdir, '{}_{}_{}'.format(id.outname_base(), pol, suffix))
            translateoptions = {'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'], 'format': 'GTiff'}
            gdal_translate(item, name_new, translateoptions)
        shutil.rmtree(outname)

class ExamineSnap(ExamineExe):
    """
    Class to check if snap is installed. This will be called with snap.__init__
    as snap_config. If you are running multiple snap versions or the package can
    not find the snap executable, you can set an path via: snap_config.set_path("path")
    """

    def __init__(self):
        super(ExamineSnap, self).__init__()

        self.status, self.path = self.examine(self.SNAP_EXECUTABLE)
        self.auxdatapath = os.path.join(expanduser("~"), '.snap/auxdata')

        # Call proesses
        # self.___pre_process()
        self.__get_etc()
        self.__read_config()

    # def ___pre_process(self):
    #     self.status, self.path = self.examine(self.SNAP_EXECUTABLE)

    # try:
    #     self.status, self.path = self.examine(self.SNAP_EXECUTABLE)
    #
    # except ValueError:
    #     warnings.warn(
    #         "There are more than one instances installed. Define witch one you want to use with snap_config.set_path(...). Otherwise not all functions will be working.",
    #         Warning)
    #
    # if self.status:
    #     pass
    # else:
    #     warnings.warn(
    #         "Snap is not installed. You can install it on: http://step.esa.int/main/download/. Otherwise not all functions will be working.")

    def __get_etc(self):
        try:
            self.etc = os.path.join(os.path.dirname(os.path.dirname(self.path)), 'etc')
            self.auxdata = os.listdir(self.etc)
            self.config_path = os.path.join(self.etc, [s for s in self.auxdata if "snap.auxdata.properties" in s][0])

        except WindowsError:
            raise AssertionError("ETC directory is not existend.")

    def set_path(self, path):
        self.path = os.path.abspath(path)
        self.__get_etc()

    def __read_config(self):
        with open(self.config_path) as config:
            self.config = []
            for line in config:
                self.config.append(line)
<commit_msg>Delete commented out lines in snap.auxil<commit_after>import sys

if sys.version_info >= (3, 0):
    from io import StringIO
    from urllib.request import urlopen
    from urllib.error import HTTPError
else:
    from cStringIO import StringIO
    from urllib2 import urlopen, HTTPError

import os
import re
import shutil
import subprocess as sp
import xml.etree.ElementTree as ET
import zipfile as zf
from ftplib import FTP
from time import strftime, gmtime
from xml.dom import minidom
import warnings
from os.path import expanduser

import pyroSAR
from pyroSAR import identify
from pyroSAR.ancillary import dissolve, finder
from pyroSAR.spatial import gdal_translate
from .._dev_config import LOOKUP, ExamineExe

def parse_recipe(name):
    name = name if name.endswith('.xml') else name + '.xml'
    absname = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'recipes', name)
    with open(absname, 'r') as workflow:
        tree = ET.fromstring(workflow.read())
    return tree

def parse_node(name):
    name = name if name.endswith('.xml') else name + '.xml'
    absname = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'recipes', 'nodes', name)
    with open(absname, 'r') as workflow:
        tree = ET.fromstring(workflow.read())
    return tree

def parse_suffix(workflow):
    nodes = workflow.findall('node')
    suffix = '_'.join(filter(None, [LOOKUP.snap.suffix[x] for x in [y.attrib['id'] for y in nodes]]))
    return suffix

def insert_node(workflow, predecessor_id, node):
    predecessor = workflow.find('.//node[@id="{}"]'.format(predecessor_id))
    position = list(workflow).index(predecessor) + 1
    workflow.insert(position, node)
    newnode = workflow[position]
    newnode.find('.//sources/sourceProduct').attrib['refid'] = predecessor.attrib['id']
    workflow[position + 1].find('.//sources/sourceProduct').attrib['refid'] = newnode.attrib['id']

def write_recipe(recipe, outfile):
    outfile = outfile if outfile.endswith('.xml') else outfile + '.xml'
    rough_string = ET.tostring(recipe, 'utf-8')
    reparsed = minidom.parseString(rough_string)
    with open(outfile, 'w') as out:
        out.write(reparsed.toprettyxml(indent='\t', newl=''))

def getOrbitContentVersions(contentVersion):
    return dict(
        [re.split('\s*=\s*', x.strip('\r')) for x in contentVersion.read().split('\n') if re.search('^[0-9]{4}', x)])

class GetAuxdata:
    def __init__(self, datasets, scenes):
        self.datasets = datasets
        self.scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes]
        self.sensors = list(set([scene.sensor for scene in scenes]))

        try:
            self.auxDataPath = os.path.join(os.environ['HOME'], '.snap/auxdata')
        except KeyError:
            self.auxDataPath = os.path.join(os.environ['USERPROFILE'], '.snap/auxdata')

    def srtm_1sec_hgt(self):
        pass
        # Wird nicht benutzt?

#        for dataset in self.datasets:
#            files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in
#                     list(set(dissolve([scene.getHGT() for scene in self.scenes])))]

def getAuxdata(datasets, scenes):
    auxDataPath = os.path.join(expanduser("~"), '.snap/auxdata')

    scenes = [identify(scene) if isinstance(scene, str) else scene for scene in scenes]
    sensors = list(set([scene.sensor for scene in scenes]))
    for dataset in datasets:
        if dataset == 'SRTM 1Sec HGT':
            files = [x.replace('hgt', 'SRTMGL1.hgt.zip') for x in
                     list(set(dissolve([scene.getHGT() for scene in scenes])))]
            for file in files:
                infile = os.path.join('http://step.esa.int/auxdata/dem/SRTMGL1', file)
                outfile = os.path.join(auxDataPath, 'dem/SRTM 1Sec HGT', file)
                if not os.path.isfile(outfile):
                    print(infile)
                    try:
                        input = urlopen(infile)
                    except HTTPError:
                        print('-> not available')
                        continue
                    with open(outfile, 'wb') as output:
                        output.write(input.read())
                    input.close()
        elif dataset == 'POEORB':
            for sensor in sensors:
                if re.search('S1[AB]', sensor):

                    dates = [(scene.start[:4], scene.start[4:6]) for scene in scenes]
                    years = list(set([x[0] for x in dates]))

                    remote_contentVersion = urlopen(
                        'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/remote_contentVersion.txt')
                    versions_remote = getOrbitContentVersions(remote_contentVersion)

                    for year in years:
                        dir_orb = os.path.join(auxDataPath, 'Orbits/Sentinel-1/POEORB', year)

                        if not os.path.isdir(dir_orb):
                            os.makedirs(dir_orb)
                        contentVersionFile = os.path.join(dir_orb, 'contentVersion.txt')

                        if os.path.isfile(contentVersionFile):
                            contentVersion = open(contentVersionFile, 'r+')
                            versions_local = getOrbitContentVersions(contentVersion)
                        else:
                            contentVersion = open(contentVersionFile, 'w')
                            versions_local = {}

                        combine = dict(set(versions_local.items()) & set(versions_remote.items()))

                        dates_select = [x for x in dates if x[0] == year]
                        months = list(set([x[1] for x in dates_select]))

                        orb_ids = sorted(
                            [x for x in ['{}-{}.zip'.format(year, month) for month in months] if not x in combine])

                        if len(orb_ids) > 0:
                            contentVersion.write('#\n#{}\n'.format(strftime('%a %b %d %H:%M:%S %Z %Y', gmtime())))

                            for orb_id in orb_ids:
                                orb_remote = urlopen(
                                    'http://step.esa.int/auxdata/orbits/Sentinel-1/POEORB/{}'.format(orb_id))
                                orb_remote_stream = zf.ZipFile(StringIO(orb_remote.read()), 'r')
                                orb_remote.close()

                                targets = [x for x in orb_remote_stream.namelist() if
                                           not os.path.isfile(os.path.join(dir_orb, x))]
                                orb_remote_stream.extractall(dir_orb, targets)
                                orb_remote_stream.close()

                                versions_local[orb_id] = versions_remote[orb_id]

                                for key, val in versions_local.iteritems():
                                    contentVersion.write('{}={}\n'.format(key, val))

                        contentVersion.close()
                    remote_contentVersion.close()
                else:
                    print('not implemented yet')
        elif dataset == 'Delft Precise Orbits':
            path_server = 'dutlru2.lr.tudelft.nl'
            subdirs = {'ASAR:': 'ODR.ENVISAT1/eigen-cg03c', 'ERS1': 'ODR.ERS-1/dgm-e04', 'ERS2': 'ODR.ERS-2/dgm-e04'}
            ftp = FTP(path_server)
            ftp.login()
            for sensor in sensors:
                if sensor in subdirs.keys():
                    path_target = os.path.join('pub/orbits', subdirs[sensor])
                    path_local = os.path.join(auxDataPath, 'Orbits/Delft Precise Orbits', subdirs[sensor])
                    ftp.cwd(path_target)
                    for item in ftp.nlst():
                        ftp.retrbinary('RETR ' + item, open(os.path.join(path_local, item), 'wb').write)
            ftp.quit()
        else:
            print('not implemented yet')

def gpt(xmlfile):
    """
    wrapper for ESA SNAP Graph Processing Tool GPT
    input is a readily formatted workflow xml file as created by function geocode in module snap.util
    """

    with open(xmlfile, 'r') as infile:
        workflow = ET.fromstring(infile.read())
    write = workflow.find('.//node[@id="Write"]')
    outname = write.find('.//parameters/file').text
    outdir = os.path.dirname(outname)
    format = write.find('.//parameters/formatName').text
    infile = workflow.find('.//node[@id="Read"]/parameters/file').text

    if format == 'GeoTiff-BigTIFF':
        cmd = ['gpt',
               # '-Dsnap.dataio.reader.tileWidth=*',
               # '-Dsnap.dataio.reader.tileHeight=1',
               '-Dsnap.dataio.bigtiff.tiling.width=256',
               '-Dsnap.dataio.bigtiff.tiling.height=256',
               # '-Dsnap.dataio.bigtiff.compression.type=LZW',
               # '-Dsnap.dataio.bigtiff.compression.quality=0.75',
               xmlfile]
    else:
        cmd = ['gpt', xmlfile]

    proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    out, err = proc.communicate()
    if proc.returncode != 0:
        print('failed: ', os.path.basename(infile))
        if os.path.isfile(outname + '.tif'):
            os.remove(outname + '.tif')
        elif os.path.isdir(outname):
            shutil.rmtree(outname)
        raise RuntimeError(err)

    if format == 'ENVI':
        id = pyroSAR.identify(infile)
        suffix = parse_suffix(workflow)
        for item in finder(outname, ['*.img']):
            pol = re.search('[HV]{2}', item).group()
            name_new = os.path.join(outdir, '{}_{}_{}'.format(id.outname_base(), pol, suffix))
            translateoptions = {'options': ['-q', '-co', 'INTERLEAVE=BAND', '-co', 'TILED=YES'], 'format': 'GTiff'}
            gdal_translate(item, name_new, translateoptions)
        shutil.rmtree(outname)

class ExamineSnap(ExamineExe):
    """
    Class to check if snap is installed. This will be called with snap.__init__
    as snap_config. If you are running multiple snap versions or the package can
    not find the snap executable, you can set an path via: snap_config.set_path("path")
    """

    def __init__(self):
        super(ExamineSnap, self).__init__()

        self.status, self.path = self.examine(self.SNAP_EXECUTABLE)
        self.auxdatapath = os.path.join(expanduser("~"), '.snap/auxdata')

        # Call proesses
        # self.___pre_process()
        self.__get_etc()
        self.__read_config()

    def __get_etc(self):
        try:
            self.etc = os.path.join(os.path.dirname(os.path.dirname(self.path)), 'etc')
            self.auxdata = os.listdir(self.etc)
            self.config_path = os.path.join(self.etc, [s for s in self.auxdata if "snap.auxdata.properties" in s][0])

        except WindowsError:
            raise AssertionError("ETC directory is not existend.")

    def set_path(self, path):
        self.path = os.path.abspath(path)
        self.__get_etc()

    def __read_config(self):
        with open(self.config_path) as config:
            self.config = []
            for line in config:
                self.config.append(line)
<|endoftext|>


<commit_before>"""
pyshtools
=========

pyshtools is an archive of scientific routines that can be used to
perform spherical harmonic transforms and reconstructions, rotations
of data expressed in spherical harmonics, and multitaper spectral
analyses on the sphere.

This module imports the following classes and submodules into the
main namespace:

    SHCoeffs - A high level class for spherical harmonic coefficients.
    SHGrid - A high level classes for global grids.
    SHWindow - A high level classes for localization windows.
    shclasses - All pyshtools classes and subclasses.
    shtools - All pyshtools routines.
    legendre - Legendre functions.
    expand - Spherical harmonic expansion routines.
    shio - Spherical harmonic I/O, storage, and conversion routines.
    spectralanalysis - Global spectral analysis routines.
    localizedspectralanalysis - Localized spectral analysis routines.
    rotate - Spherical harmonic rotation routines.
    gravmag - Gravity and magnetics routines.
    constant - pyshtools constants.
    utils - Other routines.

For further information, consult the web documentation at

   http://shtools.ipgp.fr/

and the GitHub project page at

   https://github.com/SHTOOLS/SHTOOLS
"""

from __future__ import absolute_import as _absolute_import
from __future__ import division as _division
from __future__ import print_function as _print_function

__version__ = '3.4'
__author__ = 'SHTOOLS developers'

import os as _os
import numpy as _np

# ---- Import all wrapped SHTOOLS functions into shtools submodule
from . import _SHTOOLS as shtools

# ---- Import classes into pyshtools namespace
from . import shclasses
from .shclasses import SHCoeffs, SHGrid, SHWindow

# ---- Import shtools subpackages ----
from . import shio
from . import utils

# ---- Import shtools submodules ----
from . import constant
from . import legendre
from . import expand
from . import spectralanalysis
from . import localizedspectralanalysis
from . import rotate
from . import gravmag

# ---- Bind two new functions to the list of all shtools routines ----
_SHTOOLS.PlmIndex = legendre.PlmIndex
_SHTOOLS.YilmIndexVector = shio.YilmIndexVector

# ---------------------------------------------------------------------
# ---- Fill the pyshtools module doc strings and pyshtools constant
# ---- infostrings with documentation from external files. The doc files
# ---- are generated during intitial compilation of pyshtools from md
# ---- formatted text files.
# ---------------------------------------------------------------------
_pydocfolder = _os.path.abspath(_os.path.join(_os.path.dirname(__file__),
                                              'doc'))

for _name, _func in _SHTOOLS.__dict__.items():
    if callable(_func):
        try:
            _path = _os.path.join(_pydocfolder, _name.lower() + '.doc')

            with open(_path) as _pydocfile:
                _pydoc = _pydocfile.read()

            _func.__doc__ = _pydoc
        except IOError as msg:
            print(msg)

for _name in _constant.planetsconstants.__dict__.keys():
    try:
        _path = _os.path.join(_pydocfolder, 'constant_' + _name.lower() +
                              '.doc')

        with open(_path) as _pydocfile:
            _pydoc = _pydocfile.read()

        setattr(getattr(constant, _name), '_infostring', _pydoc)

    except IOError as msg:
        print(msg)

# ---- Rewrite shtools doc string ----
shtools.__doc__ = (
    'pyshtools submodule that includes all pyshtools routines, with the\n' +
    'exception of shclasses.')

# ---- Define FortranStop exception that handles a fortran STOP.
class FortranStop(Exception):
    '''Class that handles errors generated in SHTOOLS Fortran 95 code.'''
    def __call__(self, status):
        if (status == 1):
            errmsg = 'Improper dimensions of input array.'
        elif (status == 2):
            errmsg = 'Improper bounds for input variable.'
        elif (status == 3):
            errmsg = 'Error allocating memory.'
        else:
            errmsg = 'Unhandled Fortran 95 error.'

        raise self.__class__(errmsg)

_SHTOOLS.pystop = FortranStop()

# ---- Define __all__ for use with: from pyshtools import * ----
__all__ = ['constant', 'shclasses', 'SHCoeffs', 'SHGrid', 'SHWindow',
           'legendre', 'expand', 'shio', 'spectralanalysis',
           'localizedspectralanalysis', 'rotate', 'gravmag', 'utils']
<commit_msg>first working example that handles fortran stop.<commit_after>"""
pyshtools
=========

pyshtools is an archive of scientific routines that can be used to
perform spherical harmonic transforms and reconstructions, rotations
of data expressed in spherical harmonics, and multitaper spectral
analyses on the sphere.

This module imports the following classes and submodules into the
main namespace:

    SHCoeffs - A high level class for spherical harmonic coefficients.
    SHGrid - A high level classes for global grids.
    SHWindow - A high level classes for localization windows.
    shclasses - All pyshtools classes and subclasses.
    shtools - All pyshtools routines.
    legendre - Legendre functions.
    expand - Spherical harmonic expansion routines.
    shio - Spherical harmonic I/O, storage, and conversion routines.
    spectralanalysis - Global spectral analysis routines.
    localizedspectralanalysis - Localized spectral analysis routines.
    rotate - Spherical harmonic rotation routines.
    gravmag - Gravity and magnetics routines.
    constant - pyshtools constants.
    utils - Other routines.

For further information, consult the web documentation at

   http://shtools.ipgp.fr/

and the GitHub project page at

   https://github.com/SHTOOLS/SHTOOLS
"""

from __future__ import absolute_import as _absolute_import
from __future__ import division as _division
from __future__ import print_function as _print_function

__version__ = '3.4'
__author__ = 'SHTOOLS developers'

import os as _os
import numpy as _np

# ---- Import all wrapped SHTOOLS functions into shtools submodule
from . import _SHTOOLS as shtools

# ---- Import classes into pyshtools namespace
from . import shclasses
from .shclasses import SHCoeffs, SHGrid, SHWindow

# ---- Import shtools subpackages ----
from . import shio
from . import utils

# ---- Import shtools submodules ----
from . import constant
from . import legendre
from . import expand
from . import spectralanalysis
from . import localizedspectralanalysis
from . import rotate
from . import gravmag

# ---- Bind two new functions to the list of all shtools routines ----
_SHTOOLS.PlmIndex = legendre.PlmIndex
_SHTOOLS.YilmIndexVector = shio.YilmIndexVector

# ---------------------------------------------------------------------
# ---- Fill the pyshtools module doc strings and pyshtools constant
# ---- infostrings with documentation from external files. The doc files
# ---- are generated during intitial compilation of pyshtools from md
# ---- formatted text files.
# ---------------------------------------------------------------------
_pydocfolder = _os.path.abspath(_os.path.join(_os.path.dirname(__file__),
                                              'doc'))

for _name, _func in _SHTOOLS.__dict__.items():
    if callable(_func):
        try:
            _path = _os.path.join(_pydocfolder, _name.lower() + '.doc')

            with open(_path) as _pydocfile:
                _pydoc = _pydocfile.read()

            _func.__doc__ = _pydoc
        except IOError as msg:
            print(msg)

for _name in _constant.planetsconstants.__dict__.keys():
    try:
        _path = _os.path.join(_pydocfolder, 'constant_' + _name.lower() +
                              '.doc')

        with open(_path) as _pydocfile:
            _pydoc = _pydocfile.read()

        setattr(getattr(constant, _name), '_infostring', _pydoc)

    except IOError as msg:
        print(msg)

# ---- Rewrite shtools doc string ----
shtools.__doc__ = (
    'pyshtools submodule that includes all pyshtools routines, with the\n' +
    'exception of shclasses.')

# ---- Define pystop function that recovers from a fortran STOP.
def _pystop(status):
    if (status == 1):
        raise RuntimeError('Improper dimensions of input array.')
    elif (status == 2):
        raise RuntimeError('Improper bounds for ' +
                           'input variable.')
    elif (status == 3):
        raise RuntimeError('Fortran Runtime Error: Error allocating memory.')

_SHTOOLS.pystop = _pystop

# ---- Define FortranStop exception that handles a fortran STOP.
class FortranStop(Exception):
    '''Class that handles errors generated in SHTOOLS Fortran 95 code.'''
    def __call__(self, status):
        if (status == 1):
            errmsg = 'Improper dimensions of input array.'
        elif (status == 2):
            errmsg = 'Improper bounds for input variable.'
        elif (status == 3):
            errmsg = 'Error allocating memory.'
        else:
            errmsg = 'Unhandled Fortran 95 error.'

        raise self.__class__(errmsg)

_SHTOOLS.pystop = FortranStop()

# ---- Define __all__ for use with: from pyshtools import * ----
__all__ = ['constant', 'shclasses', 'SHCoeffs', 'SHGrid', 'SHWindow',
           'legendre', 'expand', 'shio', 'spectralanalysis',
           'localizedspectralanalysis', 'rotate', 'gravmag', 'utils']
<|endoftext|>


<commit_before># Copyright 2019 Ram Rachum and collaborators.
# This program is distributed under the MIT license.
'''
PySnooper - Never use print for debugging again

Usage:

    import pysnooper

    @pysnooper.snoop()
    def your_function(x):
        ...

A log will be written to stderr showing the lines executed and variables
changed in the decorated function.

For more information, see https://github.com/cool-RR/PySnooper
'''

from .tracer import Tracer as snoop
from .variables import Attrs, Exploding, Indices, Keys
import collections

__VersionInfo = collections.namedtuple('VersionInfo',
                                       ('major', 'minor', 'micro'))

__version__ = '0.1.0'
__version_info__ = __VersionInfo(*(map(int, __version__.split('.'))))

del collections, __VersionInfo # Avoid polluting the namespace
<commit_msg>Bump version<commit_after># Copyright 2019 Ram Rachum and collaborators.
# This program is distributed under the MIT license.
'''
PySnooper - Never use print for debugging again

Usage:

    import pysnooper

    @pysnooper.snoop()
    def your_function(x):
        ...

A log will be written to stderr showing the lines executed and variables
changed in the decorated function.

For more information, see https://github.com/cool-RR/PySnooper
'''

from .tracer import Tracer as snoop
from .variables import Attrs, Exploding, Indices, Keys
import collections

__VersionInfo = collections.namedtuple('VersionInfo',
                                       ('major', 'minor', 'micro'))

__version__ = '0.2.0'
__version_info__ = __VersionInfo(*(map(int, __version__.split('.'))))

del collections, __VersionInfo # Avoid polluting the namespace
<|endoftext|>


<commit_before>import crochet
crochet.setup()

from twisted.web import resource, server

def _format_sse(msg, event=None):
    data = []
    if event is not None:
        data.append("event: {}\n".format(event))
    for line in msg.splitlines():
        data.append("data: {}\n".format(line))
    data.append("\n")
    return "".join(data)

class SSEResource(resource.Resource):
    def __init__(self):
        resource.Resource.__init__(self)
        self._listeners = []

    def get_connection_count(self):
        return len(self._listeners);

    def add_listener(self, request):
        self._listeners.append(request)

        # Write initial ok
        request.write(':ok\n\n')

        request.notifyFinish().addBoth(self.remove_listener, request)

    def remove_listener(self, reason, listener):
        self._listeners.remove(listener)

    @crochet.run_in_reactor
    def broadcast_message(self, msg, event=None):
        self.broadcast_message_async(msg, event)

    def broadcast_message_async(self, msg, event=None):
        sse = _format_sse(msg, event)
        for listener in self._listeners:
            listener.write(sse)

    def render_GET(self, request):
        request.setHeader("Content-Type", "text/event-stream")
        request.setHeader("Cache-Control", "no-cache")
        request.setHeader("Connection", "keep-alive")

        self.add_listener(request)
        return server.NOT_DONE_YET<commit_msg>Code cleanup<commit_after>import crochet
crochet.setup()

from twisted.web import resource, server

def _format_sse(msg, event=None):
    data = []

    if event is not None:
        data.append("event: {}\n".format(event))

    for line in msg.splitlines():
        data.append("data: {}\n".format(line))

    data.append("\n")

    return "".join(data)

class SSEResource(resource.Resource):

    def __init__(self):
        resource.Resource.__init__(self)
        self._listeners = []

    def get_connection_count(self):
        return len(self._listeners);

    def add_listener(self, request):
        self._listeners.append(request)

        # Write initial ok
        request.write(':ok\n\n')

        request.notifyFinish().addBoth(self.remove_listener, request)

    def remove_listener(self, reason, listener):
        self._listeners.remove(listener)

    @crochet.run_in_reactor
    def broadcast_message(self, msg, event=None):
        self.broadcast_message_async(msg, event)

    def broadcast_message_async(self, msg, event=None):
        sse = _format_sse(msg, event)
        for listener in self._listeners:
            listener.write(sse)

    def render_GET(self, request):
        request.setHeader("Content-Type", "text/event-stream")
        request.setHeader("Cache-Control", "no-cache")
        request.setHeader("Connection", "keep-alive")

        self.add_listener(request)
        return server.NOT_DONE_YET<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Fun with number caterpillars
# Number caterpillar is chain of numbers from n to 1 such that
# if number is even the next number is half
# and if number is odd the next number is plus one
# caterpillar(17) = [17, 18, 9, 10, 5, 6, 3, 4, 2, 1]
__package__ = 'raspberrypi'
__author__ = 'zuva.munshi@gmail.com'

def caterpillar(n):
    assert isinstance(n, int) and n > 0
    c = [n]
    while n > 1:
        if (n % 2 == 0):
            n = int(n / 2)
        else:
            n = n + 1
        c.append(n)
    return c

if __name__ == '__main__':
    import sys

    try:
        n = int(sys.argv[1])
    except:
        n = 1
    c = caterpillar(n)
    print(n, len(c), c)
<commit_msg>updated mersenne<commit_after>#!/usr/bin/env python3
# Fun with number caterpillars
# Number caterpillar is chain of numbers from n to 1 such that
# if number is even the next number is half
# and if number is odd the next number is plus one
# caterpillar(17) = [17, 18, 9, 10, 5, 6, 3, 4, 2, 1]
__package__ = 'raspberrypi'
__author__ = 'zuva.munshi@gmail.com'

def caterpillar(n):
    assert isinstance(n, int) and n > 0
    number_caterpillar = [n]
    while n > 1:
        if (n % 2 == 0):
            n = int(n / 2)
        else:
            n = n + 1
        number_caterpillar.append(n)
    return number_caterpillar

if __name__ == '__main__':
    import sys

    try:
        n = int(sys.argv[1])
    except:
        n = 1
    c = caterpillar(n)
    print(n, len(c), c)
<|endoftext|>


<commit_before>#FOR CLEANUP##################################################

#Add color options

#may want to add ability to read pandas data frames
#may want to add fancier plotting options

#FOR TESTING##################################################

#x=np.array([[1], [2], [3], [4]])
#x=np.array([[1, 11], [2, 12], [3, 13], [4, 14]])
#x=np.array([[1, 11, 21], [2, 12, 22], [3, 13, 33], [4, 14, 44]])
#x=np.array([[1, 11, 21, 31], [2, 12, 22, 32], [3, 13, 33, 43], [4, 14, 44, 54]])

##############################################################

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA

def plot_coords(x, *args, **kwargs):
	"""
	inputs: TxD matrix of observations
			   T-number of coords
			   D-dimensionality of each observation
			   *Nans treated as missing observations
			type (specify the type of plot)
			   see http://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes.plot for available options
	outputs: 1-, 2-, or 3-dimensional representation of the data
	"""

	def is_list(x):
		if type(x[0][0])==np.ndarray:
			return True
		elif type(x[0][0])==np.int64:
			return False

	def resize(k):
		sizes_1=np.zeros(len(k))

		for x in range(0, len(k)):
			sizes_1[x]=k[x].shape[1]

		C=max(sizes_1)
		#find the largest number of columns of all inputted arrays

		m=[]
		for idx,x in enumerate(k):
			missing=C-x.shape[1]
			add=np.zeros((x.shape[0], missing))
			y=np.append(x, add, axis=1)

			m.append(y)

		return m

	def dispatch(x):
		#determine how many dimensions (number of columns)
		if x.shape[-1]==1:
			plot1D(x)
		elif x.shape[-1]==2:
			plot2D(x)
		elif x.shape[-1]==3:
			plot3D(x)
		elif x.shape[-1]>3:
			plot3D(reduceD(x, 3))

	def dispatch_list(x):
		if x[0].shape[-1]==1:
			plot1D_list(x)
		elif x[0].shape[-1]==2:
			plot2D_list(x)
		elif x[0].shape[-1]==3:
			plot3D_list(x)
		elif x[0].shape[-1]>3:
			plot3D_list(reduceD_list(x, 3))

	def plot1D(data):
		x=np.arange(len(data))
		plot2D(np.hstack((np.transpose(x), data)))

	def plot1D_list(data):
		for i in range(0, len(data)):
			x=np.arange(len(data[i]))
			plot2D(np.hstack((np.transpose(x), data[i])))

	def plot2D(data):
		# type: (object) -> object
		#if 2d, make a scatter
		plt.plot(data[:,0], data[:,1], *args, **kwargs)

	def plot2D_list(data):
		# type: (object) -> object
		#if 2d, make a scatter
		n=len(data)
		fig,ax=plt.subplots(1,n)
		for i in range(n):
			ax[i].plot(data[i][:,0], data[i][:,1], *args, **kwargs)

	def plot3D(data):
		#if 3d, make a 3d scatter
		fig = plt.figure()
		ax = fig.add_subplot(111, projection='3d')
		ax.plot(data[:,0], data[:,1], data[:,2], *args, **kwargs)

	def plot3D_list(data):
		#if 3d, make a 3d scatter
		for i in range(0, len(data)):
			fig = plt.figure()
			ax = fig.add_subplot(111, projection='3d')
			ax.plot(data[i][:,0], data[i][:,1], data[i][:,2], *args, **kwargs)

	def reduceD(x, ndim):
		#if more than 3d, reduce to 3 (PCA), then re-run
		m = PCA(n_components=ndim, whiten=True)
		#n_components=3--> reduce to 3 dimensions
		m.fit(x)
		return m.transform(x)

	def reduceD_list(x, ndim):
		m=PCA(n_components=ndim, whiten=True)
		m.fit(x[0])
		return m.transform(x[1:])

	if is_list(x):
		dispatch_list(resize(x))
		plt.show()

		#[2] PCA over all elements
		#[3] plot

	else:
		dispatch(x)
		plt.show()

<commit_msg>updated all list plotting functions<commit_after>#FOR CLEANUP##################################################

#Add color options

#may want to add ability to read pandas data frames
#may want to add fancier plotting options

#FOR TESTING##################################################

#x=np.array([[1], [2], [3], [4]])
#x=np.array([[1, 11], [2, 12], [3, 13], [4, 14]])
#x=np.array([[1, 11, 21], [2, 12, 22], [3, 13, 33], [4, 14, 44]])
#x=np.array([[1, 11, 21, 31], [2, 12, 22, 32], [3, 13, 33, 43], [4, 14, 44, 54]])

##############################################################

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA

def plot_coords(x, *args, **kwargs):
	"""
	inputs: TxD matrix of observations
			   T-number of coords
			   D-dimensionality of each observation
			   *Nans treated as missing observations
			type (specify the type of plot)
			   see http://matplotlib.org/api/axes_api.html#matplotlib.axes.Axes.plot for available options
	outputs: 1-, 2-, or 3-dimensional representation of the data
	"""

    ##SUB FUNCTIONS##
	def is_list(x):
		if type(x[0][0])==np.ndarray:
			return True
		elif type(x[0][0])==np.int64:
			return False

	def resize(k):
		sizes_1=np.zeros(len(k))

		for x in range(0, len(k)):
			sizes_1[x]=k[x].shape[1]

		C=max(sizes_1)
		#find the largest number of columns of all inputted arrays

		m=[]
		for idx,x in enumerate(k):
			missing=C-x.shape[1]
			add=np.zeros((x.shape[0], missing))
			y=np.append(x, add, axis=1)

			m.append(y)
		return m

	def dispatch(x):
		#determine how many dimensions (number of columns)
		if x.shape[-1]==1:
			plot1D(x)
		elif x.shape[-1]==2:
			plot2D(x)
		elif x.shape[-1]==3:
			plot3D(x)
		elif x.shape[-1]>3:
			plot3D(reduceD(x, 3))

	def dispatch_list(x):
		if x[0].shape[-1]==1:
			plot1D_list(x)
		elif x[0].shape[-1]==2:
			plot2D_list(x)
		elif x[0].shape[-1]==3:
			plot3D_list(x)
		elif x[0].shape[-1]>3:
			plot3D_list(reduceD_list(x, 3))

	def plot1D(data):
		x=np.arange(len(data))
		plot2D(np.hstack((np.transpose(x), data)))

	def plot1D_list(data):
		x=[]
		for i in range(0, len(data)):
			x.append(np.transpose(np.arange(len(data[i]))))
		plot2D(np.hstack((x), data))

	def plot2D(data):
		# type: (object) -> object
		#if 2d, make a scatter
		plt.plot(data[:,0], data[:,1], *args, **kwargs)

	def plot2D_list(data):
		# type: (object) -> object
		#if 2d, make a scatter
		n=len(data)
		fig,ax=plt.subplots(1,n)
		for i in range(n):
			ax[i].plot(data[i][:,0], data[i][:,1], *args, **kwargs)

	def plot3D(data):
		#if 3d, make a 3d scatter
		fig = plt.figure()
		ax = fig.add_subplot(111, projection='3d')
		ax.plot(data[:,0], data[:,1], data[:,2], *args, **kwargs)

	def plot3D_list(data):
		#if 3d, make a 3d scatter
		n=len(data)
		fig,ax=plt.subplots(1,n, subplot_kw={'projection':'3d'})
		for i in range(n):
			#fig = plt.figure()
			#ax = fig.add_subplot(111, projection='3d')
			ax[i].plot(data[i][:,0], data[i][:,1], data[i][:,2], *args, **kwargs)

	def reduceD(x, ndim):
		#if more than 3d, reduce to 3 (PCA), then re-run
		m = PCA(n_components=ndim, whiten=True)
		#n_components=3--> reduce to 3 dimensions
		m.fit(x)
		return m.transform(x)

	def reduceD_list(x, ndim):
		m=PCA(n_components=ndim, whiten=True)
		m.fit(x[0])
		return m.transform(x[1:])

    ##MAIN FUNCTION##

	if is_list(x):
		dispatch_list(resize(x))
		plt.show()

	else:
		dispatch(x)
		plt.show()

<|endoftext|>


<commit_before>##
## Author(s):
##  - Cedric GESTES <gestes@aldebaran-robotics.com>
##  - Pierre ROULLON <proullon@aldebaran-robotics.com>
##
## Copyright (C) 2010 - 2013 Aldebaran Robotics
##

""" QiMessaging Python bindings

Provided features are very close to C++, Python style.
"""

def load_lib_qipyessaging():
    """ Load _qipyessaging.so and its dependencies.

    This makes _qipyessaging usable from a relocatable
    SDK without having to set LD_LIBRARY_PATH
    """
    import ctypes
    import os
    deps = [
            "libboost_python.so",
            "libboost_system.so",
            "libboost_chrono.so",
            "libboost_program_options.so",
            "libboost_thread.so",
            "libboost_filesystem.so",
            "libboost_regex.so",
            "libboost_locale.so",
            "libboost_signals.so",
            "libqi.so",
            "libqitype.so",
            "libqimessaging.so",
    ]
    this_dir = os.path.abspath(os.path.dirname(__file__))
    for dep in deps:
        full_path = os.path.join(this_dir, "..", dep)
        try:
            ctypes.cdll.LoadLibrary(full_path)
        except Exception:
            pass

def _on_import_module():
    import sys
    import atexit
    if sys.platform.startswith("linux"):
        load_lib_qipyessaging()

    atexit.register(_stopApplication)

#######

from _qi import Application as _Application
from _qi import ApplicationSession as _ApplicationSession
from _qi import ( FutureState, FutureTimeout, Future, Promise,
                  Property, Session, Signal,
                  createObject, registerObjectFactory,
                  async, PeriodicTask)
from . import path
from ._type import ( Void, Bool,
                     Int8, UInt8,
                     Int16, UInt16,
                     Int32, UInt32,
                     Int64, UInt64,
                     Float, Double,
                     String, List,
                     Map, Struct,
                     Object, Dynamic,
                     Buffer, AnyArguments,
                     typeof, _isinstance)
from ._binder import bind, nobind
from .logging import fatal, error, warning, info, verbose, Logger
from .translator import defaultTranslator, tr, Translator

#rename isinstance here. (isinstance should not be used in this file)
isinstance = _isinstance

_app = None

#we want to stop all thread before python start destroying
#module and the like. (this avoid callback calling python while
#it's destroying)
def _stopApplication():
    global _app
    if _app is not None:
        _app.stop()
        del _app
        _app = None

#application is a singleton, it should live till the end of the program
#because it own eventloops
def ApplicationSession(args=None):
    global _app
    if _app is None:
        if args is None:
            _app = _ApplicationSession()
        else:
            _app = _ApplicationSession(args)
    else:
        raise Exception("Application was already initialized")
    return _app

def Application(args=None):
    global _app
    if _app is None:
        if args is None:
            _app = _Application()
        else:
            _app = _Application(args)
    else:
        raise Exception("Application was already initialized")
    return _app

__all__ = ["FutureState",
           "FutureTimeout",
           "Future",
           "Promise",
           "Property",
           "Session",
           "Signal",
           "createObject",
           "registerObjectFactory",
           "async",

           "Void", "Bool", "Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64",
           "Float", "Double", "String", "List", "Map", "Struct", "Object", "Dynamic", "Buffer", "AnyArguments",
           "typeof", "isinstance",
           "bind", "nobind",
           "fatal", "error", "warning", "info", "verbose", "Logger",
           "defaultTranslator", "tr", "Translator"
]

_on_import_module()
<commit_msg>python: fix import libqipython and change init order<commit_after>##
## Author(s):
##  - Cedric GESTES <gestes@aldebaran-robotics.com>
##  - Pierre ROULLON <proullon@aldebaran-robotics.com>
##
## Copyright (C) 2010 - 2013 Aldebaran Robotics
##

""" QiMessaging Python bindings

Provided features are very close to C++, Python style.
"""

def load_lib_qipyessaging():
    """ Load _qipyessaging.so and its dependencies.

    This makes _qipyessaging usable from a relocatable
    SDK without having to set LD_LIBRARY_PATH
    """
    import ctypes
    import os
    deps = [
            "libboost_python.so",
            "libboost_system.so",
            "libboost_chrono.so",
            "libboost_program_options.so",
            "libboost_thread.so",
            "libboost_filesystem.so",
            "libboost_regex.so",
            "libboost_locale.so",
            "libboost_signals.so",
            "libqi.so",
            "libqitype.so",
            "libqimessaging.so",
            "libqipython.so",
    ]
    this_dir = os.path.abspath(os.path.dirname(__file__))
    for dep in deps:
        full_path = os.path.join(this_dir, "..", dep)
        try:
            ctypes.cdll.LoadLibrary(full_path)
        except Exception:
            pass

def _on_import_module():
    import sys
    if sys.platform.startswith("linux"):
        load_lib_qipyessaging()

#######
_on_import_module()

from _qi import Application as _Application
from _qi import ApplicationSession as _ApplicationSession
from _qi import ( FutureState, FutureTimeout, Future, Promise,
                  Property, Session, Signal,
                  createObject, registerObjectFactory,
                  async, PeriodicTask)
from . import path
from ._type import ( Void, Bool,
                     Int8, UInt8,
                     Int16, UInt16,
                     Int32, UInt32,
                     Int64, UInt64,
                     Float, Double,
                     String, List,
                     Map, Struct,
                     Object, Dynamic,
                     Buffer, AnyArguments,
                     typeof, _isinstance)
from ._binder import bind, nobind
from .logging import fatal, error, warning, info, verbose, Logger
from .translator import defaultTranslator, tr, Translator

#rename isinstance here. (isinstance should not be used in this file)
isinstance = _isinstance

_app = None

#we want to stop all thread before python start destroying
#module and the like. (this avoid callback calling python while
#it's destroying)
def _stopApplication():
    global _app
    if _app is not None:
        _app.stop()
        del _app
        _app = None

#application is a singleton, it should live till the end of the program
#because it own eventloops
def ApplicationSession(args=None):
    global _app
    if _app is None:
        if args is None:
            _app = _ApplicationSession()
        else:
            _app = _ApplicationSession(args)
    else:
        raise Exception("Application was already initialized")
    return _app

def Application(args=None):
    global _app
    if _app is None:
        if args is None:
            _app = _Application()
        else:
            _app = _Application(args)
    else:
        raise Exception("Application was already initialized")
    return _app

__all__ = ["FutureState",
           "FutureTimeout",
           "Future",
           "Promise",
           "Property",
           "Session",
           "Signal",
           "createObject",
           "registerObjectFactory",
           "async",
           "Void", "Bool", "Int8", "UInt8", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64",
           "Float", "Double", "String", "List", "Map", "Struct", "Object", "Dynamic", "Buffer", "AnyArguments",
           "typeof", "isinstance",
           "bind", "nobind",
           "fatal", "error", "warning", "info", "verbose", "Logger",
           "defaultTranslator", "tr", "Translator"
]

import atexit
atexit.register(_stopApplication)
# Do not pollute namespace
del atexit
<|endoftext|>


<commit_before>"""
Copyright (c) 2011, 2012, Regents of the University of California
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

 - Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
 - Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the
   distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
OF THE POSSIBILITY OF SUCH DAMAGE.
"""
"""
@author Stephen Dawson-Haggerty <stevedh@eecs.berkeley.edu>
"""

import os
import sys
import json
import uuid
from cStringIO import StringIO
from avro import schema, io

import util
import pkgutil

NAMESPACE = "edu.berkeley.cs.local"
SCHEMA_NAMES = schema.Names(default_namespace=NAMESPACE)
SCHEMAS = [
    # new extension types
    "uuid",
    "UnitofTime", "Duration",
    "ReadingType", "ReadingValue",

    'InstrumentMetadata', 'LocationMetadata', 'OperatorMetadata',

    # timeseries subobjects
    "Actuator",
    "Properties", "Metadata",
    "TimeSeries", "Collection",
    "Reporting",

    ]
SCHEMA_OBJECTS = []

def load_schema(sf):
    dirs = [os.path.dirname(sys.modules[__name__].__file__),
            os.path.join(os.path.dirname(sys.modules[__name__].__file__),
                         os.path.join(os.pardir, os.pardir)),
            sys.prefix]
    #print dirs
    obj = None
    for d in dirs:
        try:
            path = os.path.join(d, 'schema', sf.lower() + '.av')
            obj = json.load(open(path, 'r'))
            break
        except:
            pass
    if obj == None:
        raise Exception('Cannot load schema: ' + sf.lower())

    s = schema.make_avsc_object(obj, SCHEMA_NAMES)
    return s

# load all the schemas when we check an object
for sf in SCHEMAS:
    SCHEMA_OBJECTS.append(load_schema(sf))
REPORT_SCHEMA = load_schema('ReportData')

def convert_uuids(obj):
    id = obj.get('uuid', None)
    if util.is_string(id):
        id = uuid.UUID(id)
    if id: obj['uuid'] = id.bytes
    return id

def stringify_uuids(obj):
    id = obj.get('uuid', None)
    if id:
        id = uuid.UUID(bytes=id)
        obj['uuid'] = str(id)

def convert_readings(obj):
    r = obj.get('Readings', None)
    if not r: return
    for i, val in enumerate(r):
        if r and not isinstance(val, list):
            obj['Readings'][i] = list(val)

def remove_none(obj):
    dellist = []
    for k, v in obj.iteritems():
        if v == None: dellist.append(k)
        elif isinstance(v, dict):
            remove_none(v)
    for k in dellist:
        del obj[k]

def validate(schema, obj):
    """Validate an object against its schema.

    Right now, this just checks it against the Avro schema; however in
    the future we will want to impose additional constraints which
    aren't expressable in the schema.
    """
    if schema == 'uuid' and isinstance(obj, uuid.UUID):
        return True
    elif schema == 'Readings':
        return True

    s = SCHEMA_NAMES.get_name(schema, None)
    # swap the uuid for the byte-packed encoding we use with avro
    try:
        id = convert_uuids(obj)
        rv = io.validate(s, obj)
        if id: obj['uuid'] = id
        return rv
    except:
        return False

def filter_fields(schema, obj):
    rv = {}
    schema = util.find(lambda x: x.get("name", "") == schema, SCHEMA_OBJECTS)
    if not schema:
        return obj

    for field in [x['name'] for x in schema['fields']]:
        if field in obj:
            rv[field] = obj[field]

    return rv

def dump_report(datum):
    # have to diddle with some of the values so avro doesn't choke

    uuids = map(convert_uuids, datum.itervalues())
    map(convert_readings, datum.itervalues())

    # then just dump it to a string
    out = StringIO()
    dwriter = io.DatumWriter(writers_schema=REPORT_SCHEMA)
    dwriter.write(datum, io.BinaryEncoder(out))

    for id, p in zip(uuids, datum.itervalues()):
        if id: p['uuid'] = id

    return out.getvalue()

def load_report(data):
    input = StringIO(data)
    dreader = io.DatumReader(writers_schema=REPORT_SCHEMA,
                             readers_schema=REPORT_SCHEMA)
    v = dreader.read(io.BinaryDecoder(input))
    map(stringify_uuids, v.itervalues())
    remove_none(v)
    print v
    return v

# validate("Timeseries", None)

<commit_msg>Fix bug which prevents report being rendered.<commit_after>"""
Copyright (c) 2011, 2012, Regents of the University of California
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

 - Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
 - Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the
   distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
OF THE POSSIBILITY OF SUCH DAMAGE.
"""
"""
@author Stephen Dawson-Haggerty <stevedh@eecs.berkeley.edu>
"""

import os
import sys
import json
import uuid
from cStringIO import StringIO
from avro import schema, io

import util
import pkgutil

NAMESPACE = "edu.berkeley.cs.local"
SCHEMA_NAMES = schema.Names(default_namespace=NAMESPACE)
SCHEMAS = [
    # new extension types
    "uuid",
    "UnitofTime", "Duration",
    "ReadingType", "ReadingValue",

    'InstrumentMetadata', 'LocationMetadata', 'OperatorMetadata',

    # timeseries subobjects
    "Actuator",
    "Properties", "Metadata",
    "TimeSeries", "Collection",
    "Reporting",

    ]
SCHEMA_OBJECTS = []

def load_schema(sf):
    dirs = [os.path.dirname(sys.modules[__name__].__file__),
            os.path.join(os.path.dirname(sys.modules[__name__].__file__),
                         os.path.join(os.pardir, os.pardir)),
            sys.prefix]
    #print dirs
    obj = None
    for d in dirs:
        try:
            path = os.path.join(d, 'schema', sf.lower() + '.av')
            obj = json.load(open(path, 'r'))
            break
        except:
            pass
    if obj == None:
        raise Exception('Cannot load schema: ' + sf.lower())

    s = schema.make_avsc_object(obj, SCHEMA_NAMES)
    return s

# load all the schemas when we check an object
for sf in SCHEMAS:
    SCHEMA_OBJECTS.append(load_schema(sf))
REPORT_SCHEMA = load_schema('ReportData')

def convert_uuids(obj):
    id = obj.get('uuid', None)
    if util.is_string(id):
        id = uuid.UUID(id)
    if id: obj['uuid'] = id.bytes
    return id

def stringify_uuids(obj):
    id = obj.get('uuid', None)
    if id:
        id = uuid.UUID(bytes=id)
        obj['uuid'] = str(id)

def convert_readings(obj):
    r = obj.get('Readings', None)
    if not r: return
    for i, val in enumerate(r):
        if r and not isinstance(val, list):
            obj['Readings'][i] = list(val)

def remove_none(obj):
    dellist = []
    for k, v in obj.iteritems():
        if v == None: dellist.append(k)
        elif isinstance(v, dict):
            remove_none(v)
    for k in dellist:
        del obj[k]

def validate(schema, obj):
    """Validate an object against its schema.

    Right now, this just checks it against the Avro schema; however in
    the future we will want to impose additional constraints which
    aren't expressable in the schema.
    """
    if schema == 'uuid' and isinstance(obj, uuid.UUID):
        return True
    elif schema == 'Readings':
        return True

    s = SCHEMA_NAMES.get_name(schema, None)
    # swap the uuid for the byte-packed encoding we use with avro
    try:
        id = convert_uuids(obj)
        rv = io.validate(s, obj)
        if id: obj['uuid'] = id
        return rv
    except:
        return False

def filter_fields(schema, obj):
    rv = {}
    schema = util.find(lambda x: x.name == schema, SCHEMA_OBJECTS)
    if not schema:
        return obj

    for field in [x.name for x in schema.fields]:
        if field in obj:
            rv[field] = obj[field]

    return rv

def dump_report(datum):
    # have to diddle with some of the values so avro doesn't choke

    uuids = map(convert_uuids, datum.itervalues())
    map(convert_readings, datum.itervalues())

    # then just dump it to a string
    out = StringIO()
    dwriter = io.DatumWriter(writers_schema=REPORT_SCHEMA)
    dwriter.write(datum, io.BinaryEncoder(out))

    for id, p in zip(uuids, datum.itervalues()):
        if id: p['uuid'] = id

    return out.getvalue()

def load_report(data):
    input = StringIO(data)
    dreader = io.DatumReader(writers_schema=REPORT_SCHEMA,
                             readers_schema=REPORT_SCHEMA)
    v = dreader.read(io.BinaryDecoder(input))
    map(stringify_uuids, v.itervalues())
    remove_none(v)
    print v
    return v

# validate("Timeseries", None)

<|endoftext|>