<commit_before>#!/usr/bin/env python
"""
|jedi| is mostly being tested by what I would call "Blackbox Tests". These
tests are just testing the interface and do input/output testing. This makes a
lot of sense for |jedi|. Jedi supports so many different code structures, that
it is just stupid to write 200'000 unittests in the manner of
``regression.py``. Also, it is impossible to do doctests/unittests on most of
the internal data structures. That's why |jedi| uses mostly these kind of
tests.

There are different kind of tests:

- completions / goto_definitions ``#?``
- goto_assignments: ``#!``
- usages: ``#<``

How to run tests?
+++++++++++++++++

Jedi uses pytest_ to run unit and integration tests.  To run tests,
simply run ``py.test``.  You can also use tox_ to run tests for
multiple Python versions.

.. _pytest: http://pytest.org
.. _tox: http://testrun.org/tox

Integration test cases are located in ``test/completion`` directory
and each test cases are indicated by the comment ``#?`` (completions /
definitions), ``#!`` (assignments) and ``#<`` (usages).  There is also
support for third party libraries. In a normal test run they are not
being executed, you have to provide a ``--thirdparty`` option.

In addition to standard `-k` and `-m` options in py.test, you can use
`-T` (`--test-files`) option to specify integration test cases to run.
It takes the format of ``FILE_NAME[:LINE[,LINE[,...]]]`` where
``FILE_NAME`` is a file in ``test/completion`` and ``LINE`` is a line
number of the test comment.  Here is some recipes:

Run tests only in ``basic.py`` and ``imports.py``::

    py.test test/test_integration.py -T basic.py -T imports.py

Run test at line 4, 6, and 8 in ``basic.py``::

    py.test test/test_integration.py -T basic.py:4,6,8

See ``py.test --help`` for more information.

If you want to debug a test, just use the ``--pdb`` option.

Alternate Test Runner
+++++++++++++++++++++

If you don't like the output of ``py.test``, there's an alternate test runner
that you can start by running ``./run.py``. The above example could be run by::

    ./run.py basic 4 6 8

The advantage of this runner is simplicity and more customized error reports.
Using both runners will help you to have a quicker overview of what's
happening.

Auto-Completion
+++++++++++++++

Uses comments to specify a test in the next line. The comment says, which
results are expected. The comment always begins with `#?`. The last row
symbolizes the cursor.

For example::

    #? ['real']
    a = 3; a.rea

Because it follows ``a.rea`` and a is an ``int``, which has a ``real``
property.

Goto Definitions
++++++++++++++++

Definition tests use the same symbols like completion tests. This is
possible because the completion tests are defined with a list::

    #? int()
    ab = 3; ab

Goto Assignments
++++++++++++++++

Tests look like this::

    abc = 1
    #! ['abc=1']
    abc

Additionally it is possible to add a number which describes to position of
the test (otherwise it's just end of line)::

    #! 2 ['abc=1']
    abc

Usages
++++++

Tests look like this::

    abc = 1
    #< abc@1,0 abc@3,0
    abc
"""
import os
import re
from ast import literal_eval
from io import StringIO
from functools import reduce

import jedi
from jedi._compatibility import unicode, is_py3

TEST_COMPLETIONS = 0
TEST_DEFINITIONS = 1
TEST_ASSIGNMENTS = 2
TEST_USAGES = 3

class IntegrationTestCase(object):
    def __init__(self, test_type, correct, line_nr, column, start, line,
                 path=None):
        self.test_type = test_type
        self.correct = correct
        self.line_nr = line_nr
        self.column = column
        self.start = start
        self.line = line
        self.path = path
        self.skip = None

    @property
    def module_name(self):
        return re.sub('.*/|\.py$', '', self.path)

    @property
    def line_nr_test(self):
        """The test is always defined on the line before."""
        return self.line_nr - 1

    def __repr__(self):
        return '<%s: %s:%s:%s>' % (self.__class__.__name__, self.module_name,
                                   self.line_nr_test, self.line.rstrip())

    def script(self):
        return jedi.Script(self.source, self.line_nr, self.column, self.path)

    def run(self, compare_cb):
        testers = {
            TEST_COMPLETIONS: self.run_completion,
            TEST_DEFINITIONS: self.run_goto_definitions,
            TEST_ASSIGNMENTS: self.run_goto_assignments,
            TEST_USAGES: self.run_usages,
        }
        return testers[self.test_type](compare_cb)

    def run_completion(self, compare_cb):
        completions = self.script().completions()
        #import cProfile; cProfile.run('script.completions()')

        comp_str = set([c.name for c in completions])
        return compare_cb(self, comp_str, set(literal_eval(self.correct)))

    def run_goto_definitions(self, compare_cb):
        def definition(correct, correct_start, path):
            def defs(line_nr, indent):
                s = jedi.Script(self.source, line_nr, indent, path)
                return set(s.goto_definitions())

            should_be = set()
            number = 0
            for index in re.finditer('(?: +|$)', correct):
                if correct == ' ':
                    continue
                # -1 for the comment, +3 because of the comment start `#? `
                start = index.start()
                number += 1
                try:
                    should_be |= defs(self.line_nr - 1, start + correct_start)
                except Exception:
                    print('could not resolve %s indent %s'
                          % (self.line_nr - 1, start))
                    raise
            # because the objects have different ids, `repr`, then compare.
            should_str = set(r.desc_with_module for r in should_be)
            if len(should_str) < number:
                raise Exception('Solution @%s not right, '
                   'too few test results: %s' % (self.line_nr - 1, should_str))
            return should_str

        script = self.script()
        should_str = definition(self.correct, self.start, script.path)
        result = script.goto_definitions()
        is_str = set(r.desc_with_module for r in result)
        return compare_cb(self, is_str, should_str)

    def run_goto_assignments(self, compare_cb):
        result = self.script().goto_assignments()
        comp_str = str(sorted(str(r.description) for r in result))
        return compare_cb(self, comp_str, self.correct)

    def run_usages(self, compare_cb):
        result = self.script().usages()
        self.correct = self.correct.strip()
        compare = sorted((r.module_name, r.line, r.column) for r in result)
        wanted = []
        if not self.correct:
            positions = []
        else:
            positions = literal_eval(self.correct)
        for pos_tup in positions:
            if type(pos_tup[0]) == str:
                # this means that there is a module specified
                wanted.append(pos_tup)
            else:
                line = pos_tup[0]
                if pos_tup[0] is not None:
                    line += self.line_nr
                wanted.append((self.module_name, line, pos_tup[1]))

        return compare_cb(self, compare, sorted(wanted))

def collect_file_tests(lines, lines_to_execute):
    makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column,
                                             start, line)
    start = None
    correct = None
    test_type = None
    for line_nr, line in enumerate(lines, 1):
        if correct:
            r = re.match('^(\d+)\s*(.*)$', correct)
            if r:
                column = int(r.group(1))
                correct = r.group(2)
                start += r.regs[2][0]  # second group, start index
            else:
                column = len(line) - 1  # -1 for the \n
            if test_type == '!':
                yield makecase(TEST_ASSIGNMENTS)
            elif test_type == '<':
                yield makecase(TEST_USAGES)
            elif correct.startswith('['):
                yield makecase(TEST_COMPLETIONS)
            else:
                yield makecase(TEST_DEFINITIONS)
            correct = None
        else:
            try:
                r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]+)', line)
                # test_type is ? for completion and ! for goto_assignments
                test_type = r.group(1)
                correct = r.group(2)
                start = r.start()
            except AttributeError:
                correct = None
            else:
                # skip the test, if this is not specified test
                if lines_to_execute and line_nr not in lines_to_execute:
                    correct = None

def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
    for f_name in os.listdir(base_dir):
        files_to_execute = [a for a in test_files.items() if a[0] in f_name]
        lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, [])
        if f_name.endswith(".py") and (not test_files or files_to_execute):
            skip = None
            if check_thirdparty:
                lib = f_name.replace('_.py', '')
                try:
                    # there is always an underline at the end.
                    # It looks like: completion/thirdparty/pylab_.py
                    __import__(lib)
                except ImportError:
                    skip = 'Thirdparty-Library %s not found.' % lib

            path = os.path.join(base_dir, f_name)
            source = open(path).read()
            if not is_py3:
                source = unicode(source, 'UTF-8')
            for case in collect_file_tests(StringIO(source),
                                           lines_to_execute):
                case.path = path
                case.source = source
                if skip:
                    case.skip = skip
                yield case

docoptstr = """
Using run.py to make debugging easier with integration tests.

An alternative testing format, which is much more hacky, but very nice to
work with.

Usage:
    run.py [--pdb] [--debug] [--thirdparty] [<rest>...]
    run.py --help

Options:
    -h --help       Show this screen.
    --pdb           Enable pdb debugging on fail.
    -d, --debug     Enable text output debugging (please install ``colorama``).
    --thirdparty    Also run thirdparty tests (in ``completion/thirdparty``).
"""
if __name__ == '__main__':
    import docopt
    arguments = docopt.docopt(docoptstr)

    import time
    t_start = time.time()
    # Sorry I didn't use argparse here. It's because argparse is not in the
    # stdlib in 2.5.
    import sys

    if arguments['--debug']:
        jedi.set_debug_function()

    # get test list, that should be executed
    test_files = {}
    last = None
    for arg in arguments['<rest>']:
        if arg.isdigit():
            if last is None:
                continue
            test_files[last].append(int(arg))
        else:
            test_files[arg] = []
            last = arg

    # completion tests:
    completion_test_dir = '../test/completion'
    summary = []
    tests_fail = 0

    # execute tests
    cases = list(collect_dir_tests(completion_test_dir, test_files))
    if test_files or arguments['--thirdparty']:
        completion_test_dir += '/thirdparty'
        cases += collect_dir_tests(completion_test_dir, test_files, True)

    def file_change(current, tests, fails):
        if current is not None:
            current = os.path.basename(current)
        print('%s \t\t %s tests and %s fails.' % (current, tests, fails))

    def report(case, actual, desired):
        if actual == desired:
            return 0
        else:
            print("\ttest fail @%d, actual = %s, desired = %s"
                  % (case.line_nr - 1, actual, desired))
            return 1

    import traceback
    current = cases[0].path if cases else None
    count = fails = 0
    for c in cases:
        if current != c.path:
            file_change(current, count, fails)
            current = c.path
            count = fails = 0

        try:
            if c.run(report):
                tests_fail += 1
                fails += 1
        except Exception:
            traceback.print_exc()
            print("\ttest fail @%d" % (c.line_nr - 1))
            tests_fail += 1
            fails += 1
            if arguments['--pdb']:
                import pdb
                pdb.post_mortem()

        count += 1

    file_change(current, count, fails)

    print('\nSummary: (%s fails of %s tests) in %.3fs' % (tests_fail,
                                        len(cases), time.time() - t_start))
    for s in summary:
        print(s)

    exit_code = 1 if tests_fail else 0
    sys.exit(exit_code)
<commit_msg>fix module name of integration tests for Windows<commit_after>#!/usr/bin/env python
"""
|jedi| is mostly being tested by what I would call "Blackbox Tests". These
tests are just testing the interface and do input/output testing. This makes a
lot of sense for |jedi|. Jedi supports so many different code structures, that
it is just stupid to write 200'000 unittests in the manner of
``regression.py``. Also, it is impossible to do doctests/unittests on most of
the internal data structures. That's why |jedi| uses mostly these kind of
tests.

There are different kind of tests:

- completions / goto_definitions ``#?``
- goto_assignments: ``#!``
- usages: ``#<``

How to run tests?
+++++++++++++++++

Jedi uses pytest_ to run unit and integration tests.  To run tests,
simply run ``py.test``.  You can also use tox_ to run tests for
multiple Python versions.

.. _pytest: http://pytest.org
.. _tox: http://testrun.org/tox

Integration test cases are located in ``test/completion`` directory
and each test cases are indicated by the comment ``#?`` (completions /
definitions), ``#!`` (assignments) and ``#<`` (usages).  There is also
support for third party libraries. In a normal test run they are not
being executed, you have to provide a ``--thirdparty`` option.

In addition to standard `-k` and `-m` options in py.test, you can use
`-T` (`--test-files`) option to specify integration test cases to run.
It takes the format of ``FILE_NAME[:LINE[,LINE[,...]]]`` where
``FILE_NAME`` is a file in ``test/completion`` and ``LINE`` is a line
number of the test comment.  Here is some recipes:

Run tests only in ``basic.py`` and ``imports.py``::

    py.test test/test_integration.py -T basic.py -T imports.py

Run test at line 4, 6, and 8 in ``basic.py``::

    py.test test/test_integration.py -T basic.py:4,6,8

See ``py.test --help`` for more information.

If you want to debug a test, just use the ``--pdb`` option.

Alternate Test Runner
+++++++++++++++++++++

If you don't like the output of ``py.test``, there's an alternate test runner
that you can start by running ``./run.py``. The above example could be run by::

    ./run.py basic 4 6 8

The advantage of this runner is simplicity and more customized error reports.
Using both runners will help you to have a quicker overview of what's
happening.

Auto-Completion
+++++++++++++++

Uses comments to specify a test in the next line. The comment says, which
results are expected. The comment always begins with `#?`. The last row
symbolizes the cursor.

For example::

    #? ['real']
    a = 3; a.rea

Because it follows ``a.rea`` and a is an ``int``, which has a ``real``
property.

Goto Definitions
++++++++++++++++

Definition tests use the same symbols like completion tests. This is
possible because the completion tests are defined with a list::

    #? int()
    ab = 3; ab

Goto Assignments
++++++++++++++++

Tests look like this::

    abc = 1
    #! ['abc=1']
    abc

Additionally it is possible to add a number which describes to position of
the test (otherwise it's just end of line)::

    #! 2 ['abc=1']
    abc

Usages
++++++

Tests look like this::

    abc = 1
    #< abc@1,0 abc@3,0
    abc
"""
import os
import re
from ast import literal_eval
from io import StringIO
from functools import reduce

import jedi
from jedi._compatibility import unicode, is_py3

TEST_COMPLETIONS = 0
TEST_DEFINITIONS = 1
TEST_ASSIGNMENTS = 2
TEST_USAGES = 3

class IntegrationTestCase(object):
    def __init__(self, test_type, correct, line_nr, column, start, line,
                 path=None):
        self.test_type = test_type
        self.correct = correct
        self.line_nr = line_nr
        self.column = column
        self.start = start
        self.line = line
        self.path = path
        self.skip = None

    @property
    def module_name(self):
        return os.path.splitext(os.path.basename(self.path))[0]

    @property
    def line_nr_test(self):
        """The test is always defined on the line before."""
        return self.line_nr - 1

    def __repr__(self):
        return '<%s: %s:%s:%s>' % (self.__class__.__name__, self.module_name,
                                   self.line_nr_test, self.line.rstrip())

    def script(self):
        return jedi.Script(self.source, self.line_nr, self.column, self.path)

    def run(self, compare_cb):
        testers = {
            TEST_COMPLETIONS: self.run_completion,
            TEST_DEFINITIONS: self.run_goto_definitions,
            TEST_ASSIGNMENTS: self.run_goto_assignments,
            TEST_USAGES: self.run_usages,
        }
        return testers[self.test_type](compare_cb)

    def run_completion(self, compare_cb):
        completions = self.script().completions()
        #import cProfile; cProfile.run('script.completions()')

        comp_str = set([c.name for c in completions])
        return compare_cb(self, comp_str, set(literal_eval(self.correct)))

    def run_goto_definitions(self, compare_cb):
        def definition(correct, correct_start, path):
            def defs(line_nr, indent):
                s = jedi.Script(self.source, line_nr, indent, path)
                return set(s.goto_definitions())

            should_be = set()
            number = 0
            for index in re.finditer('(?: +|$)', correct):
                if correct == ' ':
                    continue
                # -1 for the comment, +3 because of the comment start `#? `
                start = index.start()
                number += 1
                try:
                    should_be |= defs(self.line_nr - 1, start + correct_start)
                except Exception:
                    print('could not resolve %s indent %s'
                          % (self.line_nr - 1, start))
                    raise
            # because the objects have different ids, `repr`, then compare.
            should_str = set(r.desc_with_module for r in should_be)
            if len(should_str) < number:
                raise Exception('Solution @%s not right, '
                   'too few test results: %s' % (self.line_nr - 1, should_str))
            return should_str

        script = self.script()
        should_str = definition(self.correct, self.start, script.path)
        result = script.goto_definitions()
        is_str = set(r.desc_with_module for r in result)
        return compare_cb(self, is_str, should_str)

    def run_goto_assignments(self, compare_cb):
        result = self.script().goto_assignments()
        comp_str = str(sorted(str(r.description) for r in result))
        return compare_cb(self, comp_str, self.correct)

    def run_usages(self, compare_cb):
        result = self.script().usages()
        self.correct = self.correct.strip()
        compare = sorted((r.module_name, r.line, r.column) for r in result)
        wanted = []
        if not self.correct:
            positions = []
        else:
            positions = literal_eval(self.correct)
        for pos_tup in positions:
            if type(pos_tup[0]) == str:
                # this means that there is a module specified
                wanted.append(pos_tup)
            else:
                line = pos_tup[0]
                if pos_tup[0] is not None:
                    line += self.line_nr
                wanted.append((self.module_name, line, pos_tup[1]))

        return compare_cb(self, compare, sorted(wanted))

def collect_file_tests(lines, lines_to_execute):
    makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column,
                                             start, line)
    start = None
    correct = None
    test_type = None
    for line_nr, line in enumerate(lines, 1):
        if correct:
            r = re.match('^(\d+)\s*(.*)$', correct)
            if r:
                column = int(r.group(1))
                correct = r.group(2)
                start += r.regs[2][0]  # second group, start index
            else:
                column = len(line) - 1  # -1 for the \n
            if test_type == '!':
                yield makecase(TEST_ASSIGNMENTS)
            elif test_type == '<':
                yield makecase(TEST_USAGES)
            elif correct.startswith('['):
                yield makecase(TEST_COMPLETIONS)
            else:
                yield makecase(TEST_DEFINITIONS)
            correct = None
        else:
            try:
                r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]+)', line)
                # test_type is ? for completion and ! for goto_assignments
                test_type = r.group(1)
                correct = r.group(2)
                start = r.start()
            except AttributeError:
                correct = None
            else:
                # skip the test, if this is not specified test
                if lines_to_execute and line_nr not in lines_to_execute:
                    correct = None

def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
    for f_name in os.listdir(base_dir):
        files_to_execute = [a for a in test_files.items() if a[0] in f_name]
        lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, [])
        if f_name.endswith(".py") and (not test_files or files_to_execute):
            skip = None
            if check_thirdparty:
                lib = f_name.replace('_.py', '')
                try:
                    # there is always an underline at the end.
                    # It looks like: completion/thirdparty/pylab_.py
                    __import__(lib)
                except ImportError:
                    skip = 'Thirdparty-Library %s not found.' % lib

            path = os.path.join(base_dir, f_name)
            source = open(path).read()
            if not is_py3:
                source = unicode(source, 'UTF-8')
            for case in collect_file_tests(StringIO(source),
                                           lines_to_execute):
                case.path = path
                case.source = source
                if skip:
                    case.skip = skip
                yield case

docoptstr = """
Using run.py to make debugging easier with integration tests.

An alternative testing format, which is much more hacky, but very nice to
work with.

Usage:
    run.py [--pdb] [--debug] [--thirdparty] [<rest>...]
    run.py --help

Options:
    -h --help       Show this screen.
    --pdb           Enable pdb debugging on fail.
    -d, --debug     Enable text output debugging (please install ``colorama``).
    --thirdparty    Also run thirdparty tests (in ``completion/thirdparty``).
"""
if __name__ == '__main__':
    import docopt
    arguments = docopt.docopt(docoptstr)

    import time
    t_start = time.time()
    # Sorry I didn't use argparse here. It's because argparse is not in the
    # stdlib in 2.5.
    import sys

    if arguments['--debug']:
        jedi.set_debug_function()

    # get test list, that should be executed
    test_files = {}
    last = None
    for arg in arguments['<rest>']:
        if arg.isdigit():
            if last is None:
                continue
            test_files[last].append(int(arg))
        else:
            test_files[arg] = []
            last = arg

    # completion tests:
    completion_test_dir = '../test/completion'
    summary = []
    tests_fail = 0

    # execute tests
    cases = list(collect_dir_tests(completion_test_dir, test_files))
    if test_files or arguments['--thirdparty']:
        completion_test_dir += '/thirdparty'
        cases += collect_dir_tests(completion_test_dir, test_files, True)

    def file_change(current, tests, fails):
        if current is not None:
            current = os.path.basename(current)
        print('%s \t\t %s tests and %s fails.' % (current, tests, fails))

    def report(case, actual, desired):
        if actual == desired:
            return 0
        else:
            print("\ttest fail @%d, actual = %s, desired = %s"
                  % (case.line_nr - 1, actual, desired))
            return 1

    import traceback
    current = cases[0].path if cases else None
    count = fails = 0
    for c in cases:
        if current != c.path:
            file_change(current, count, fails)
            current = c.path
            count = fails = 0

        try:
            if c.run(report):
                tests_fail += 1
                fails += 1
        except Exception:
            traceback.print_exc()
            print("\ttest fail @%d" % (c.line_nr - 1))
            tests_fail += 1
            fails += 1
            if arguments['--pdb']:
                import pdb
                pdb.post_mortem()

        count += 1

    file_change(current, count, fails)

    print('\nSummary: (%s fails of %s tests) in %.3fs' % (tests_fail,
                                        len(cases), time.time() - t_start))
    for s in summary:
        print(s)

    exit_code = 1 if tests_fail else 0
    sys.exit(exit_code)
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

import json
import math
import os
import shutil
import sys

from cctbx import sgtbx, crystal
from cctbx.sgtbx import bravais_types

from xia2.Driver.DriverFactory import DriverFactory
from xia2.Handlers.Phil import PhilIndex
from xia2.Handlers.Streams import Chatter, Debug
from xia2.Handlers.Syminfo import Syminfo
# this was rather complicated - now simpler!
from xia2.lib.SymmetryLib import (clean_reindex_operator, lauegroup_to_lattice,
                                  spacegroup_name_xHM_to_old)
# XDS_ASCII meddling things
from xia2.Modules.XDS_ASCII import remove_misfits
from dxtbx.serialize import load

def DialsSymmetry(DriverType = None):
  '''A factory for DialsSymmetryWrapper classes.'''

  DriverInstance = DriverFactory.Driver(DriverType)

  class DialsSymmetryWrapper(DriverInstance.__class__):
    '''A wrapper for dials.symmetry'''

    def __init__(self):
      # generic things
      super(DialsSymmetryWrapper, self).__init__()

      self.set_executable('dials.symmetry')

      self._input_laue_group = None

      self._experiments_filenames = []
      self._reflections_filenames = []
      self._output_experiments_filename = None
      self._output_reflections_filename = None

      self._hklin = None
      self._hklout = None
      self._pointgroup = None
      self._spacegroup = None
      self._reindex_matrix = None
      self._reindex_operator = None
      self._spacegroup_reindex_matrix = None
      self._spacegroup_reindex_operator = None
      self._confidence = 0.0
      self._hklref = None
      self._xdsin = None
      self._probably_twinned = False
      self._allow_out_of_sequence_files = False

      self._relative_length_tolerance = 0.05
      self._absolute_angle_tolerance = 2

      # space to store all possible solutions, to allow discussion of
      # the correct lattice with the indexer... this should be a
      # list containing e.g. 'tP'
      self._possible_lattices = []

      self._lattice_to_laue = { }

      # all "likely" spacegroups...
      self._likely_spacegroups = []

      # and unit cell information
      self._cell_info = { }
      self._cell = None

      self._json = None

    #def set_hklref(self, hklref):
      #self._hklref = hklref

    def set_hklin(self, hklin):
      self._hklin = hklin

    def get_hklin(self):
      return self._hklin

    def add_experiments(self, experiments):
      self._experiments_filenames.append(experiments)

    def add_reflections(self, reflections):
      self._reflections_filenames.append(reflections)

    def set_experiments_filename(self, experiments_filename):
      self._experiments_filenames = [experiments_filename]

    def set_reflections_filename(self, reflections_filename):
      self._reflections_filenames = [reflections_filename]

    def set_output_experiments_filename(self, experiments_filename):
      self._output_experiments_filename = experiments_filename

    def set_output_reflections_filename(self, reflections_filename):
      self._output_reflections_filename = reflections_filename

    def get_output_reflections_filename(self):
      return self._output_reflections_filename

    def get_output_experiments_filename(self):
      return self._output_experiments_filename

    def set_json(self, json):
      self._json = json

    def set_allow_out_of_sequence_files(self, allow=True):
      self._allow_out_of_sequence_files = allow

    def set_tolerance(self, relative_length_tolerance=0.05, absolute_angle_tolerance=2):
      self._relative_length_tolerance=relative_length_tolerance
      self._absolute_angle_tolerance=absolute_angle_tolerance

    #def get_hklref(self):
      #return self._hklref

    #def check_hklref(self):
      #if self._hklref is None:
        #raise RuntimeError('hklref not defined')
      #if not os.path.exists(self._hklref):
        #raise RuntimeError('hklref %s does not exist' % self._hklref)

    def set_correct_lattice(self, lattice):
      '''In a rerunning situation, set the correct lattice, which will
      assert a correct lauegroup based on the previous run of the
      program...'''

      if self._lattice_to_laue == { }:
        raise RuntimeError('no lattice to lauegroup mapping')

      if lattice not in self._lattice_to_laue:
        raise RuntimeError('lattice %s not possible' % lattice)

      self._input_laue_group = self._lattice_to_laue[lattice]

    def decide_pointgroup(self, ignore_errors=False, batches=None):
      '''Decide on the correct pointgroup for hklin.'''

      self.clear_command_line()

      if self._hklref:
        self.add_command_line('hklref')
        self.add_command_line(self._hklref)

      if self._hklin is not None:
        assert os.path.isfile(self._hklin)
        self.add_command_line("'%s'" % self._hklin)
      else:
        assert self._experiments_filenames# is not None
        assert self._reflections_filenames# is not None
        for exp in self._experiments_filenames:
          self.add_command_line("'%s'" % exp)
        for refl in self._reflections_filenames:
          self.add_command_line("'%s'" % refl)

        if not self._output_experiments_filename:
          self._output_experiments_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_experiments.json' % self.get_xpid())
        if not self._output_reflections_filename:
          self._output_reflections_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_reflections.pickle' % self.get_xpid())

        self.add_command_line("output.experiments='%s'" % self._output_experiments_filename)
        self.add_command_line("output.reflections='%s'" % self._output_reflections_filename)

      self.add_command_line('relative_length_tolerance=%s' % self._relative_length_tolerance)
      self.add_command_line('absolute_angle_tolerance=%s' % self._absolute_angle_tolerance)
      if not self._json:
        self._json = os.path.join(self.get_working_directory(),
                                '%d_dials_symmetry.json' % self.get_xpid())

      self.add_command_line("output.json='%s'" % self._json)

      self.start()

      self.close_wait()

      # check for errors
      self.check_for_errors()

      output = self.get_all_output()

      assert os.path.exists(self._json)
      with open(self._json, 'rb') as f:
        d = json.load(f)

      best_solution = d['subgroup_scores'][0]
      patterson_group = sgtbx.space_group(
        str(best_solution['patterson_group']))
      if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
        patterson_group = patterson_group.build_derived_acentric_group()

      exp = load.experiment_list(
        self.get_output_experiments_filename(), check_format=0)[0]
      unit_cell = exp.crystal.get_unit_cell()
      cs = crystal.symmetry(unit_cell=unit_cell,
        space_group=patterson_group)
      cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting()
      cs_reference = cs.as_reference_setting()
      self._pointgroup = cs_reference.space_group().type().lookup_symbol()

      self._confidence = best_solution['confidence']
      self._totalprob = best_solution['likelihood']
      cb_op_inp_min = sgtbx.change_of_basis_op(str(d['cb_op_inp_min']))
      cb_op_min_best = sgtbx.change_of_basis_op(str(best_solution['cb_op']))
      cb_op_inp_best = cb_op_min_best * cb_op_inp_min * cb_op_best_to_ref
      self._reindex_operator = cb_op_inp_best.as_xyz()
      self._reindex_matrix = cb_op_inp_best.c().r().as_double()

      if not self._input_laue_group and not self._hklref:
        for score in d['subgroup_scores']:
          patterson_group = sgtbx.space_group(str(score['patterson_group']))
          if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
            patterson_group = patterson_group.build_derived_acentric_group()
          cs = crystal.symmetry(unit_cell=unit_cell,
            space_group=patterson_group)
          cs_reference = cs.as_reference_setting()
          patterson_group = cs_reference.space_group()

          netzc = score['z_cc_net']
          # record this as a possible lattice if its Z score is positive
          lattice = str(bravais_types.bravais_lattice(
            group=patterson_group))
          if not lattice in self._possible_lattices:
            if netzc > 0.0:
              self._possible_lattices.append(lattice)
            self._lattice_to_laue[lattice] = patterson_group.type().lookup_symbol()
          self._likely_spacegroups.append(patterson_group.type().lookup_symbol())
      return

    def get_reindex_matrix(self):
      return self._reindex_matrix

    def get_reindex_operator(self):
      return self._reindex_operator

    def get_pointgroup(self):
      return self._pointgroup

    def get_cell(self):
      return self._cell

    def get_probably_twinned(self):
      return self._probably_twinned

    #def get_spacegroup(self):
      #return self._spacegroup

    #def get_spacegroup_reindex_operator(self):
      #return self._spacegroup_reindex_operator

    #def get_spacegroup_reindex_matrix(self):
      #return self._spacegroup_reindex_matrix

    # FIXME spacegroup != pointgroup
    decide_spacegroup = decide_pointgroup
    get_spacegroup = get_pointgroup
    get_spacegroup_reindex_operator = get_reindex_operator
    get_spacegroup_reindex_matrix = get_reindex_matrix

    def get_likely_spacegroups(self):
      return self._likely_spacegroups

    def get_confidence(self):
      return self._confidence

    def get_possible_lattices(self):
      return self._possible_lattices

  return DialsSymmetryWrapper()

if __name__ == '__main__':
  p = DialsSymmetry()

  hklin = sys.argv[1]

  p.set_hklin(hklin)

  p.decide_pointgroup()

  pass
<commit_msg>Further attempts to fix dials-full symmetry errors<commit_after>from __future__ import absolute_import, division, print_function

import json
import math
import os
import shutil
import sys

from cctbx import sgtbx, crystal, uctbx
from cctbx.sgtbx import bravais_types

from xia2.Driver.DriverFactory import DriverFactory
from xia2.Handlers.Phil import PhilIndex
from xia2.Handlers.Streams import Chatter, Debug
from xia2.Handlers.Syminfo import Syminfo
# this was rather complicated - now simpler!
from xia2.lib.SymmetryLib import (clean_reindex_operator, lauegroup_to_lattice,
                                  spacegroup_name_xHM_to_old)
# XDS_ASCII meddling things
from xia2.Modules.XDS_ASCII import remove_misfits

def DialsSymmetry(DriverType = None):
  '''A factory for DialsSymmetryWrapper classes.'''

  DriverInstance = DriverFactory.Driver(DriverType)

  class DialsSymmetryWrapper(DriverInstance.__class__):
    '''A wrapper for dials.symmetry'''

    def __init__(self):
      # generic things
      super(DialsSymmetryWrapper, self).__init__()

      self.set_executable('dials.symmetry')

      self._input_laue_group = None

      self._experiments_filenames = []
      self._reflections_filenames = []
      self._output_experiments_filename = None
      self._output_reflections_filename = None

      self._hklin = None
      self._hklout = None
      self._pointgroup = None
      self._spacegroup = None
      self._reindex_matrix = None
      self._reindex_operator = None
      self._spacegroup_reindex_matrix = None
      self._spacegroup_reindex_operator = None
      self._confidence = 0.0
      self._hklref = None
      self._xdsin = None
      self._probably_twinned = False
      self._allow_out_of_sequence_files = False

      self._relative_length_tolerance = 0.05
      self._absolute_angle_tolerance = 2

      # space to store all possible solutions, to allow discussion of
      # the correct lattice with the indexer... this should be a
      # list containing e.g. 'tP'
      self._possible_lattices = []

      self._lattice_to_laue = { }

      # all "likely" spacegroups...
      self._likely_spacegroups = []

      # and unit cell information
      self._cell_info = { }
      self._cell = None

      self._json = None

    #def set_hklref(self, hklref):
      #self._hklref = hklref

    def set_hklin(self, hklin):
      self._hklin = hklin

    def get_hklin(self):
      return self._hklin

    def add_experiments(self, experiments):
      self._experiments_filenames.append(experiments)

    def add_reflections(self, reflections):
      self._reflections_filenames.append(reflections)

    def set_experiments_filename(self, experiments_filename):
      self._experiments_filenames = [experiments_filename]

    def set_reflections_filename(self, reflections_filename):
      self._reflections_filenames = [reflections_filename]

    def set_output_experiments_filename(self, experiments_filename):
      self._output_experiments_filename = experiments_filename

    def set_output_reflections_filename(self, reflections_filename):
      self._output_reflections_filename = reflections_filename

    def get_output_reflections_filename(self):
      return self._output_reflections_filename

    def get_output_experiments_filename(self):
      return self._output_experiments_filename

    def set_json(self, json):
      self._json = json

    def set_allow_out_of_sequence_files(self, allow=True):
      self._allow_out_of_sequence_files = allow

    def set_tolerance(self, relative_length_tolerance=0.05, absolute_angle_tolerance=2):
      self._relative_length_tolerance=relative_length_tolerance
      self._absolute_angle_tolerance=absolute_angle_tolerance

    #def get_hklref(self):
      #return self._hklref

    #def check_hklref(self):
      #if self._hklref is None:
        #raise RuntimeError('hklref not defined')
      #if not os.path.exists(self._hklref):
        #raise RuntimeError('hklref %s does not exist' % self._hklref)

    def set_correct_lattice(self, lattice):
      '''In a rerunning situation, set the correct lattice, which will
      assert a correct lauegroup based on the previous run of the
      program...'''

      if self._lattice_to_laue == { }:
        raise RuntimeError('no lattice to lauegroup mapping')

      if lattice not in self._lattice_to_laue:
        raise RuntimeError('lattice %s not possible' % lattice)

      self._input_laue_group = self._lattice_to_laue[lattice]

    def decide_pointgroup(self, ignore_errors=False, batches=None):
      '''Decide on the correct pointgroup for hklin.'''

      self.clear_command_line()

      if self._hklref:
        self.add_command_line('hklref')
        self.add_command_line(self._hklref)

      if self._hklin is not None:
        assert os.path.isfile(self._hklin)
        self.add_command_line("'%s'" % self._hklin)
      else:
        assert self._experiments_filenames# is not None
        assert self._reflections_filenames# is not None
        for exp in self._experiments_filenames:
          self.add_command_line("'%s'" % exp)
        for refl in self._reflections_filenames:
          self.add_command_line("'%s'" % refl)

        if not self._output_experiments_filename:
          self._output_experiments_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_experiments.json' % self.get_xpid())
        if not self._output_reflections_filename:
          self._output_reflections_filename = os.path.join(
            self.get_working_directory(), '%d_reindexed_reflections.pickle' % self.get_xpid())

        self.add_command_line("output.experiments='%s'" % self._output_experiments_filename)
        self.add_command_line("output.reflections='%s'" % self._output_reflections_filename)

      self.add_command_line('relative_length_tolerance=%s' % self._relative_length_tolerance)
      self.add_command_line('absolute_angle_tolerance=%s' % self._absolute_angle_tolerance)
      if not self._json:
        self._json = os.path.join(self.get_working_directory(),
                                '%d_dials_symmetry.json' % self.get_xpid())

      self.add_command_line("output.json='%s'" % self._json)

      self.start()

      self.close_wait()

      # check for errors
      self.check_for_errors()

      output = self.get_all_output()

      assert os.path.exists(self._json)
      with open(self._json, 'rb') as f:
        d = json.load(f)

      best_solution = d['subgroup_scores'][0]
      patterson_group = sgtbx.space_group(
        str(best_solution['patterson_group']))
      if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
        patterson_group = patterson_group.build_derived_acentric_group()

      cb_op_inp_min = sgtbx.change_of_basis_op(str(d['cb_op_inp_min']))
      cb_op_min_best = sgtbx.change_of_basis_op(str(best_solution['cb_op']))

      input_cell = uctbx.unit_cell(d['input_symmetry']['unit_cell'])
      cb_op_inp_best = cb_op_min_best * cb_op_inp_min
      best_cell = input_cell.change_basis(
        cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_best)))
      # ^^ not necessarily in reference setting e.g I2/m not C2/m

      cs = crystal.symmetry(unit_cell=best_cell,
        space_group=patterson_group)
      cb_op_best_to_ref = cs.change_of_basis_op_to_reference_setting()
      cs_reference = cs.as_reference_setting()
      self._pointgroup = cs_reference.space_group().type().lookup_symbol()

      self._confidence = best_solution['confidence']
      self._totalprob = best_solution['likelihood']
      cb_op_inp_best = cb_op_inp_best * cb_op_best_to_ref
      self._reindex_operator = cb_op_inp_best.as_xyz()
      self._reindex_matrix = cb_op_inp_best.c().r().as_double()

      if not self._input_laue_group and not self._hklref:
        for score in d['subgroup_scores']:
          patterson_group = sgtbx.space_group(str(score['patterson_group']))
          if PhilIndex.params.xia2.settings.symmetry.chirality in (None, 'chiral'):
            patterson_group = patterson_group.build_derived_acentric_group()

          cb_op_min_this = sgtbx.change_of_basis_op(str(score['cb_op']))
          cb_op_inp_this = cb_op_min_this * cb_op_inp_min
          unit_cell = input_cell.change_basis(
            cb_op=sgtbx.change_of_basis_op(str(cb_op_inp_this)))
          cs = crystal.symmetry(unit_cell=unit_cell,
            space_group=patterson_group)
          cs_reference = cs.as_reference_setting()
          patterson_group = cs_reference.space_group()

          netzc = score['z_cc_net']
          # record this as a possible lattice if its Z score is positive
          lattice = str(bravais_types.bravais_lattice(
            group=patterson_group))
          if not lattice in self._possible_lattices:
            if netzc > 0.0:
              self._possible_lattices.append(lattice)
            self._lattice_to_laue[lattice] = patterson_group.type().lookup_symbol()
          self._likely_spacegroups.append(patterson_group.type().lookup_symbol())
      return

    def get_reindex_matrix(self):
      return self._reindex_matrix

    def get_reindex_operator(self):
      return self._reindex_operator

    def get_pointgroup(self):
      return self._pointgroup

    def get_cell(self):
      return self._cell

    def get_probably_twinned(self):
      return self._probably_twinned

    #def get_spacegroup(self):
      #return self._spacegroup

    #def get_spacegroup_reindex_operator(self):
      #return self._spacegroup_reindex_operator

    #def get_spacegroup_reindex_matrix(self):
      #return self._spacegroup_reindex_matrix

    # FIXME spacegroup != pointgroup
    decide_spacegroup = decide_pointgroup
    get_spacegroup = get_pointgroup
    get_spacegroup_reindex_operator = get_reindex_operator
    get_spacegroup_reindex_matrix = get_reindex_matrix

    def get_likely_spacegroups(self):
      return self._likely_spacegroups

    def get_confidence(self):
      return self._confidence

    def get_possible_lattices(self):
      return self._possible_lattices

  return DialsSymmetryWrapper()

if __name__ == '__main__':
  p = DialsSymmetry()

  hklin = sys.argv[1]

  p.set_hklin(hklin)

  p.decide_pointgroup()

  pass
<|endoftext|>


<commit_before>from markupsafe import escape
from markupsafe import Markup

from . import json as json
from .app import Flask as Flask
from .app import Request as Request
from .app import Response as Response
from .blueprints import Blueprint as Blueprint
from .config import Config as Config
from .ctx import after_this_request as after_this_request
from .ctx import copy_current_request_context as copy_current_request_context
from .ctx import has_app_context as has_app_context
from .ctx import has_request_context as has_request_context
from .globals import current_app as current_app
from .globals import g as g
from .globals import request as request
from .globals import session as session
from .helpers import abort as abort
from .helpers import flash as flash
from .helpers import get_flashed_messages as get_flashed_messages
from .helpers import get_template_attribute as get_template_attribute
from .helpers import make_response as make_response
from .helpers import redirect as redirect
from .helpers import send_file as send_file
from .helpers import send_from_directory as send_from_directory
from .helpers import stream_with_context as stream_with_context
from .helpers import url_for as url_for
from .json import jsonify as jsonify
from .signals import appcontext_popped as appcontext_popped
from .signals import appcontext_pushed as appcontext_pushed
from .signals import appcontext_tearing_down as appcontext_tearing_down
from .signals import before_render_template as before_render_template
from .signals import got_request_exception as got_request_exception
from .signals import message_flashed as message_flashed
from .signals import request_finished as request_finished
from .signals import request_started as request_started
from .signals import request_tearing_down as request_tearing_down
from .signals import signals_available as signals_available
from .signals import template_rendered as template_rendered
from .templating import render_template as render_template
from .templating import render_template_string as render_template_string
from .templating import stream_template as stream_template
from .templating import stream_template_string as stream_template_string

__version__ = "2.2.3"

def __getattr__(name):
    if name == "_app_ctx_stack":
        import warnings
        from .globals import __app_ctx_stack

        warnings.warn(
            "'_app_ctx_stack' is deprecated and will be removed in Flask 2.3.",
            DeprecationWarning,
            stacklevel=2,
        )
        return __app_ctx_stack

    if name == "_request_ctx_stack":
        import warnings
        from .globals import __request_ctx_stack

        warnings.warn(
            "'_request_ctx_stack' is deprecated and will be removed in Flask 2.3.",
            DeprecationWarning,
            stacklevel=2,
        )
        return __request_ctx_stack

    raise AttributeError(name)
<commit_msg>Add missing .dev to version (#4801)<commit_after>from markupsafe import escape
from markupsafe import Markup

from . import json as json
from .app import Flask as Flask
from .app import Request as Request
from .app import Response as Response
from .blueprints import Blueprint as Blueprint
from .config import Config as Config
from .ctx import after_this_request as after_this_request
from .ctx import copy_current_request_context as copy_current_request_context
from .ctx import has_app_context as has_app_context
from .ctx import has_request_context as has_request_context
from .globals import current_app as current_app
from .globals import g as g
from .globals import request as request
from .globals import session as session
from .helpers import abort as abort
from .helpers import flash as flash
from .helpers import get_flashed_messages as get_flashed_messages
from .helpers import get_template_attribute as get_template_attribute
from .helpers import make_response as make_response
from .helpers import redirect as redirect
from .helpers import send_file as send_file
from .helpers import send_from_directory as send_from_directory
from .helpers import stream_with_context as stream_with_context
from .helpers import url_for as url_for
from .json import jsonify as jsonify
from .signals import appcontext_popped as appcontext_popped
from .signals import appcontext_pushed as appcontext_pushed
from .signals import appcontext_tearing_down as appcontext_tearing_down
from .signals import before_render_template as before_render_template
from .signals import got_request_exception as got_request_exception
from .signals import message_flashed as message_flashed
from .signals import request_finished as request_finished
from .signals import request_started as request_started
from .signals import request_tearing_down as request_tearing_down
from .signals import signals_available as signals_available
from .signals import template_rendered as template_rendered
from .templating import render_template as render_template
from .templating import render_template_string as render_template_string
from .templating import stream_template as stream_template
from .templating import stream_template_string as stream_template_string

__version__ = "2.2.3.dev"

def __getattr__(name):
    if name == "_app_ctx_stack":
        import warnings
        from .globals import __app_ctx_stack

        warnings.warn(
            "'_app_ctx_stack' is deprecated and will be removed in Flask 2.3.",
            DeprecationWarning,
            stacklevel=2,
        )
        return __app_ctx_stack

    if name == "_request_ctx_stack":
        import warnings
        from .globals import __request_ctx_stack

        warnings.warn(
            "'_request_ctx_stack' is deprecated and will be removed in Flask 2.3.",
            DeprecationWarning,
            stacklevel=2,
        )
        return __request_ctx_stack

    raise AttributeError(name)
<|endoftext|>


<commit_before>from __future__ import unicode_literals
from bst import Node
from random import randint
import pytest

@pytest.fixture()
def rand_setup():
    root = Node(randint(1, 100))
    for idx in range(20):
        val = randint(1, 100)
        try:
            root.insert(val)
        except AttributeError:
            continue

    return root

@pytest.fixture()
def fixed_setup():
    root = Node(25)
    root.insert(15)
    root.insert(30)
    root.insert(9)
    root.insert(17)
    root.insert(21)
    root.insert(39)
    root.insert(12)
    root.insert(24)
    root.insert(40)
    root.insert(14)

    return root

def test_insert(rand_setup):
    pre_size = rand_setup.size()
    new = 200
    rand_setup.insert(new)
    assert rand_setup.insert(new) is None
    rand_setup.insert(new)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_contains(rand_setup):
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.contains(rand) is True

def test_size(rand_setup):
    pre_size = rand_setup.size()
    new = 200
    rand_setup.insert(new)
    rand_setup.insert(new)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_depth(fixed_setup):
    assert fixed_setup.left.depth() == 4
    assert fixed_setup.right.depth() == 3
    fixed_setup.insert(13)
    assert fixed_setup.left.depth() == 5

def test_balance(rand_setup):
    left = rand_setup.left.depth() if rand_setup.left is not None else 0
    right = rand_setup.right.depth() if rand_setup.right is not None else 0
    if left > right:
        assert rand_setup.balance() == 1
    elif right > left:
        assert rand_setup.balance() == -1
    else:
        assert rand_setup.balance() == 0
<commit_msg>Fix test balance; need to improve test architecture.<commit_after>from __future__ import unicode_literals
from bst import Node
from random import randint
import pytest

@pytest.fixture()
def rand_setup():
    root = Node(randint(1, 100))
    for idx in range(20):
        val = randint(1, 100)
        try:
            root.insert(val)
        except AttributeError:
            continue

    return root

@pytest.fixture()
def fixed_setup():
    root = Node(25)
    root.insert(15)
    root.insert(30)
    root.insert(9)
    root.insert(17)
    root.insert(21)
    root.insert(39)
    root.insert(12)
    root.insert(24)
    root.insert(40)
    root.insert(14)

    return root

def test_insert(rand_setup):
    pre_size = rand_setup.size()
    new = 200
    rand_setup.insert(new)
    assert rand_setup.insert(new) is None
    rand_setup.insert(new)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_contains(rand_setup):
    rand = randint(1, 100)
    rand_setup.insert(rand)
    assert rand_setup.contains(rand) is True

def test_size(rand_setup):
    pre_size = rand_setup.size()
    new = 200
    rand_setup.insert(new)
    rand_setup.insert(new)
    post_size = rand_setup.size()
    assert post_size > pre_size
    assert post_size == pre_size + 1

def test_depth(fixed_setup):
    assert fixed_setup.left.depth() == 4
    assert fixed_setup.right.depth() == 3
    fixed_setup.insert(13)
    assert fixed_setup.left.depth() == 5

def test_balance(rand_setup):
    left = rand_setup.left.depth() if rand_setup.left is not None else 0
    right = rand_setup.right.depth() if rand_setup.right is not None else 0
    if left > right:
        assert rand_setup.balance() > 0
    elif right > left:
        assert rand_setup.balance() < 0
    else:
        assert rand_setup.balance() == 0
<|endoftext|>


<commit_before>import _gdal
import core
import graph
import math
from ModestMaps.Core import Point, Coordinate
from osgeo import gdal
from osgeo.osr import SpatialReference
import pads
import numpy as np
import StringIO
import TileStache

class LayerRasterNode(graph.Node):
    def __init__(self, pads_, opacities=None):
        super(LayerRasterNode, self).__init__()
        self.outputs['raster'] = pads.TiledRasterOutputPad(self._render)
        self.pads = pads_
        if opacities is None:
            self.opacities = (1,) * len(self.pads)
        else:
            self.opacities = opacities
            if len(self.opacities) != len(self.pads):
                raise ValueError('opacities: expected sequence of length %s' % (len(self.pads),))

    def _render(self, tiles):
        rv = []
        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size))
        return rv

    def _render_tile(self, envelope, size):
        if len(self.pads) == 0:
            return None

        output = None
        for response, opacity in zip([pad(envelope, size) for pad in self.pads], self.opacities):
            if response is None:
                continue

            type_, raster = response

            if type_ == pads.ContentType.NONE:
                continue

            if type_ is not pads.ContentType.RASTER:
                raise RuntimeError('Input is not raster')

            layer = raster.to_rgba()
            if layer is None:
                print('layer failed to convert')
                continue

            if output is None:
                output = layer
                continue

            one_minus_alpha = np.atleast_3d(1.0 - opacity * layer[:,:,3])

            output[:,:,:3] *= np.repeat(one_minus_alpha, 3, 2)
            output[:,:,:3] += opacity * layer[:,:,:3]

            output[:,:,3] *= one_minus_alpha[:,:,0]
            output[:,:,3] += opacity * layer[:,:,3]

        if output is None:
            return None

        return core.Raster(output, envelope, to_rgba=lambda x: x)

class GDALDatasetRasterNode(graph.Node):
    def __init__(self, dataset):
        super(GDALDatasetRasterNode, self).__init__()
        self.dataset = dataset
        self.spatial_reference = SpatialReference()
        self.spatial_reference.ImportFromWkt(self.dataset.GetProjection())

        source_pad = pads.TiledRasterOutputPad(self._render, tile_size=256)
        self.outputs['raster'] = pads.ReprojectingOutputPad(self.spatial_reference, source_pad)

        self.envelope = _gdal.dataset_envelope(self.dataset, self.spatial_reference)
        self.boundary = core.boundary_from_envelope(self.envelope)

        self.is_palette = any([
            self.dataset.GetRasterBand(i).GetColorInterpretation() == gdal.GCI_PaletteIndex
            for i in xrange(1, self.dataset.RasterCount+1)])

    def _to_rgba(self, array):
        rgba = core.to_rgba_unknown(array)
        if np.any(rgba[:,:,3] != 1.0):
            mask_alpha = rgba[:,:,3]
        else:
            mask_alpha = 1.0

        for i in xrange(1, self.dataset.RasterCount+1):
            band = self.dataset.GetRasterBand(i)
            interp = band.GetColorInterpretation()
            data = array[:,:,i-1] * mask_alpha

            if interp == gdal.GCI_RedBand:
                rgba[:,:,0] = data / 255.0
            elif interp == gdal.GCI_GreenBand:
                rgba[:,:,1] = data / 255.0
            elif interp == gdal.GCI_BlueBand:
                rgba[:,:,2] = data / 255.0
            elif interp == gdal.GCI_AlphaBand:
                rgba[:,:,3] = data / 255.0
            elif interp == gdal.GCI_GrayIndex:
                rgba[:,:,:3] = np.repeat(np.atleast_3d(data / 255.0), 3, 2)
            elif interp == gdal.GCI_PaletteIndex:
                table = band.GetColorTable()
                entries = np.array([tuple(table.GetColorEntry(i)) for i in xrange(table.GetCount())])

                # FIXME: This assumes the palette is RGBA
                rgba = np.float32(entries[np.int32(data)]) / 255.0
                for i in xrange(4):
                    rgba[:,:,i] *= mask_alpha

        return rgba

    def _render(self, tiles):
        rv = []
        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size))
        return rv

    def _render_tile(self, envelope, size):
        assert envelope.spatial_reference.IsSame(self.spatial_reference)

        # check if the requested area is contained within the dataset bounds
        ds_boundary = self.boundary.transform_to(
                envelope.spatial_reference,
                min(self.envelope.size()) / max(size),
                min(envelope.size()) / max(size),
        )
        requested_boundary = core.boundary_from_envelope(envelope)
        if not ds_boundary.geometry.Intersects(requested_boundary.geometry):
            # early out if the dataset is nowhere near the requested envelope
            return None

        # Get the destination raster
        raster = _gdal.create_render_dataset(
                envelope, size, prototype_ds=self.dataset)
        ds = raster.dataset

        desired_srs_wkt = envelope.spatial_reference.ExportToWkt()
        gdal.ReprojectImage(
                self.dataset, ds,
                self.dataset.GetProjection(),
                desired_srs_wkt,
                gdal.GRA_NearestNeighbour if self.is_palette else gdal.GRA_Bilinear)

        # Create a mask raster
        mask_raster = _gdal.create_render_dataset(envelope, size, data_type=gdal.GDT_Float32)
        mask_ds = mask_raster.dataset
        band = mask_ds.GetRasterBand(1)
        band.SetColorInterpretation(gdal.GCI_Undefined)
        band.SetNoDataValue(float('nan'))
        band.Fill(float('nan'))
        gdal.ReprojectImage(
                self.dataset, mask_ds,
                self.dataset.GetProjection(),
                desired_srs_wkt,
                gdal.GRA_NearestNeighbour)
        mask_band = mask_ds.GetRasterBand(1).GetMaskBand()

        return core.Raster.from_dataset(ds, mask_band=mask_band, to_rgba=self._to_rgba)

class TileStacheRasterNode(graph.Node):
    def __init__(self, layer):
        super(TileStacheRasterNode, self).__init__()
        self.outputs['raster'] = pads.TiledRasterOutputPad(self._render, tile_size=256)

        self.layer = layer
        self.preferred_srs = SpatialReference()
        self.preferred_srs.ImportFromProj4(layer.projection.srs)
        self.preferred_srs_wkt = self.preferred_srs.ExportToWkt()

        # Calculate the bounds of the zoom level 0 tile
        bounds = [
            self.layer.projection.coordinateProj(Coordinate(0,0,0)),
            self.layer.projection.coordinateProj(Coordinate(1,1,0)),
        ]

        self.proj_axes = (bounds[1].x - bounds[0].x, bounds[1].y - bounds[0].y)
        self.proj_size = map(abs, self.proj_axes)
        self.proj_origin = (bounds[0].x, bounds[0].y)

        min_x = min([p.x for p in bounds])
        max_x = max([p.x for p in bounds])
        min_y = min([p.y for p in bounds])
        max_y = max([p.y for p in bounds])

        self.proj_bounds = (min_x, min_y, max_x-min_x, max_y-min_y)

    def _zoom_for_envelope(self, envelope, size):
        # How many tiles should cover each axis
        n_tiles = map(lambda x: x/256.0, size)

        # Over what range?
        envelope_size = map(abs, envelope.offset())
        proj_range = min(envelope_size)

        # Giving a rough tile size in projection coordinate of...
        zoomed_tile_size = map(lambda x: x[0]/x[1], zip(envelope_size, n_tiles))

        # And convert to a zoom
        zooms = map(lambda x: math.log(x[0], 2) - math.log(x[1], 2), zip(self.proj_size, zoomed_tile_size))

        # Which is closer to the precise zoom we want?
        zoom = max(zooms)
        floor_diff = abs(math.pow(2, zoom) - math.pow(2, math.floor(zoom)))
        ceil_diff = abs(math.pow(2, zoom) - math.pow(2, math.ceil(zoom)))

        # Choose an overall zoom from these
        int_zoom = int(math.ceil(zoom)) if ceil_diff < floor_diff else int(math.floor(zoom))
        return max(0, min(18, int_zoom))

    def _render(self, tiles):
        rv = []

        zooms = []
        for envelope, size in tiles:
            try:
                pref_envelope = envelope.transform_to(
                        self.preferred_srs,
                        min(envelope.size()) / float(max(size)))
            except core.ProjectionError:
                # Skip projection errors
                continue
            zooms.append(self._zoom_for_envelope(pref_envelope, size))

        # Choose the 75th percentile zoom
        zooms.sort()
        zoom = zooms[(len(zooms)>>1) + (len(zooms)>>2)]

        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size, zoom))
        return rv

    def _render_tile(self, envelope, size, zoom):
        # Get the destination raster
        raster = _gdal.create_render_dataset(envelope, size)
        assert size == (raster.dataset.RasterXSize, raster.dataset.RasterYSize)

        # Convert the envelope into the preferred spatial reference
        try:
            pref_envelope = envelope.transform_to(
                    self.preferred_srs,
                    min(envelope.size()) / float(max(size)))
        except core.ProjectionError:
            print('Ignoring projection error and returning empty raster')
            return raster

        # How many tiles overall in x and y at this zoom
        n_proj_tiles = math.pow(2, zoom)

        # Compute the tile co-ordinates of each corner
        corners = [
                self.layer.projection.projCoordinate(Point(pref_envelope.left, pref_envelope.top)),
                self.layer.projection.projCoordinate(Point(pref_envelope.right, pref_envelope.top)),
                self.layer.projection.projCoordinate(Point(pref_envelope.left, pref_envelope.bottom)),
                self.layer.projection.projCoordinate(Point(pref_envelope.right, pref_envelope.bottom)),
        ]
        corners = [c.zoomTo(zoom) for c in corners]

        corner_rows = [int(math.floor(x.row)) for x in corners]
        corner_columns = [int(math.floor(x.column)) for x in corners]

        # Get each tile image
        png_driver = gdal.GetDriverByName('PNG')
        desired_srs_wkt = envelope.spatial_reference.ExportToWkt()
        assert png_driver is not None
        for r in xrange(min(corner_rows), max(corner_rows)+1):
            if r < 0 or r >= n_proj_tiles:
                continue
            for c in xrange(min(corner_columns), max(corner_columns)+1):
                tile_tl_point = self.layer.projection.coordinateProj(Coordinate(r,c,zoom))
                tile_br_point = self.layer.projection.coordinateProj(Coordinate(r+1,c+1,zoom))

                c = c % n_proj_tiles
                if c < 0:
                    c += n_proj_tiles
                tile_coord = Coordinate(r, c, zoom)

                try:
                    tile_type, png_data = TileStache.getTile(self.layer, tile_coord, 'png')
                    if tile_type != 'image/png':
                        print('Did not get PNG data when fetching tile %s. Skipping.' % (tile_coord))
                        continue
                except IOError as e:
                    print('Ignoring error fetching tile %s (%s).' % (tile_coord, e))
                    continue

                gdal.FileFromMemBuffer('/vsimem/tmptile.png', png_data)
                tile_raster = gdal.Open('/vsimem/tmptile.png')
                tile_raster.SetProjection(self.preferred_srs_wkt)

                xscale = (tile_br_point.x - tile_tl_point.x) / tile_raster.RasterXSize
                yscale = (tile_br_point.y - tile_tl_point.y) / tile_raster.RasterXSize
                tile_raster.SetGeoTransform((
                    tile_tl_point.x, xscale, 0.0,
                    tile_tl_point.y, 0.0, yscale,
                ))

                gdal.ReprojectImage(
                        tile_raster, raster.dataset,
                        self.preferred_srs_wkt, desired_srs_wkt, gdal.GRA_Bilinear)

                tile_raster = None
                gdal.Unlink('/vsimem/tmptile.png')

        return core.Raster.from_dataset(raster.dataset, to_rgba=core.RgbaFromBands(
            (
                (core.RgbaFromBands.RED,    1.0/255.0),
                (core.RgbaFromBands.GREEN,  1.0/255.0),
                (core.RgbaFromBands.BLUE,   1.0/255.0),
            ),
            True))
<commit_msg>GDALDatasetRenderNode: don't needlessly transform boundary<commit_after>import _gdal
import core
import graph
import math
from ModestMaps.Core import Point, Coordinate
from osgeo import gdal
from osgeo.osr import SpatialReference
import pads
import numpy as np
import StringIO
import TileStache

class LayerRasterNode(graph.Node):
    def __init__(self, pads_, opacities=None):
        super(LayerRasterNode, self).__init__()
        self.outputs['raster'] = pads.TiledRasterOutputPad(self._render)
        self.pads = pads_
        if opacities is None:
            self.opacities = (1,) * len(self.pads)
        else:
            self.opacities = opacities
            if len(self.opacities) != len(self.pads):
                raise ValueError('opacities: expected sequence of length %s' % (len(self.pads),))

    def _render(self, tiles):
        rv = []
        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size))
        return rv

    def _render_tile(self, envelope, size):
        if len(self.pads) == 0:
            return None

        output = None
        for response, opacity in zip([pad(envelope, size) for pad in self.pads], self.opacities):
            if response is None:
                continue

            type_, raster = response

            if type_ == pads.ContentType.NONE:
                continue

            if type_ is not pads.ContentType.RASTER:
                raise RuntimeError('Input is not raster')

            layer = raster.to_rgba()
            if layer is None:
                print('layer failed to convert')
                continue

            if output is None:
                output = layer
                continue

            one_minus_alpha = np.atleast_3d(1.0 - opacity * layer[:,:,3])

            output[:,:,:3] *= np.repeat(one_minus_alpha, 3, 2)
            output[:,:,:3] += opacity * layer[:,:,:3]

            output[:,:,3] *= one_minus_alpha[:,:,0]
            output[:,:,3] += opacity * layer[:,:,3]

        if output is None:
            return None

        return core.Raster(output, envelope, to_rgba=lambda x: x)

class GDALDatasetRasterNode(graph.Node):
    def __init__(self, dataset):
        super(GDALDatasetRasterNode, self).__init__()
        self.dataset = dataset
        self.spatial_reference = SpatialReference()
        self.spatial_reference.ImportFromWkt(self.dataset.GetProjection())

        source_pad = pads.TiledRasterOutputPad(self._render, tile_size=256)
        self.outputs['raster'] = pads.ReprojectingOutputPad(self.spatial_reference, source_pad)

        self.envelope = _gdal.dataset_envelope(self.dataset, self.spatial_reference)
        self.boundary = core.boundary_from_envelope(self.envelope)

        self.is_palette = any([
            self.dataset.GetRasterBand(i).GetColorInterpretation() == gdal.GCI_PaletteIndex
            for i in xrange(1, self.dataset.RasterCount+1)])

    def _to_rgba(self, array):
        rgba = core.to_rgba_unknown(array)
        if np.any(rgba[:,:,3] != 1.0):
            mask_alpha = rgba[:,:,3]
        else:
            mask_alpha = 1.0

        for i in xrange(1, self.dataset.RasterCount+1):
            band = self.dataset.GetRasterBand(i)
            interp = band.GetColorInterpretation()
            data = array[:,:,i-1] * mask_alpha

            if interp == gdal.GCI_RedBand:
                rgba[:,:,0] = data / 255.0
            elif interp == gdal.GCI_GreenBand:
                rgba[:,:,1] = data / 255.0
            elif interp == gdal.GCI_BlueBand:
                rgba[:,:,2] = data / 255.0
            elif interp == gdal.GCI_AlphaBand:
                rgba[:,:,3] = data / 255.0
            elif interp == gdal.GCI_GrayIndex:
                rgba[:,:,:3] = np.repeat(np.atleast_3d(data / 255.0), 3, 2)
            elif interp == gdal.GCI_PaletteIndex:
                table = band.GetColorTable()
                entries = np.array([tuple(table.GetColorEntry(i)) for i in xrange(table.GetCount())])

                # FIXME: This assumes the palette is RGBA
                rgba = np.float32(entries[np.int32(data)]) / 255.0
                for i in xrange(4):
                    rgba[:,:,i] *= mask_alpha

        return rgba

    def _render(self, tiles):
        rv = []
        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size))
        return rv

    def _render_tile(self, envelope, size):
        assert envelope.spatial_reference.IsSame(self.spatial_reference)

        # check if the requested area is contained within the dataset bounds
        requested_boundary = core.boundary_from_envelope(envelope)
        if not self.boundary.geometry.Intersects(requested_boundary.geometry):
            # early out if the dataset is nowhere near the requested envelope
            return None

        # Get the destination raster
        raster = _gdal.create_render_dataset(
                envelope, size, prototype_ds=self.dataset)
        ds = raster.dataset

        desired_srs_wkt = envelope.spatial_reference.ExportToWkt()
        gdal.ReprojectImage(
                self.dataset, ds,
                self.dataset.GetProjection(),
                desired_srs_wkt,
                gdal.GRA_NearestNeighbour if self.is_palette else gdal.GRA_Bilinear)

        # Create a mask raster
        mask_raster = _gdal.create_render_dataset(envelope, size, data_type=gdal.GDT_Float32)
        mask_ds = mask_raster.dataset
        band = mask_ds.GetRasterBand(1)
        band.SetColorInterpretation(gdal.GCI_Undefined)
        band.SetNoDataValue(float('nan'))
        band.Fill(float('nan'))
        gdal.ReprojectImage(
                self.dataset, mask_ds,
                self.dataset.GetProjection(),
                desired_srs_wkt,
                gdal.GRA_NearestNeighbour)
        mask_band = mask_ds.GetRasterBand(1).GetMaskBand()

        return core.Raster.from_dataset(ds, mask_band=mask_band, to_rgba=self._to_rgba)

class TileStacheRasterNode(graph.Node):
    def __init__(self, layer):
        super(TileStacheRasterNode, self).__init__()
        self.outputs['raster'] = pads.TiledRasterOutputPad(self._render, tile_size=256)

        self.layer = layer
        self.preferred_srs = SpatialReference()
        self.preferred_srs.ImportFromProj4(layer.projection.srs)
        self.preferred_srs_wkt = self.preferred_srs.ExportToWkt()

        # Calculate the bounds of the zoom level 0 tile
        bounds = [
            self.layer.projection.coordinateProj(Coordinate(0,0,0)),
            self.layer.projection.coordinateProj(Coordinate(1,1,0)),
        ]

        self.proj_axes = (bounds[1].x - bounds[0].x, bounds[1].y - bounds[0].y)
        self.proj_size = map(abs, self.proj_axes)
        self.proj_origin = (bounds[0].x, bounds[0].y)

        min_x = min([p.x for p in bounds])
        max_x = max([p.x for p in bounds])
        min_y = min([p.y for p in bounds])
        max_y = max([p.y for p in bounds])

        self.proj_bounds = (min_x, min_y, max_x-min_x, max_y-min_y)

    def _zoom_for_envelope(self, envelope, size):
        # How many tiles should cover each axis
        n_tiles = map(lambda x: x/256.0, size)

        # Over what range?
        envelope_size = map(abs, envelope.offset())
        proj_range = min(envelope_size)

        # Giving a rough tile size in projection coordinate of...
        zoomed_tile_size = map(lambda x: x[0]/x[1], zip(envelope_size, n_tiles))

        # And convert to a zoom
        zooms = map(lambda x: math.log(x[0], 2) - math.log(x[1], 2), zip(self.proj_size, zoomed_tile_size))

        # Which is closer to the precise zoom we want?
        zoom = max(zooms)
        floor_diff = abs(math.pow(2, zoom) - math.pow(2, math.floor(zoom)))
        ceil_diff = abs(math.pow(2, zoom) - math.pow(2, math.ceil(zoom)))

        # Choose an overall zoom from these
        int_zoom = int(math.ceil(zoom)) if ceil_diff < floor_diff else int(math.floor(zoom))
        return max(0, min(18, int_zoom))

    def _render(self, tiles):
        rv = []

        zooms = []
        for envelope, size in tiles:
            try:
                pref_envelope = envelope.transform_to(
                        self.preferred_srs,
                        min(envelope.size()) / float(max(size)))
            except core.ProjectionError:
                # Skip projection errors
                continue
            zooms.append(self._zoom_for_envelope(pref_envelope, size))

        # Choose the 75th percentile zoom
        zooms.sort()
        zoom = zooms[(len(zooms)>>1) + (len(zooms)>>2)]

        for envelope, size in tiles:
            rv.append(self._render_tile(envelope, size, zoom))
        return rv

    def _render_tile(self, envelope, size, zoom):
        # Get the destination raster
        raster = _gdal.create_render_dataset(envelope, size)
        assert size == (raster.dataset.RasterXSize, raster.dataset.RasterYSize)

        # Convert the envelope into the preferred spatial reference
        try:
            pref_envelope = envelope.transform_to(
                    self.preferred_srs,
                    min(envelope.size()) / float(max(size)))
        except core.ProjectionError:
            print('Ignoring projection error and returning empty raster')
            return raster

        # How many tiles overall in x and y at this zoom
        n_proj_tiles = math.pow(2, zoom)

        # Compute the tile co-ordinates of each corner
        corners = [
                self.layer.projection.projCoordinate(Point(pref_envelope.left, pref_envelope.top)),
                self.layer.projection.projCoordinate(Point(pref_envelope.right, pref_envelope.top)),
                self.layer.projection.projCoordinate(Point(pref_envelope.left, pref_envelope.bottom)),
                self.layer.projection.projCoordinate(Point(pref_envelope.right, pref_envelope.bottom)),
        ]
        corners = [c.zoomTo(zoom) for c in corners]

        corner_rows = [int(math.floor(x.row)) for x in corners]
        corner_columns = [int(math.floor(x.column)) for x in corners]

        # Get each tile image
        png_driver = gdal.GetDriverByName('PNG')
        desired_srs_wkt = envelope.spatial_reference.ExportToWkt()
        assert png_driver is not None
        for r in xrange(min(corner_rows), max(corner_rows)+1):
            if r < 0 or r >= n_proj_tiles:
                continue
            for c in xrange(min(corner_columns), max(corner_columns)+1):
                tile_tl_point = self.layer.projection.coordinateProj(Coordinate(r,c,zoom))
                tile_br_point = self.layer.projection.coordinateProj(Coordinate(r+1,c+1,zoom))

                c = c % n_proj_tiles
                if c < 0:
                    c += n_proj_tiles
                tile_coord = Coordinate(r, c, zoom)

                try:
                    tile_type, png_data = TileStache.getTile(self.layer, tile_coord, 'png')
                    if tile_type != 'image/png':
                        print('Did not get PNG data when fetching tile %s. Skipping.' % (tile_coord))
                        continue
                except IOError as e:
                    print('Ignoring error fetching tile %s (%s).' % (tile_coord, e))
                    continue

                gdal.FileFromMemBuffer('/vsimem/tmptile.png', png_data)
                tile_raster = gdal.Open('/vsimem/tmptile.png')
                tile_raster.SetProjection(self.preferred_srs_wkt)

                xscale = (tile_br_point.x - tile_tl_point.x) / tile_raster.RasterXSize
                yscale = (tile_br_point.y - tile_tl_point.y) / tile_raster.RasterXSize
                tile_raster.SetGeoTransform((
                    tile_tl_point.x, xscale, 0.0,
                    tile_tl_point.y, 0.0, yscale,
                ))

                gdal.ReprojectImage(
                        tile_raster, raster.dataset,
                        self.preferred_srs_wkt, desired_srs_wkt, gdal.GRA_Bilinear)

                tile_raster = None
                gdal.Unlink('/vsimem/tmptile.png')

        return core.Raster.from_dataset(raster.dataset, to_rgba=core.RgbaFromBands(
            (
                (core.RgbaFromBands.RED,    1.0/255.0),
                (core.RgbaFromBands.GREEN,  1.0/255.0),
                (core.RgbaFromBands.BLUE,   1.0/255.0),
            ),
            True))
<|endoftext|>


<commit_before>#!/usr/bin/env python

# pylint: disable=protected-access,too-many-public-methods,missing-docstring

import logging
import json
import os
import types
import unittest

from mock import patch

os.environ["DEN_ACCESS_TOKEN"] = "TEST"
from den import record

logging.basicConfig(filename=os.devnull)

class RecordTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        with open("responses.txt", "r") as f:
            cls.responses = [l for l in f]

    def test_missing_env_variable_raises_key_error(self):
        with patch.dict("os.environ", {}):
            del os.environ["DEN_ACCESS_TOKEN"]
            with self.assertRaisesRegexp(KeyError, r"Please set the environment variable 'DEN_ACCESS_TOKEN'."):
                reload(record)

    def test_get_api_url(self):
        expected = "https://developer-api.nest.com?auth=%s" % record.NEST_API_ACCESS_TOKEN
        actual = record._get_api_url()
        self.assertEqual(expected, actual)

        expected = "https://developer-api.nest.com/structures?auth=%s" % record.NEST_API_ACCESS_TOKEN
        for path in ["structures", "/structures", "/structures/", "structures/"]:
            self.assertEqual(expected, record._get_api_url(path))

    def test_is_event(self):
        self.assertTrue(record._is_event("event:"))
        self.assertFalse(record._is_event("event"))
        self.assertFalse(record._is_event("data:"))

    def test_is_data(self):
        self.assertTrue(record._is_data("data:"))
        self.assertFalse(record._is_data("data"))
        self.assertFalse(record._is_data("event:"))

    def test_process_event_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_event(line))

    def test_process_event_returns_none_for_keep_alive_line(self):
        self.assertIsNone(record._process_event("event: keep-alive"))
        self.assertIsNone(record._process_event("event:keep-alive"))

    def test_process_event_returns_event_for_valid_line(self):
        expected = "Test event"
        actual = record._process_event("event: " + expected)
        self.assertEqual(expected, actual)
        actual = record._process_event("event:" + expected)
        self.assertEqual(expected, actual)

    def test_process_data_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_data(line))

    def test_process_data_returns_list_for_valid_line(self):
        expected = [{"key": "val"}]
        actual = record._process_data("data: " + json.dumps(expected))
        self.assertEqual(expected, actual)
        actual = record._process_data("data:" + json.dumps(expected))
        self.assertEqual(expected, actual)

    def test_process_returns_none_for_non_data_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process(line))

    def test_process_returns_none_for_invalid_data_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process(line))

    def test_process_returns_list_for_valid_data_line(self):
        expected = [{"key": "val"}]
        actual = record._process("data: " + json.dumps(expected))
        self.assertEqual(expected, actual)
        actual = record._process("data:" + json.dumps(expected))
        self.assertEqual(expected, actual)

        for r in self.responses:
            actual = record._process(r)
            if r.startswith("event"):
                self.assertIsNone(actual)
            elif r.startswith("data"):
                self.assertIsInstance(actual, types.DictType)

    def test_get_structures_returns_empty_list_for_invalid_data(self):
        expected = []
        actual = record._get_structures(None)
        self.assertEqual(expected, actual)
        actual = record._get_structures({})
        self.assertEqual(expected, actual)
        actual = record._get_structures({"data": {}})
        self.assertEqual(expected, actual)
        actual = record._get_structures({"data": {"structures": {}}})
        self.assertEqual(expected, actual)

    def test_get_structures_returns_list_for_valid_data(self):
        expected = [{}]
        actual = record._get_structures({"data": {"structures": {"STRUCTUREID": {}}}})
        self.assertEqual(expected, actual)
        structure = {"name": "Home",
                     "away": "home",
                     "time_zone": "America/New_York",
                     "postal_code": "19335",
                     "thermostats": ["THERMOSTATID0", "THERMOSTATID1"],
                     "country_code": "US",
                     "structure_id": "STRUCTUREID"}
        expected = [structure]
        actual = record._get_structures({"data": {"structures": {"STRUCTUREID": structure}}})
        self.assertEqual(expected, actual)

        for r in self.responses:
            result = record._process(r)
            actual = record._get_structures(result)
            self.assertIsInstance(actual, types.ListType)
            if result:
                self.assertEqual(1, len(actual))
                self.assertIn("structure_id", actual[0])
                self.assertIn("thermostats", actual[0])
            else:
                self.assertEqual([], actual)

    def test_get_structure_data_returns_empty_structure_for_invalid_data(self):
        expected = [{"name": "structures", "columns": [], "points": []}]
        actual = record._get_structure_data(None)
        self.assertEqual(expected, actual)
        actual = record._get_structure_data({})
        self.assertEqual(expected, actual)
        actual = record._get_structure_data({"data": {"structures": {"": {"thermostats_is_missing": "not here"}}}})
        self.assertNotIn("thermostats", actual)

    def test_get_structure_data_returns_list_for_valid_data(self):
        for r in self.responses:
            result = record._process(r)
            if result:
                actual = record._get_structure_data(result)
                self.assertIsInstance(actual, types.ListType)
                self.assertEqual("structures", actual[0]["name"])
                self.assertIsInstance(actual[0]["columns"], types.ListType)
                self.assertIsInstance(actual[0]["points"], types.ListType)
                self.assertNotIn("thermostats", actual[0])

    def test_get_thermostats_returns_empty_list_for_invalid_data(self):
        expected = []
        actual = record._get_thermostats(None)
        self.assertEqual(expected, actual)
        actual = record._get_thermostats("")
        self.assertEqual(expected, actual)
        actual = record._get_thermostats({})
        self.assertEqual(expected, actual)

    def test_get_thermostats_returns_list_for_valid_data(self):
        for r in self.responses:
            result = record._process(r)
            actual = record._get_thermostats(result)
            if actual:
                self.assertIsInstance(actual, types.ListType)
                self.assertEqual(1, len(actual))

    def test_get_thermostat_data_returns_empty_structure_for_invalid_data(self):
        expected = [{"name": "thermostats", "columns": [], "points": []}]
        actual = record._get_thermostat_data(None)
        self.assertEqual(expected, actual)
        actual = record._get_thermostat_data({})
        self.assertEqual(expected, actual)

if __name__ == "__main__":
    unittest.main(verbosity=2)
<commit_msg>record - Add test.<commit_after>#!/usr/bin/env python

# pylint: disable=protected-access,too-many-public-methods,missing-docstring

import logging
import json
import os
import types
import unittest

from mock import patch

os.environ["DEN_ACCESS_TOKEN"] = "TEST"
from den import record

logging.basicConfig(filename=os.devnull)

class RecordTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        with open("responses.txt", "r") as f:
            cls.responses = [l for l in f]

    def test_missing_env_variable_raises_key_error(self):
        with patch.dict("os.environ", {}):
            del os.environ["DEN_ACCESS_TOKEN"]
            with self.assertRaisesRegexp(KeyError, r"Please set the environment variable 'DEN_ACCESS_TOKEN'."):
                reload(record)

    def test_get_api_url(self):
        expected = "https://developer-api.nest.com?auth=%s" % record.NEST_API_ACCESS_TOKEN
        actual = record._get_api_url()
        self.assertEqual(expected, actual)

        expected = "https://developer-api.nest.com/structures?auth=%s" % record.NEST_API_ACCESS_TOKEN
        for path in ["structures", "/structures", "/structures/", "structures/"]:
            self.assertEqual(expected, record._get_api_url(path))

    def test_is_event(self):
        self.assertTrue(record._is_event("event:"))
        self.assertFalse(record._is_event("event"))
        self.assertFalse(record._is_event("data:"))

    def test_is_data(self):
        self.assertTrue(record._is_data("data:"))
        self.assertFalse(record._is_data("data"))
        self.assertFalse(record._is_data("event:"))

    def test_process_event_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_event(line))

    def test_process_event_returns_none_for_keep_alive_line(self):
        self.assertIsNone(record._process_event("event: keep-alive"))
        self.assertIsNone(record._process_event("event:keep-alive"))

    def test_process_event_returns_event_for_valid_line(self):
        expected = "Test event"
        actual = record._process_event("event: " + expected)
        self.assertEqual(expected, actual)
        actual = record._process_event("event:" + expected)
        self.assertEqual(expected, actual)

    def test_process_data_returns_none_for_invalid_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process_data(line))

    def test_process_data_returns_list_for_valid_line(self):
        expected = [{"key": "val"}]
        actual = record._process_data("data: " + json.dumps(expected))
        self.assertEqual(expected, actual)
        actual = record._process_data("data:" + json.dumps(expected))
        self.assertEqual(expected, actual)

    def test_process_returns_none_for_non_data_line(self):
        invalid_lines = [
            "",
            ":",
            "event:",
            "event: "
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process(line))

    def test_process_returns_none_for_invalid_data_line(self):
        invalid_lines = [
            "",
            ":",
            "data:",
            "data: ",
            "data: not JSON"
        ]
        for line in invalid_lines:
            self.assertIsNone(record._process(line))

    def test_process_returns_list_for_valid_data_line(self):
        expected = [{"key": "val"}]
        actual = record._process("data: " + json.dumps(expected))
        self.assertEqual(expected, actual)
        actual = record._process("data:" + json.dumps(expected))
        self.assertEqual(expected, actual)

        for r in self.responses:
            actual = record._process(r)
            if r.startswith("event"):
                self.assertIsNone(actual)
            elif r.startswith("data"):
                self.assertIsInstance(actual, types.DictType)

    def test_get_structures_returns_empty_list_for_invalid_data(self):
        expected = []
        actual = record._get_structures(None)
        self.assertEqual(expected, actual)
        actual = record._get_structures({})
        self.assertEqual(expected, actual)
        actual = record._get_structures({"data": {}})
        self.assertEqual(expected, actual)
        actual = record._get_structures({"data": {"structures": {}}})
        self.assertEqual(expected, actual)

    def test_get_structures_returns_list_for_valid_data(self):
        expected = [{}]
        actual = record._get_structures({"data": {"structures": {"STRUCTUREID": {}}}})
        self.assertEqual(expected, actual)
        structure = {"name": "Home",
                     "away": "home",
                     "time_zone": "America/New_York",
                     "postal_code": "19335",
                     "thermostats": ["THERMOSTATID0", "THERMOSTATID1"],
                     "country_code": "US",
                     "structure_id": "STRUCTUREID"}
        expected = [structure]
        actual = record._get_structures({"data": {"structures": {"STRUCTUREID": structure}}})
        self.assertEqual(expected, actual)

        for r in self.responses:
            result = record._process(r)
            actual = record._get_structures(result)
            self.assertIsInstance(actual, types.ListType)
            if result:
                self.assertEqual(1, len(actual))
                self.assertIn("structure_id", actual[0])
                self.assertIn("thermostats", actual[0])
            else:
                self.assertEqual([], actual)

    def test_get_structure_data_returns_empty_structure_for_invalid_data(self):
        expected = [{"name": "structures", "columns": [], "points": []}]
        actual = record._get_structure_data(None)
        self.assertEqual(expected, actual)
        actual = record._get_structure_data({})
        self.assertEqual(expected, actual)
        actual = record._get_structure_data({"data": {"structures": {"": {"thermostats_is_missing": "not here"}}}})
        self.assertNotIn("thermostats", actual)

    def test_get_structure_data_returns_list_for_valid_data(self):
        for r in self.responses:
            result = record._process(r)
            if result:
                actual = record._get_structure_data(result)
                self.assertIsInstance(actual, types.ListType)
                self.assertEqual("structures", actual[0]["name"])
                self.assertIsInstance(actual[0]["columns"], types.ListType)
                self.assertIsInstance(actual[0]["points"], types.ListType)
                self.assertNotIn("thermostats", actual[0])

    def test_get_thermostats_returns_empty_list_for_invalid_data(self):
        expected = []
        actual = record._get_thermostats(None)
        self.assertEqual(expected, actual)
        actual = record._get_thermostats("")
        self.assertEqual(expected, actual)
        actual = record._get_thermostats({})
        self.assertEqual(expected, actual)

    def test_get_thermostats_returns_list_for_valid_data(self):
        for r in self.responses:
            result = record._process(r)
            actual = record._get_thermostats(result)
            if actual:
                self.assertIsInstance(actual, types.ListType)
                self.assertEqual(1, len(actual))

    def test_get_thermostat_data_returns_empty_structure_for_invalid_data(self):
        expected = [{"name": "thermostats", "columns": [], "points": []}]
        actual = record._get_thermostat_data(None)
        self.assertEqual(expected, actual)
        actual = record._get_thermostat_data({})
        self.assertEqual(expected, actual)

    def test_get_thermostat_data_returns_list_for_valid_data(self):
        for r in self.responses:
            result = record._process(r)
            if result:
                actual = record._get_thermostat_data(result)
                self.assertIsInstance(actual, types.ListType)
                self.assertIsInstance(actual[0]["columns"], types.ListType)
                self.assertIsInstance(actual[0]["points"], types.ListType)
                self.assertIsInstance(actual[0]["points"][0], types.ListType)
                self.assertEqual(len(actual[0]["columns"]), len(actual[0]["points"][0]))
                self.assertEqual("thermostats", actual[0]["name"])

if __name__ == "__main__":
    unittest.main(verbosity=2)
<|endoftext|>