<commit_before>tutorial_tests = """
Let's try a simple generator:

    >>> def f():
    ...    yield 1
    ...    yield 2

    >>> for i in f():
    ...     print i
    1
    2
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    2
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 2, in g
    StopIteration

"return" stops the generator:

    >>> def f():
    ...     yield 1
    ...     return
    ...     yield 2 # never reached
    ...
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 3, in f
    StopIteration
    >>> g.next() # once stopped, can't be resumed
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration

"raise StopIteration" stops the generator too:

    >>> def f():
    ...     yield 1
    ...     return
    ...     yield 2 # never reached
    ...
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration

However, they are not exactly equivalent:

    >>> def g1():
    ...     try:
    ...         return
    ...     except:
    ...         yield 1
    ...
    >>> list(g1())
    []

    >>> def g2():
    ...     try:
    ...         raise StopIteration
    ...     except:
    ...         yield 42
    >>> print list(g2())
    [42]

This may be surprising at first:

    >>> def g3():
    ...     try:
    ...         return
    ...     finally:
    ...         yield 1
    ...
    >>> list(g3())
    [1]

Let's create an alternate range() function implemented as a generator:

    >>> def yrange(n):
    ...     for i in range(n):
    ...         yield i
    ...
    >>> list(yrange(5))
    [0, 1, 2, 3, 4]

Generators always return to the most recent caller:

    >>> def creator():
    ...     r = yrange(5)
    ...     print "creator", r.next()
    ...     return r
    ...
    >>> def caller():
    ...     r = creator()
    ...     for i in r:
    ...             print "caller", i
    ...
    >>> caller()
    creator 0
    caller 1
    caller 2
    caller 3
    caller 4

Generators can call other generators:

    >>> def zrange(n):
    ...     for i in yrange(n):
    ...         yield i
    ...
    >>> list(zrange(5))
    [0, 1, 2, 3, 4]

"""

# The examples from PEP 255.

pep_tests = """

Specification: Return

    Note that return isn't always equivalent to raising StopIteration:  the
    difference lies in how enclosing try/except constructs are treated.
    For example,

        >>> def f1():
        ...     try:
        ...         return
        ...     except:
        ...        yield 1
        >>> print list(f1())
        []

    because, as in any function, return simply exits, but

        >>> def f2():
        ...     try:
        ...         raise StopIteration
        ...     except:
        ...         yield 42
        >>> print list(f2())
        [42]

    because StopIteration is captured by a bare "except", as is any
    exception.

Specification: Generators and Exception Propagation

    >>> def f():
    ...     return 1/0
    >>> def g():
    ...     yield f()  # the zero division exception propagates
    ...     yield 42   # and we'll never get here
    >>> k = g()
    >>> k.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 2, in g
      File "<stdin>", line 2, in f
    ZeroDivisionError: integer division or modulo by zero
    >>> k.next()  # and the generator cannot be resumed
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>>

Specification: Try/Except/Finally

    >>> def f():
    ...     try:
    ...         yield 1
    ...         try:
    ...             yield 2
    ...             1/0
    ...             yield 3  # never get here
    ...         except ZeroDivisionError:
    ...             yield 4
    ...             yield 5
    ...             raise
    ...         except:
    ...             yield 6
    ...         yield 7     # the "raise" above stops this
    ...     except:
    ...         yield 8
    ...     yield 9
    ...     try:
    ...         x = 12
    ...     finally:
    ...         yield 10
    ...     yield 11
    >>> print list(f())
    [1, 2, 4, 5, 8, 9, 10, 11]
    >>>

Guido's binary tree example.

    >>> # A binary tree class.
    >>> class Tree:
    ...
    ...     def __init__(self, label, left=None, right=None):
    ...         self.label = label
    ...         self.left = left
    ...         self.right = right
    ...
    ...     def __repr__(self, level=0, indent="    "):
    ...         s = level*indent + `self.label`
    ...         if self.left:
    ...             s = s + "\\n" + self.left.__repr__(level+1, indent)
    ...         if self.right:
    ...             s = s + "\\n" + self.right.__repr__(level+1, indent)
    ...         return s
    ...
    ...     def __iter__(self):
    ...         return inorder(self)

    >>> # Create a Tree from a list.
    >>> def tree(list):
    ...     n = len(list)
    ...     if n == 0:
    ...         return []
    ...     i = n / 2
    ...     return Tree(list[i], tree(list[:i]), tree(list[i+1:]))

    >>> # Show it off: create a tree.
    >>> t = tree("ABCDEFGHIJKLMNOPQRSTUVWXYZ")

    >>> # A recursive generator that generates Tree leaves in in-order.
    >>> def inorder(t):
    ...     if t:
    ...         for x in inorder(t.left):
    ...             yield x
    ...         yield t.label
    ...         for x in inorder(t.right):
    ...             yield x

    >>> # Show it off: create a tree.
    ... t = tree("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
    ... # Print the nodes of the tree in in-order.
    ... for x in t:
    ...     print x,
    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z

    >>> # A non-recursive generator.
    >>> def inorder(node):
    ...     stack = []
    ...     while node:
    ...         while node.left:
    ...             stack.append(node)
    ...             node = node.left
    ...         yield node.label
    ...         while not node.right:
    ...             try:
    ...                 node = stack.pop()
    ...             except IndexError:
    ...                 return
    ...             yield node.label
    ...         node = node.right

    >>> # Exercise the non-recursive generator.
    >>> for x in t:
    ...     print x,
    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z

"""

# A few examples from Iterator-List and Python-Dev email.

email_tests = """

The difference between yielding None and returning it.

>>> def g():
...     for i in range(3):
...         yield None
...     yield None
...     return
>>> list(g())
[None, None, None, None]

Ensure that explicitly raising StopIteration acts like any other exception
in try/except, not like a return.

>>> def g():
...     yield 1
...     try:
...         raise StopIteration
...     except:
...         yield 2
...     yield 3
>>> list(g())
[1, 2, 3]

A generator can't be resumed while it's already running.

>>> def g():
...     i = me.next()
...     yield i
>>> me = g()
>>> me.next()
Traceback (most recent call last):
 ...
  File "<string>", line 2, in g
ValueError: generator already executing
"""

# Fun tests (for sufficiently warped notions of "fun").

fun_tests = """

Build up to a recursive Sieve of Eratosthenes generator.

>>> def firstn(g, n):
...     return [g.next() for i in range(n)]

>>> def intsfrom(i):
...     while 1:
...         yield i
...         i += 1

>>> firstn(intsfrom(5), 7)
[5, 6, 7, 8, 9, 10, 11]

>>> def exclude_multiples(n, ints):
...     for i in ints:
...         if i % n:
...             yield i

>>> firstn(exclude_multiples(3, intsfrom(1)), 6)
[1, 2, 4, 5, 7, 8]

>>> def sieve(ints):
...     prime = ints.next()
...     yield prime
...     not_divisible_by_prime = exclude_multiples(prime, ints)
...     for p in sieve(not_divisible_by_prime):
...         yield p

>>> primes = sieve(intsfrom(2))
>>> firstn(primes, 20)
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71]

Another famous problem:  generate all integers of the form
    2**i * 3**j  * 5**k
in increasing order, where i,j,k >= 0.  Trickier than it may look at first!
Try writing it without generators, and correctly, and without generating
3 internal results for each result output.

>>> def times(n, g):
...     for i in g:
...         yield n * i
>>> firstn(times(10, intsfrom(1)), 10)
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100]

>>> def merge(g, h):
...     ng = g.next()
...     nh = h.next()
...     while 1:
...         if ng < nh:
...             yield ng
...             ng = g.next()
...         elif ng > nh:
...             yield nh
...             nh = h.next()
...         else:
...             yield ng
...             ng = g.next()
...             nh = h.next()

This works, but is doing a whale of a lot or redundant work -- it's not
clear how to get the internal uses of m235 to share a single generator.
Note that me_times2 (etc) each need to see every element in the result
sequence.  So this is an example where lazy lists are more natural (you
can look at the head of a lazy list any number of times).

>>> def m235():
...     yield 1
...     me_times2 = times(2, m235())
...     me_times3 = times(3, m235())
...     me_times5 = times(5, m235())
...     for i in merge(merge(me_times2,
...                          me_times3),
...                    me_times5):
...         yield i

>>> result = m235()
>>> for i in range(5):
...     print firstn(result, 15)
[1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 16, 18, 20, 24]
[25, 27, 30, 32, 36, 40, 45, 48, 50, 54, 60, 64, 72, 75, 80]
[81, 90, 96, 100, 108, 120, 125, 128, 135, 144, 150, 160, 162, 180, 192]
[200, 216, 225, 240, 243, 250, 256, 270, 288, 300, 320, 324, 360, 375, 384]
[400, 405, 432, 450, 480, 486, 500, 512, 540, 576, 600, 625, 640, 648, 675]
"""

__test__ = {"tut": tutorial_tests,
            "pep": pep_tests,
            "email": email_tests,
            "fun": fun_tests}

# Magic test name that regrtest.py invokes *after* importing this module.
# This worms around a bootstrap problem.
# Note that doctest and regrtest both look in sys.argv for a "-v" argument,
# so this works as expected in both ways of running regrtest.
def test_main():
    import doctest, test_generators
    doctest.testmod(test_generators)

# This part isn't needed for regrtest, but for running the test directly.
if __name__ == "__main__":
    test_main()
<commit_msg>Another variant of the 2-3-5 test, mixing generators with a LazyList class. Good news:  Some of this stuff is pretty sophisticated (read nuts), and I haven't bumped into a bug yet. Bad news:  If I run the doctest in an infinite loop, memory is clearly leaking.<commit_after>tutorial_tests = """
Let's try a simple generator:

    >>> def f():
    ...    yield 1
    ...    yield 2

    >>> for i in f():
    ...     print i
    1
    2
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    2
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 2, in g
    StopIteration

"return" stops the generator:

    >>> def f():
    ...     yield 1
    ...     return
    ...     yield 2 # never reached
    ...
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 3, in f
    StopIteration
    >>> g.next() # once stopped, can't be resumed
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration

"raise StopIteration" stops the generator too:

    >>> def f():
    ...     yield 1
    ...     return
    ...     yield 2 # never reached
    ...
    >>> g = f()
    >>> g.next()
    1
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>> g.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration

However, they are not exactly equivalent:

    >>> def g1():
    ...     try:
    ...         return
    ...     except:
    ...         yield 1
    ...
    >>> list(g1())
    []

    >>> def g2():
    ...     try:
    ...         raise StopIteration
    ...     except:
    ...         yield 42
    >>> print list(g2())
    [42]

This may be surprising at first:

    >>> def g3():
    ...     try:
    ...         return
    ...     finally:
    ...         yield 1
    ...
    >>> list(g3())
    [1]

Let's create an alternate range() function implemented as a generator:

    >>> def yrange(n):
    ...     for i in range(n):
    ...         yield i
    ...
    >>> list(yrange(5))
    [0, 1, 2, 3, 4]

Generators always return to the most recent caller:

    >>> def creator():
    ...     r = yrange(5)
    ...     print "creator", r.next()
    ...     return r
    ...
    >>> def caller():
    ...     r = creator()
    ...     for i in r:
    ...             print "caller", i
    ...
    >>> caller()
    creator 0
    caller 1
    caller 2
    caller 3
    caller 4

Generators can call other generators:

    >>> def zrange(n):
    ...     for i in yrange(n):
    ...         yield i
    ...
    >>> list(zrange(5))
    [0, 1, 2, 3, 4]

"""

# The examples from PEP 255.

pep_tests = """

Specification: Return

    Note that return isn't always equivalent to raising StopIteration:  the
    difference lies in how enclosing try/except constructs are treated.
    For example,

        >>> def f1():
        ...     try:
        ...         return
        ...     except:
        ...        yield 1
        >>> print list(f1())
        []

    because, as in any function, return simply exits, but

        >>> def f2():
        ...     try:
        ...         raise StopIteration
        ...     except:
        ...         yield 42
        >>> print list(f2())
        [42]

    because StopIteration is captured by a bare "except", as is any
    exception.

Specification: Generators and Exception Propagation

    >>> def f():
    ...     return 1/0
    >>> def g():
    ...     yield f()  # the zero division exception propagates
    ...     yield 42   # and we'll never get here
    >>> k = g()
    >>> k.next()
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
      File "<stdin>", line 2, in g
      File "<stdin>", line 2, in f
    ZeroDivisionError: integer division or modulo by zero
    >>> k.next()  # and the generator cannot be resumed
    Traceback (most recent call last):
      File "<stdin>", line 1, in ?
    StopIteration
    >>>

Specification: Try/Except/Finally

    >>> def f():
    ...     try:
    ...         yield 1
    ...         try:
    ...             yield 2
    ...             1/0
    ...             yield 3  # never get here
    ...         except ZeroDivisionError:
    ...             yield 4
    ...             yield 5
    ...             raise
    ...         except:
    ...             yield 6
    ...         yield 7     # the "raise" above stops this
    ...     except:
    ...         yield 8
    ...     yield 9
    ...     try:
    ...         x = 12
    ...     finally:
    ...         yield 10
    ...     yield 11
    >>> print list(f())
    [1, 2, 4, 5, 8, 9, 10, 11]
    >>>

Guido's binary tree example.

    >>> # A binary tree class.
    >>> class Tree:
    ...
    ...     def __init__(self, label, left=None, right=None):
    ...         self.label = label
    ...         self.left = left
    ...         self.right = right
    ...
    ...     def __repr__(self, level=0, indent="    "):
    ...         s = level*indent + `self.label`
    ...         if self.left:
    ...             s = s + "\\n" + self.left.__repr__(level+1, indent)
    ...         if self.right:
    ...             s = s + "\\n" + self.right.__repr__(level+1, indent)
    ...         return s
    ...
    ...     def __iter__(self):
    ...         return inorder(self)

    >>> # Create a Tree from a list.
    >>> def tree(list):
    ...     n = len(list)
    ...     if n == 0:
    ...         return []
    ...     i = n / 2
    ...     return Tree(list[i], tree(list[:i]), tree(list[i+1:]))

    >>> # Show it off: create a tree.
    >>> t = tree("ABCDEFGHIJKLMNOPQRSTUVWXYZ")

    >>> # A recursive generator that generates Tree leaves in in-order.
    >>> def inorder(t):
    ...     if t:
    ...         for x in inorder(t.left):
    ...             yield x
    ...         yield t.label
    ...         for x in inorder(t.right):
    ...             yield x

    >>> # Show it off: create a tree.
    ... t = tree("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
    ... # Print the nodes of the tree in in-order.
    ... for x in t:
    ...     print x,
    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z

    >>> # A non-recursive generator.
    >>> def inorder(node):
    ...     stack = []
    ...     while node:
    ...         while node.left:
    ...             stack.append(node)
    ...             node = node.left
    ...         yield node.label
    ...         while not node.right:
    ...             try:
    ...                 node = stack.pop()
    ...             except IndexError:
    ...                 return
    ...             yield node.label
    ...         node = node.right

    >>> # Exercise the non-recursive generator.
    >>> for x in t:
    ...     print x,
    A B C D E F G H I J K L M N O P Q R S T U V W X Y Z

"""

# A few examples from Iterator-List and Python-Dev email.

email_tests = """

The difference between yielding None and returning it.

>>> def g():
...     for i in range(3):
...         yield None
...     yield None
...     return
>>> list(g())
[None, None, None, None]

Ensure that explicitly raising StopIteration acts like any other exception
in try/except, not like a return.

>>> def g():
...     yield 1
...     try:
...         raise StopIteration
...     except:
...         yield 2
...     yield 3
>>> list(g())
[1, 2, 3]

A generator can't be resumed while it's already running.

>>> def g():
...     i = me.next()
...     yield i
>>> me = g()
>>> me.next()
Traceback (most recent call last):
 ...
  File "<string>", line 2, in g
ValueError: generator already executing
"""

# Fun tests (for sufficiently warped notions of "fun").

fun_tests = """

Build up to a recursive Sieve of Eratosthenes generator.

>>> def firstn(g, n):
...     return [g.next() for i in range(n)]

>>> def intsfrom(i):
...     while 1:
...         yield i
...         i += 1

>>> firstn(intsfrom(5), 7)
[5, 6, 7, 8, 9, 10, 11]

>>> def exclude_multiples(n, ints):
...     for i in ints:
...         if i % n:
...             yield i

>>> firstn(exclude_multiples(3, intsfrom(1)), 6)
[1, 2, 4, 5, 7, 8]

>>> def sieve(ints):
...     prime = ints.next()
...     yield prime
...     not_divisible_by_prime = exclude_multiples(prime, ints)
...     for p in sieve(not_divisible_by_prime):
...         yield p

>>> primes = sieve(intsfrom(2))
>>> firstn(primes, 20)
[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71]

Another famous problem:  generate all integers of the form
    2**i * 3**j  * 5**k
in increasing order, where i,j,k >= 0.  Trickier than it may look at first!
Try writing it without generators, and correctly, and without generating
3 internal results for each result output.

>>> def times(n, g):
...     for i in g:
...         yield n * i
>>> firstn(times(10, intsfrom(1)), 10)
[10, 20, 30, 40, 50, 60, 70, 80, 90, 100]

>>> def merge(g, h):
...     ng = g.next()
...     nh = h.next()
...     while 1:
...         if ng < nh:
...             yield ng
...             ng = g.next()
...         elif ng > nh:
...             yield nh
...             nh = h.next()
...         else:
...             yield ng
...             ng = g.next()
...             nh = h.next()

This works, but is doing a whale of a lot or redundant work -- it's not
clear how to get the internal uses of m235 to share a single generator.
Note that me_times2 (etc) each need to see every element in the result
sequence.  So this is an example where lazy lists are more natural (you
can look at the head of a lazy list any number of times).

>>> def m235():
...     yield 1
...     me_times2 = times(2, m235())
...     me_times3 = times(3, m235())
...     me_times5 = times(5, m235())
...     for i in merge(merge(me_times2,
...                          me_times3),
...                    me_times5):
...         yield i

>>> result = m235()
>>> for i in range(5):
...     print firstn(result, 15)
[1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 16, 18, 20, 24]
[25, 27, 30, 32, 36, 40, 45, 48, 50, 54, 60, 64, 72, 75, 80]
[81, 90, 96, 100, 108, 120, 125, 128, 135, 144, 150, 160, 162, 180, 192]
[200, 216, 225, 240, 243, 250, 256, 270, 288, 300, 320, 324, 360, 375, 384]
[400, 405, 432, 450, 480, 486, 500, 512, 540, 576, 600, 625, 640, 648, 675]

Heh.  Here's one way to get a shared list, complete with an excruciating
namespace renaming trick.  The *pretty* part is that the times() and merge()
functions can be reused as-is, because they only assume their stream
arguments are iterable -- a LazyList is the same as a generator to times().

>>> class LazyList:
...     def __init__(self, g):
...         self.sofar = []
...         self.fetch = g.next
...
...     def __getitem__(self, i):
...         sofar, fetch = self.sofar, self.fetch
...         while i >= len(sofar):
...             sofar.append(fetch())
...         return sofar[i]

>>> def m235():
...     yield 1
...     # Gack:  m235  below actually refers to a LazyList.
...     me_times2 = times(2, m235)
...     me_times3 = times(3, m235)
...     me_times5 = times(5, m235)
...     for i in merge(merge(me_times2,
...                          me_times3),
...                    me_times5):
...         yield i

>>> m235 = LazyList(m235())
>>> for i in range(5):
...     print [m235[j] for j in range(15*i, 15*(i+1))]
[1, 2, 3, 4, 5, 6, 8, 9, 10, 12, 15, 16, 18, 20, 24]
[25, 27, 30, 32, 36, 40, 45, 48, 50, 54, 60, 64, 72, 75, 80]
[81, 90, 96, 100, 108, 120, 125, 128, 135, 144, 150, 160, 162, 180, 192]
[200, 216, 225, 240, 243, 250, 256, 270, 288, 300, 320, 324, 360, 375, 384]
[400, 405, 432, 450, 480, 486, 500, 512, 540, 576, 600, 625, 640, 648, 675]
"""

__test__ = {"tut": tutorial_tests,
            "pep": pep_tests,
            "email": email_tests,
            "fun": fun_tests}

# Magic test name that regrtest.py invokes *after* importing this module.
# This worms around a bootstrap problem.
# Note that doctest and regrtest both look in sys.argv for a "-v" argument,
# so this works as expected in both ways of running regrtest.
def test_main():
    import doctest, test_generators
    doctest.testmod(test_generators)

# This part isn't needed for regrtest, but for running the test directly.
if __name__ == "__main__":
    test_main()
<|endoftext|>


<commit_before>import pendulum

from share.disambiguation.criteria import MatchByOneToMany
from share.exceptions import MergeRequired
from share.models import AbstractCreativeWork
from share.management.commands import BaseShareCommand
from share.models import IngestJob
from share.tasks.jobs import IngestJobConsumer
from share.util import ensure_iterable

class Command(BaseShareCommand):
    MAX_RETRIES = 10

    def add_arguments(self, parser):
        parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)')
        parser.add_argument('--commit', action='store_true', help='Should the script actually commit?')
        parser.add_argument('--noninteractive', action='store_true', help='Should the script merge objects without asking?')
        parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or after this date')
        parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or before this date')

    def handle(self, *args, **options):
        dry_run = options.get('dry')
        interactive = not options.get('noninteractive')

        qs = IngestJob.objects.filter(
            error_type='MergeRequired',
            status=IngestJob.STATUS.failed,
            source_config__source__canonical=True
        ).order_by('date_created')
        if options.get('from'):
            qs = qs.filter(date_modified__gte=options.get('from'))
        if options.get('until'):
            qs = qs.filter(date_modified__lte=options.get('until'))

        for job in qs.select_related('suid', 'source_config'):
            with self.rollback_unless_commit(options.get('commit')):
                self.stdout.write('\n\nTrying job {!r}'.format(job), style_func=self.style.HTTP_INFO)
                self._try_job(job, dry_run, interactive)

    def _try_job(self, job, dry_run, interactive):
        for i in range(self.MAX_RETRIES):
            self.stdout.write('Attempt {} of {}:'.format(i + 1, self.MAX_RETRIES))
            try:
                IngestJobConsumer().consume(job_id=job.id, exhaust=False)
            except MergeRequired as e:
                (_, *dupe_sets) = e.args

                if not self._fix_merge_error(dupe_sets, dry_run, interactive):
                    return

                if dry_run:
                    return
                continue
            except Exception as e:
                self.stdout.write('Failed in a way we cant fix:', style_func=self.style.ERROR)
                self.stdout.write(str(e))
                return
            self.stdout.write('Success!', style_func=self.style.SUCCESS)
            return
        self.stdout.write('Failed to fix after {} tries'.format(self.MAX_RETRIES), style_func=self.style.ERROR)

    def _fix_merge_error(self, dupe_sets, dry_run, interactive):
        for dupes in dupe_sets:
            if len(dupes) < 2:
                continue
            model = dupes[0]._meta.concrete_model
            if any(d._meta.concrete_model is not model for d in dupes):
                raise ValueError('Cannot merge across tables: {}'.format(dupes))

            if not self._merge_together(model, dupes, dry_run, interactive):
                return False
        return True

    def _merge_together(self, model, dupes, dry_run, interactive):
        criteria = [
            c for c in ensure_iterable(model.matching_criteria)
            if isinstance(c, MatchByOneToMany)
        ]

        if len(criteria) != 1:
            self.stdout.write('Cannot automatically fix errors on {}, need a MatchByOneToMany criterion'.format(model), style_func=self.style.ERROR)
            return False

        ids = [d.id for d in dupes]
        field_name = criteria[0].relation_name
        relation_field = model._meta.get_field(field_name)
        remote_field = relation_field.remote_field
        qs = relation_field.related_model.objects.filter(**{remote_field.name + '__in': ids})

        conflicts = {}
        for inst in qs:
            conflicts.setdefault(getattr(inst, remote_field.name), []).append(inst)

        (winner, _), *conflicts = sorted(conflicts.items(), key=lambda x: len(x[1]), reverse=True)

        self.stdout.write('\tMerging extra {} into {!r}'.format(model._meta.verbose_name_plural, winner))
        for conflict, evidence in conflicts:
            for inst in evidence:
                self.stdout.write('\t\t{!r}: {!r} -> {!r}'.format(inst, getattr(inst, remote_field.name), winner))

        if interactive and not self.input_confirm('OK? (y/n) '):
            self.stdout.write('Skipping...', style_func=self.style.WARNING)
            return False

        for conflict, evidence in conflicts:
            if not dry_run:
                for inst in evidence:
                    inst.administrative_change(**{remote_field.name: winner})
            if isinstance(conflict, AbstractCreativeWork) and not conflict.identifiers.exists():
                self.stdout.write('\t\tDeleting {!r}'.format(conflict))
                if not dry_run:
                    conflict.administrative_change(is_deleted=True)

        self.stdout.write('Corrected merge error!', style_func=self.style.SUCCESS)
        return True
<commit_msg>Handle non-list dupes<commit_after>import pendulum

from share.disambiguation.criteria import MatchByOneToMany
from share.exceptions import MergeRequired
from share.models import AbstractCreativeWork
from share.management.commands import BaseShareCommand
from share.models import IngestJob
from share.tasks.jobs import IngestJobConsumer
from share.util import ensure_iterable

class Command(BaseShareCommand):
    MAX_RETRIES = 10

    def add_arguments(self, parser):
        parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)')
        parser.add_argument('--commit', action='store_true', help='Should the script actually commit?')
        parser.add_argument('--noninteractive', action='store_true', help='Should the script merge objects without asking?')
        parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or after this date')
        parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or before this date')

    def handle(self, *args, **options):
        dry_run = options.get('dry')
        interactive = not options.get('noninteractive')

        qs = IngestJob.objects.filter(
            error_type='MergeRequired',
            status=IngestJob.STATUS.failed,
            source_config__source__canonical=True
        ).order_by('date_created')
        if options.get('from'):
            qs = qs.filter(date_modified__gte=options.get('from'))
        if options.get('until'):
            qs = qs.filter(date_modified__lte=options.get('until'))

        for job in qs.select_related('suid', 'source_config'):
            with self.rollback_unless_commit(options.get('commit')):
                self.stdout.write('\n\nTrying job {!r}'.format(job), style_func=self.style.HTTP_INFO)
                self._try_job(job, dry_run, interactive)

    def _try_job(self, job, dry_run, interactive):
        for i in range(self.MAX_RETRIES):
            self.stdout.write('Attempt {} of {}:'.format(i + 1, self.MAX_RETRIES))
            try:
                IngestJobConsumer().consume(job_id=job.id, exhaust=False)
            except MergeRequired as e:
                (_, *dupe_sets) = e.args

                if not self._fix_merge_error(dupe_sets, dry_run, interactive):
                    return

                if dry_run:
                    return
                continue
            except Exception as e:
                self.stdout.write('Failed in a way we cant fix:', style_func=self.style.ERROR)
                self.stdout.write(str(e))
                return
            self.stdout.write('Success!', style_func=self.style.SUCCESS)
            return
        self.stdout.write('Failed to fix after {} tries'.format(self.MAX_RETRIES), style_func=self.style.ERROR)

    def _fix_merge_error(self, dupe_sets, dry_run, interactive):
        for dupes in dupe_sets:
            if len(dupes) < 2:
                continue
            model = list(dupes)[0]._meta.concrete_model
            if any(d._meta.concrete_model is not model for d in dupes):
                raise ValueError('Cannot merge across tables: {}'.format(dupes))

            if not self._merge_together(model, dupes, dry_run, interactive):
                return False
        return True

    def _merge_together(self, model, dupes, dry_run, interactive):
        criteria = [
            c for c in ensure_iterable(model.matching_criteria)
            if isinstance(c, MatchByOneToMany)
        ]

        if len(criteria) != 1:
            self.stdout.write('Cannot automatically fix errors on {}, need a MatchByOneToMany criterion'.format(model), style_func=self.style.ERROR)
            return False

        ids = [d.id for d in dupes]
        field_name = criteria[0].relation_name
        relation_field = model._meta.get_field(field_name)
        remote_field = relation_field.remote_field
        qs = relation_field.related_model.objects.filter(**{remote_field.name + '__in': ids})

        conflicts = {}
        for inst in qs:
            conflicts.setdefault(getattr(inst, remote_field.name), []).append(inst)

        (winner, _), *conflicts = sorted(conflicts.items(), key=lambda x: len(x[1]), reverse=True)

        self.stdout.write('\tMerging extra {} into {!r}'.format(model._meta.verbose_name_plural, winner))
        for conflict, evidence in conflicts:
            for inst in evidence:
                self.stdout.write('\t\t{!r}: {!r} -> {!r}'.format(inst, getattr(inst, remote_field.name), winner))

        if interactive and not self.input_confirm('OK? (y/n) '):
            self.stdout.write('Skipping...', style_func=self.style.WARNING)
            return False

        for conflict, evidence in conflicts:
            if not dry_run:
                for inst in evidence:
                    inst.administrative_change(**{remote_field.name: winner})
            if isinstance(conflict, AbstractCreativeWork) and not conflict.identifiers.exists():
                self.stdout.write('\t\tDeleting {!r}'.format(conflict))
                if not dry_run:
                    conflict.administrative_change(is_deleted=True)

        self.stdout.write('Corrected merge error!', style_func=self.style.SUCCESS)
        return True
<|endoftext|>


<commit_before>from __future__ import division

# LIBTBX_SET_DISPATCHER_NAME dev.dials.duelling_profiles

import iotbx.phil

phil_scope = iotbx.phil.parse("""\
  method = *example rt0 flat
    .type = choice
  id = None
    .type = int
    .multiple = True
  id_start = None
    .type = int
  id_end = None
    .type = int
  scale = 1.0
    .type = float
  show = False
    .type = bool
  rs_node_size = 0.0
    .type = float
  min_isum = None
    .type = float
  debug = False
    .type = bool
""", process_includes=True)

help_message = '''

Examples::

  dev.dials.duelling_profiles experiments.json integrated.pickle

'''

def model_background(shoebox, mean_bg):
  from scitbx.random import variate, poisson_distribution
  dz, dy, dx = shoebox.focus()
  g = variate(poisson_distribution(mean = mean_bg))
  for k in range(dz):
    for j in range(dy):
      for i in range(dx):
        shoebox[k, j, i] += g.next()
  return

def random_vector_2D(vector, sd):
  '''Form random orthonormal basis including vector, rotate vector by random
  amount sd in both normal directions.'''
  import random
  o0 = vector.ortho()
  o1 = vector.cross(o0)
  return vector.rotate(o0, random.gauss(0, sd)).rotate(o1, random.gauss(0, sd))

def random_vector_cone(vector, sd):
  '''Pick orthogonal axis to vector, rotate by random angle about vector,
  rotate vector about this by sd in radians.'''
  import random
  import math
  o0 = vector.ortho()
  o1 = vector.rotate(o0, random.random() * 2.0 * math.pi)
  return vector.rotate(o1, random.gauss(0, sd))

def model_reflection_example(reflection, experiment, params):
  hkl = reflection['miller_index']
  i0 = reflection['intensity.sum.value'] / reflection['dqe']
  s1 = reflection['s1']
  xyz = reflection['xyzcal.px']
  pixels = reflection['shoebox']
  mean_bg = reflection['background.mean']
  crystal = experiment.crystal
  profile = experiment.profile
  Amat = crystal.get_A_at_scan_point(int(xyz[2]))
  return

def predict_angles(p0_star, experiment, s0=None):
  '''Predict Ewald sphere crossing angle for RLP x'''

  from scitbx import matrix
  import math

  # Kabsch frame
  a = matrix.col(experiment.goniometer.get_rotation_axis())
  if s0 is None:
    b = matrix.col(experiment.beam.get_s0())
  else:
    b = s0

  m2 = a.normalize()
  m1 = m2.cross(b.normalize())
  m3 = m1.cross(m2)

  p0_sqr = p0_star.dot(p0_star)
  rho = math.sqrt(p0_sqr - p0_star.dot(m2) ** 2)
  p_star_m3 = (-0.5 * p0_sqr - p0_star.dot(m2) * b.dot(m2)) / b.dot(m3)
  p_star_m2 = p0_star.dot(m2)
  p_star_m1 = math.sqrt(rho ** 2 - p_star_m3 ** 2)

  p0_star_m1 = p0_star.dot(m1)
  p0_star_m2 = p0_star.dot(m2)
  p0_star_m3 = p0_star.dot(m3)

  cp1 = + p_star_m1 * p0_star_m1 +  p_star_m3 * p0_star_m3
  cp2 = - p_star_m1 * p0_star_m1 +  p_star_m3 * p0_star_m3
  sp1 = + p_star_m1 * p0_star_m3 -  p_star_m3 * p0_star_m1
  sp2 = - p_star_m1 * p0_star_m3 -  p_star_m3 * p0_star_m1

  # FIXME determine order of these (entering, exiting) so usage may be
  # more efficient...

  return math.atan2(sp1, cp1), math.atan2(sp2, cp2)

def profile_correlation(data, model):
  '''Compute CC between reflection profiles data and model.'''

  from dials.array_family import flex

  assert(data.focus() == model.focus())

  sel = data.as_1d() > 0

  model = model.as_1d().select(sel)
  data = data.as_1d().select(sel).as_double()

  assert(len(data) == len(model))

  correlation = flex.linear_correlation(data, model)
  return correlation.coefficient()

def abs_angle(a, b):
  import math
  return abs(math.atan2(math.sin(b - a), math.cos(b - a)))

def model_reflection_rt0(reflection, experiment, params):
  import math
  from scitbx import matrix
  from dials.array_family import flex

  d2r = math.pi / 180.0

  hkl = reflection['miller_index']
  xyz = reflection['xyzcal.px']
  xyz_mm = reflection['xyzcal.mm']

  if params.debug:
    print 'hkl = %d %d %d' % hkl
    print 'xyz px = %f %f %f' % xyz
    print 'xyz mm = %f %f %f' % xyz_mm

  Amat = matrix.sqr(experiment.crystal.get_A_at_scan_point(int(xyz[2])))
  p0_star = Amat * hkl

  angles = predict_angles(p0_star, experiment)

  if params.debug:
    print 'angles = %f %f' % angles

  angle = angles[0] if (abs(angles[0] - xyz_mm[2]) <
                        abs(angles[1] - xyz_mm[2])) else angles[1]

  i0 = reflection['intensity.sum.value']# / reflection['dqe']

  if params.min_isum:
    if i0 < params.min_isum:
      return

  s1 = reflection['s1']
  a = matrix.col(experiment.goniometer.get_rotation_axis())
  s0 = matrix.col(experiment.beam.get_s0())

  if params.debug:
    print 's1 = %f %f %f' % s1

  pixels = reflection['shoebox']
  pixels.flatten()
  data = pixels.data
  dz, dy, dx = data.focus()

  # since now 2D data
  data.reshape(flex.grid(dy, dx))

  if params.show:
    print 'Observed reflection (flattened in Z):'
    print
    for j in range(dy):
      for i in range(dx):
        print '%4d' % data[(j, i)],
      print

  sigma_m = experiment.profile.sigma_m() * d2r
  sigma_b = experiment.profile.sigma_b() * d2r

  r0 = xyz_mm[2]

  detector = experiment.detector

  patch = flex.double(dy * dx, 0)
  patch.reshape(flex.grid(dy, dx))

  bbox = reflection['bbox']

  scale = params.scale
  if params.show:
    print '%d rays' % (int(round(i0 * scale)))
  for i in range(int(round(i0 * scale))):
    b = random_vector_cone(s0, sigma_b)
    p0 = random_vector_cone(Amat * hkl, sigma_m)
    if params.rs_node_size > 0:
      ns = params.rs_node_size
      import random
      dp0 = matrix.col((random.gauss(0, ns),
                        random.gauss(0, ns),
                        random.gauss(0, ns)))
      p0 += dp0
    angles = predict_angles(p0, experiment, b)
    r = angles[0] if (abs_angle(angles[0], r0) <
                      abs_angle(angles[1], r0)) else angles[1]
    p = p0.rotate(a, r)
    s1 = p + b
    panel, xy = detector.get_ray_intersection(s1)

    # FIXME DO NOT USE THIS FUNCTION EVENTUALLY...
    x, y = detector[panel].millimeter_to_pixel(xy)
    if x < bbox[0] or x >= bbox[1]:
      continue
    if y < bbox[2] or y >= bbox[3]:
      continue
    x -= bbox[0]
    y -= bbox[2]
    # FIXME in here try to work out probability distribution along path
    # length through the detector sensitive surface i.e. deposit fractional
    # counts along pixels (and allow for DQE i.e. photon passing right through
    # the detector)
    patch[(int(y), int(x))] += 1.0 / scale

  if params.show:
    print 'Simulated reflection (flattened in Z):'
    print
    for j in range(dy):
      for i in range(dx):
        print '%4d' % int(patch[(j, i)]),
      print

  print 'Correlation coefficient: %.3f isum: %.1f ' % (
      profile_correlation(data, patch), i0)

  return

def model_reflection_flat(reflection, experiment, params):
  pixels = reflection['shoebox']
  pixels.flatten()
  data = pixels.data
  dz, dy, dx = data.focus()
  return

def main(reflections, experiment, params):
  nref0 = len(reflections)

  method = params.method
  ids = params.id

  if 'intensity.prf.variance' in reflections:
    selection = reflections.get_flags(
      reflections.flags.integrated,
      all=True)
  else:
    selection = reflections.get_flags(
      reflections.flags.integrated_sum)
  reflections = reflections.select(selection)

  nref1 = len(reflections)

  print 'Removed %d invalid reflections, %d remain' % (nref0 - nref1, nref1)

  for j, reflection in enumerate(reflections):
    if ids:
      if j in ids:
        globals()['model_reflection_%s' % method](reflection, experiment, params)
    elif params.id_start and params.id_end:
      if j < params.id_start or j >= params.id_end:
        continue
      print j
      globals()['model_reflection_%s' % method](reflection, experiment, params)

    else:
      print j
      globals()['model_reflection_%s' % method](reflection, experiment, params)

  return

def run(args):
  from dials.util.options import OptionParser
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_reflections
  import libtbx.load_env

  usage = "%s [options] integrated.pickle experiments.json" % (
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=False)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if len(experiments) != 1 or len(reflections) != 1:
    parser.print_help()
    exit()

  if not 'shoebox' in reflections[0]:
    print 'Please add shoeboxes to reflection pickle'
    exit()

  main(reflections[0], experiments[0], params)

if __name__ == '__main__':
  import sys
  run(sys.argv[1:])
<commit_msg>identified prediction order; added test for this<commit_after>from __future__ import division

# LIBTBX_SET_DISPATCHER_NAME dev.dials.duelling_profiles

import iotbx.phil

phil_scope = iotbx.phil.parse("""\
  method = *example rt0 flat predict
    .type = choice
  id = None
    .type = int
    .multiple = True
  id_start = None
    .type = int
  id_end = None
    .type = int
  scale = 1.0
    .type = float
  show = False
    .type = bool
  rs_node_size = 0.0
    .type = float
  min_isum = None
    .type = float
  debug = False
    .type = bool
""", process_includes=True)

help_message = '''

Examples::

  dev.dials.duelling_profiles experiments.json integrated.pickle

'''

def model_background(shoebox, mean_bg):
  from scitbx.random import variate, poisson_distribution
  dz, dy, dx = shoebox.focus()
  g = variate(poisson_distribution(mean = mean_bg))
  for k in range(dz):
    for j in range(dy):
      for i in range(dx):
        shoebox[k, j, i] += g.next()
  return

def random_vector_2D(vector, sd):
  '''Form random orthonormal basis including vector, rotate vector by random
  amount sd in both normal directions.'''
  import random
  o0 = vector.ortho()
  o1 = vector.cross(o0)
  return vector.rotate(o0, random.gauss(0, sd)).rotate(o1, random.gauss(0, sd))

def random_vector_cone(vector, sd):
  '''Pick orthogonal axis to vector, rotate by random angle about vector,
  rotate vector about this by sd in radians.'''
  import random
  import math
  o0 = vector.ortho()
  o1 = vector.rotate(o0, random.random() * 2.0 * math.pi)
  return vector.rotate(o1, random.gauss(0, sd))

def model_reflection_example(reflection, experiment, params):
  hkl = reflection['miller_index']
  i0 = reflection['intensity.sum.value'] / reflection['dqe']
  s1 = reflection['s1']
  xyz = reflection['xyzcal.px']
  pixels = reflection['shoebox']
  mean_bg = reflection['background.mean']
  crystal = experiment.crystal
  profile = experiment.profile
  Amat = crystal.get_A_at_scan_point(int(xyz[2]))
  return

def model_reflection_predict(reflection, experiment, params):
  import math
  from scitbx import matrix
  from dials.array_family import flex

  d2r = math.pi / 180.0

  hkl = reflection['miller_index']
  xyz = reflection['xyzcal.px']
  xyz_mm = reflection['xyzcal.mm']

  if params.debug:
    print 'hkl = %d %d %d' % hkl
    print 'xyz px = %f %f %f' % xyz
    print 'xyz mm = %f %f %f' % xyz_mm

  Amat = matrix.sqr(experiment.crystal.get_A_at_scan_point(int(xyz[2])))
  p0_star = Amat * hkl

  angles = predict_angles(p0_star, experiment)

  if params.debug:
    print 'angles = %f %f' % angles

  angle = angles[0] if reflection['entering'] else angles[1]

  if abs_angle(angle, xyz_mm[2]) > 1.0e-3:
    raise RuntimeError, '%f %f' % (angle, xyz_mm[2])

  return

def predict_angles(p0_star, experiment, s0=None):
  '''Predict Ewald sphere crossing angle for RLP x, returned in order
  (entering, exiting).'''

  from scitbx import matrix
  import math

  a = matrix.col(experiment.goniometer.get_rotation_axis())
  if s0 is None:
    b = matrix.col(experiment.beam.get_s0())
  else:
    b = s0

  m2 = a.normalize()
  m1 = m2.cross(b.normalize())
  m3 = m1.cross(m2)

  p0_sqr = p0_star.dot(p0_star)
  rho = math.sqrt(p0_sqr - p0_star.dot(m2) ** 2)
  p_star_m3 = (-0.5 * p0_sqr - p0_star.dot(m2) * b.dot(m2)) / b.dot(m3)
  p_star_m2 = p0_star.dot(m2)
  p_star_m1 = math.sqrt(rho ** 2 - p_star_m3 ** 2)

  p0_star_m1 = p0_star.dot(m1)
  p0_star_m2 = p0_star.dot(m2)
  p0_star_m3 = p0_star.dot(m3)

  cp1 = + p_star_m1 * p0_star_m1 +  p_star_m3 * p0_star_m3
  cp2 = - p_star_m1 * p0_star_m1 +  p_star_m3 * p0_star_m3
  sp1 = + p_star_m1 * p0_star_m3 -  p_star_m3 * p0_star_m1
  sp2 = - p_star_m1 * p0_star_m3 -  p_star_m3 * p0_star_m1

  return math.atan2(sp1, cp1), math.atan2(sp2, cp2)

def profile_correlation(data, model):
  '''Compute CC between reflection profiles data and model.'''

  from dials.array_family import flex

  assert(data.focus() == model.focus())

  sel = data.as_1d() > 0

  model = model.as_1d().select(sel)
  data = data.as_1d().select(sel).as_double()

  assert(len(data) == len(model))

  correlation = flex.linear_correlation(data, model)
  return correlation.coefficient()

def abs_angle(a, b):
  import math
  return abs(math.atan2(math.sin(b - a), math.cos(b - a)))

def model_reflection_rt0(reflection, experiment, params):
  import math
  from scitbx import matrix
  from dials.array_family import flex

  d2r = math.pi / 180.0

  hkl = reflection['miller_index']
  xyz = reflection['xyzcal.px']
  xyz_mm = reflection['xyzcal.mm']

  if params.debug:
    print 'hkl = %d %d %d' % hkl
    print 'xyz px = %f %f %f' % xyz
    print 'xyz mm = %f %f %f' % xyz_mm

  Amat = matrix.sqr(experiment.crystal.get_A_at_scan_point(int(xyz[2])))
  p0_star = Amat * hkl

  angles = predict_angles(p0_star, experiment)

  if params.debug:
    print 'angles = %f %f' % angles

  angle = angles[0] if (abs(angles[0] - xyz_mm[2]) <
                        abs(angles[1] - xyz_mm[2])) else angles[1]

  i0 = reflection['intensity.sum.value']# / reflection['dqe']

  if params.min_isum:
    if i0 < params.min_isum:
      return

  s1 = reflection['s1']
  a = matrix.col(experiment.goniometer.get_rotation_axis())
  s0 = matrix.col(experiment.beam.get_s0())

  if params.debug:
    print 's1 = %f %f %f' % s1

  pixels = reflection['shoebox']
  pixels.flatten()
  data = pixels.data
  dz, dy, dx = data.focus()

  # since now 2D data
  data.reshape(flex.grid(dy, dx))

  if params.show:
    print 'Observed reflection (flattened in Z):'
    print
    for j in range(dy):
      for i in range(dx):
        print '%5d' % data[(j, i)],
      print

  sigma_m = experiment.profile.sigma_m() * d2r
  sigma_b = experiment.profile.sigma_b() * d2r

  r0 = xyz_mm[2]

  detector = experiment.detector

  patch = flex.double(dy * dx, 0)
  patch.reshape(flex.grid(dy, dx))

  bbox = reflection['bbox']

  scale = params.scale
  if params.show:
    print '%d rays' % (int(round(i0 * scale)))
  for i in range(int(round(i0 * scale))):
    b = random_vector_cone(s0, sigma_b)
    p0 = random_vector_cone(Amat * hkl, sigma_m)
    if params.rs_node_size > 0:
      ns = params.rs_node_size
      import random
      dp0 = matrix.col((random.gauss(0, ns),
                        random.gauss(0, ns),
                        random.gauss(0, ns)))
      p0 += dp0
    angles = predict_angles(p0, experiment, b)
    r = angles[0] if reflection['entering'] else angles[1]
    p = p0.rotate(a, r)
    s1 = p + b
    panel, xy = detector.get_ray_intersection(s1)

    # FIXME DO NOT USE THIS FUNCTION EVENTUALLY...
    x, y = detector[panel].millimeter_to_pixel(xy)
    if x < bbox[0] or x >= bbox[1]:
      continue
    if y < bbox[2] or y >= bbox[3]:
      continue
    x -= bbox[0]
    y -= bbox[2]
    # FIXME in here try to work out probability distribution along path
    # length through the detector sensitive surface i.e. deposit fractional
    # counts along pixels (and allow for DQE i.e. photon passing right through
    # the detector)
    patch[(int(y), int(x))] += 1.0 / scale

  if params.show:
    print 'Simulated reflection (flattened in Z):'
    print
    for j in range(dy):
      for i in range(dx):
        print '%5d' % int(patch[(j, i)]),
      print

  print 'Correlation coefficient: %.3f isum: %.1f ' % (
      profile_correlation(data, patch), i0)

  return

def model_reflection_flat(reflection, experiment, params):
  pixels = reflection['shoebox']
  pixels.flatten()
  data = pixels.data
  dz, dy, dx = data.focus()
  return

def main(reflections, experiment, params):
  nref0 = len(reflections)

  method = params.method
  ids = params.id

  if 'intensity.prf.variance' in reflections:
    selection = reflections.get_flags(
      reflections.flags.integrated,
      all=True)
  else:
    selection = reflections.get_flags(
      reflections.flags.integrated_sum)
  reflections = reflections.select(selection)

  nref1 = len(reflections)

  print 'Removed %d invalid reflections, %d remain' % (nref0 - nref1, nref1)

  for j, reflection in enumerate(reflections):
    if ids:
      if j in ids:
        globals()['model_reflection_%s' % method](reflection, experiment, params)
    elif params.id_start and params.id_end:
      if j < params.id_start or j >= params.id_end:
        continue
      print j
      globals()['model_reflection_%s' % method](reflection, experiment, params)

    else:
      print j
      globals()['model_reflection_%s' % method](reflection, experiment, params)

  return

def run(args):
  from dials.util.options import OptionParser
  from dials.util.options import flatten_experiments
  from dials.util.options import flatten_datablocks
  from dials.util.options import flatten_reflections
  import libtbx.load_env

  usage = "%s [options] integrated.pickle experiments.json" % (
    libtbx.env.dispatcher_name)

  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_experiments=True,
    read_reflections=True,
    check_format=False,
    epilog=help_message)

  params, options = parser.parse_args(show_diff_phil=False)
  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  if len(experiments) != 1 or len(reflections) != 1:
    parser.print_help()
    exit()

  if not 'shoebox' in reflections[0]:
    print 'Please add shoeboxes to reflection pickle'
    exit()

  main(reflections[0], experiments[0], params)

if __name__ == '__main__':
  import sys
  run(sys.argv[1:])
<|endoftext|>


<commit_before># Test just the SSL support in the socket module, in a moderately bogus way.

import sys
import unittest
from test import test_support
import socket
import errno
import threading
import subprocess
import time
import os
import urllib

# Optionally test SSL support, if we have it in the tested platform
skip_expected = not hasattr(socket, "ssl")

class ConnectedTests(unittest.TestCase):

    def urlopen(self, host, *args, **kwargs):
        # Connecting to remote hosts is flaky.  Make it more robust
        # by retrying the connection several times.
        for i in range(3):
            try:
                return urllib.urlopen(host, *args, **kwargs)
            except IOError, e:
                last_exc = e
                continue
            except:
                raise
        raise last_exc

    def testBasic(self):
        socket.RAND_status()
        try:
            socket.RAND_egd(1)
        except TypeError:
            pass
        else:
            print "didn't raise TypeError"
        socket.RAND_add("this is a random string", 75.0)

        with test_support.transient_internet():
            f = self.urlopen('https://sf.net')
        buf = f.read()
        f.close()

    def testTimeout(self):
        def error_msg(extra_msg):
            print >> sys.stderr, """\
        WARNING:  an attempt to connect to %r %s, in
        test_timeout.  That may be legitimate, but is not the outcome we
        hoped for.  If this message is seen often, testTimeout should be
        changed to use a more reliable address.""" % (ADDR, extra_msg)

        # A service which issues a welcome banner (without need to write
        # anything).
        ADDR = "pop.gmail.com", 995

        s = socket.socket()
        s.settimeout(30.0)
        try:
            s.connect(ADDR)
        except socket.timeout:
            error_msg('timed out')
            return
        except socket.error, exc:  # In case connection is refused.
            if exc.args[0] == errno.ECONNREFUSED:
                error_msg('was refused')
                return
            else:
                raise

        ss = socket.ssl(s)
        # Read part of return welcome banner twice.
        ss.read(1)
        ss.read(1)
        s.close()

class BasicTests(unittest.TestCase):

    def testRudeShutdown(self):
        # Some random port to connect to.
        PORT = [9934]

        listener_ready = threading.Event()
        listener_gone = threading.Event()

        # `listener` runs in a thread.  It opens a socket listening on
        # PORT, and sits in an accept() until the main thread connects.
        # Then it rudely closes the socket, and sets Event `listener_gone`
        # to let the main thread know the socket is gone.
        def listener():
            s = socket.socket()
            PORT[0] = test_support.bind_port(s, '', PORT[0])
            s.listen(5)
            listener_ready.set()
            s.accept()
            s = None # reclaim the socket object, which also closes it
            listener_gone.set()

        def connector():
            listener_ready.wait()
            s = socket.socket()
            s.connect(('localhost', PORT[0]))
            listener_gone.wait()
            try:
                ssl_sock = socket.ssl(s)
            except socket.sslerror:
                pass
            else:
                raise test_support.TestFailed(
                      'connecting to closed SSL socket should have failed')

        t = threading.Thread(target=listener)
        t.start()
        connector()
        t.join()

    def connect(self, s, host_port):
        # Connecting to remote hosts is flaky.  Make it more robust
        # by retrying the connection several times.
        for i in range(3):
            try:
                return s.connect(host_port)
            except IOError, e:
                last_exc = e
                continue
            except:
                raise
        raise last_exc

    def test_978833(self):
        if test_support.verbose:
            print "test_978833 ..."

        import os, httplib, ssl
        with test_support.transient_internet():
            s = socket.socket(socket.AF_INET)
            self.connect(s, ("svn.python.org", 443))
            fd = s._sock.fileno()
            sock = ssl.wrap_socket(s)
            s = None
            sock.close()
            try:
                os.fstat(fd)
            except OSError:
                pass
            else:
                raise test_support.TestFailed("Failed to close socket")

class OpenSSLTests(unittest.TestCase):

    def testBasic(self):
        s = socket.socket()
        s.connect(("localhost", 4433))
        ss = socket.ssl(s)
        ss.write("Foo\n")
        i = ss.read(4)
        self.assertEqual(i, "Foo\n")
        s.close()

    def testMethods(self):
        # read & write is already tried in the Basic test
        # now we'll try to get the server info about certificates
        # this came from the certificate I used, one I found in /usr/share/openssl
        info = "/C=PT/ST=Queensland/L=Lisboa/O=Neuronio, Lda./OU=Desenvolvimento/CN=brutus.neuronio.pt/emailAddress=sampo@iki.fi"

        s = socket.socket()
        s.connect(("localhost", 4433))
        ss = socket.ssl(s)
        cert = ss.server()
        self.assertEqual(cert, info)
        cert = ss.issuer()
        self.assertEqual(cert, info)
        s.close()

class OpenSSLServer(threading.Thread):
    def __init__(self):
        self.s = None
        self.keepServing = True
        self._external()
        if self.haveServer:
            threading.Thread.__init__(self)

    def _external(self):
        # let's find the .pem files
        curdir = os.path.dirname(__file__) or os.curdir
        cert_file = os.path.join(curdir, "ssl_cert.pem")
        if not os.access(cert_file, os.F_OK):
            raise ValueError("No cert file found! (tried %r)" % cert_file)
        key_file = os.path.join(curdir, "ssl_key.pem")
        if not os.access(key_file, os.F_OK):
            raise ValueError("No key file found! (tried %r)" % key_file)

        try:
            cmd = "openssl s_server -cert %s -key %s -quiet" % (cert_file, key_file)
            self.s = subprocess.Popen(cmd.split(), stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT)
            time.sleep(1)
        except:
            self.haveServer = False
        else:
            # let's try if it is actually up
            try:
                s = socket.socket()
                s.connect(("localhost", 4433))
                s.close()
                if self.s.stdout.readline() != "ERROR\n":
                    raise ValueError
            except:
                self.haveServer = False
            else:
                self.haveServer = True

    def run(self):
        while self.keepServing:
            time.sleep(.5)
            l = self.s.stdout.readline()
            self.s.stdin.write(l)

    def shutdown(self):
        self.keepServing = False
        if not self.s:
            return
        if sys.platform == "win32":
            subprocess.TerminateProcess(int(self.s._handle), -1)
        else:
            os.kill(self.s.pid, 15)

def test_main():
    if not hasattr(socket, "ssl"):
        raise test_support.TestSkipped("socket module has no ssl support")

    tests = [BasicTests]

    if test_support.is_resource_enabled('network'):
        tests.append(ConnectedTests)

    # in these platforms we can kill the openssl process
    if sys.platform in ("sunos5", "darwin", "linux1",
                        "linux2", "win32", "hp-ux11"):

        server = OpenSSLServer()
        if server.haveServer:
            tests.append(OpenSSLTests)
            server.start()
    else:
        server = None

    thread_info = test_support.threading_setup()

    try:
        test_support.run_unittest(*tests)
    finally:
        if server is not None and server.haveServer:
            server.shutdown()

    test_support.threading_cleanup(*thread_info)

if __name__ == "__main__":
    test_main()
<commit_msg>The test requires the network resource<commit_after># Test just the SSL support in the socket module, in a moderately bogus way.

import sys
import unittest
from test import test_support
import socket
import errno
import threading
import subprocess
import time
import os
import urllib

# Optionally test SSL support, if we have it in the tested platform
skip_expected = not hasattr(socket, "ssl")

class ConnectedTests(unittest.TestCase):

    def urlopen(self, host, *args, **kwargs):
        # Connecting to remote hosts is flaky.  Make it more robust
        # by retrying the connection several times.
        for i in range(3):
            try:
                return urllib.urlopen(host, *args, **kwargs)
            except IOError, e:
                last_exc = e
                continue
            except:
                raise
        raise last_exc

    def testBasic(self):
        socket.RAND_status()
        try:
            socket.RAND_egd(1)
        except TypeError:
            pass
        else:
            print "didn't raise TypeError"
        socket.RAND_add("this is a random string", 75.0)

        with test_support.transient_internet():
            f = self.urlopen('https://sf.net')
        buf = f.read()
        f.close()

    def testTimeout(self):
        def error_msg(extra_msg):
            print >> sys.stderr, """\
        WARNING:  an attempt to connect to %r %s, in
        test_timeout.  That may be legitimate, but is not the outcome we
        hoped for.  If this message is seen often, testTimeout should be
        changed to use a more reliable address.""" % (ADDR, extra_msg)

        # A service which issues a welcome banner (without need to write
        # anything).
        ADDR = "pop.gmail.com", 995

        s = socket.socket()
        s.settimeout(30.0)
        try:
            s.connect(ADDR)
        except socket.timeout:
            error_msg('timed out')
            return
        except socket.error, exc:  # In case connection is refused.
            if exc.args[0] == errno.ECONNREFUSED:
                error_msg('was refused')
                return
            else:
                raise

        ss = socket.ssl(s)
        # Read part of return welcome banner twice.
        ss.read(1)
        ss.read(1)
        s.close()

class BasicTests(unittest.TestCase):

    def testRudeShutdown(self):
        # Some random port to connect to.
        PORT = [9934]

        listener_ready = threading.Event()
        listener_gone = threading.Event()

        # `listener` runs in a thread.  It opens a socket listening on
        # PORT, and sits in an accept() until the main thread connects.
        # Then it rudely closes the socket, and sets Event `listener_gone`
        # to let the main thread know the socket is gone.
        def listener():
            s = socket.socket()
            PORT[0] = test_support.bind_port(s, '', PORT[0])
            s.listen(5)
            listener_ready.set()
            s.accept()
            s = None # reclaim the socket object, which also closes it
            listener_gone.set()

        def connector():
            listener_ready.wait()
            s = socket.socket()
            s.connect(('localhost', PORT[0]))
            listener_gone.wait()
            try:
                ssl_sock = socket.ssl(s)
            except socket.sslerror:
                pass
            else:
                raise test_support.TestFailed(
                      'connecting to closed SSL socket should have failed')

        t = threading.Thread(target=listener)
        t.start()
        connector()
        t.join()

    def connect(self, s, host_port):
        # Connecting to remote hosts is flaky.  Make it more robust
        # by retrying the connection several times.
        for i in range(3):
            try:
                return s.connect(host_port)
            except IOError, e:
                last_exc = e
                continue
            except:
                raise
        raise last_exc

    def test_978833(self):
        if not test_support.is_resource_enabled("network"):
            return
        if test_support.verbose:
            print "test_978833 ..."

        import os, httplib, ssl
        with test_support.transient_internet():
            s = socket.socket(socket.AF_INET)
            self.connect(s, ("svn.python.org", 443))
            fd = s._sock.fileno()
            sock = ssl.wrap_socket(s)
            s = None
            sock.close()
            try:
                os.fstat(fd)
            except OSError:
                pass
            else:
                raise test_support.TestFailed("Failed to close socket")

class OpenSSLTests(unittest.TestCase):

    def testBasic(self):
        s = socket.socket()
        s.connect(("localhost", 4433))
        ss = socket.ssl(s)
        ss.write("Foo\n")
        i = ss.read(4)
        self.assertEqual(i, "Foo\n")
        s.close()

    def testMethods(self):
        # read & write is already tried in the Basic test
        # now we'll try to get the server info about certificates
        # this came from the certificate I used, one I found in /usr/share/openssl
        info = "/C=PT/ST=Queensland/L=Lisboa/O=Neuronio, Lda./OU=Desenvolvimento/CN=brutus.neuronio.pt/emailAddress=sampo@iki.fi"

        s = socket.socket()
        s.connect(("localhost", 4433))
        ss = socket.ssl(s)
        cert = ss.server()
        self.assertEqual(cert, info)
        cert = ss.issuer()
        self.assertEqual(cert, info)
        s.close()

class OpenSSLServer(threading.Thread):
    def __init__(self):
        self.s = None
        self.keepServing = True
        self._external()
        if self.haveServer:
            threading.Thread.__init__(self)

    def _external(self):
        # let's find the .pem files
        curdir = os.path.dirname(__file__) or os.curdir
        cert_file = os.path.join(curdir, "ssl_cert.pem")
        if not os.access(cert_file, os.F_OK):
            raise ValueError("No cert file found! (tried %r)" % cert_file)
        key_file = os.path.join(curdir, "ssl_key.pem")
        if not os.access(key_file, os.F_OK):
            raise ValueError("No key file found! (tried %r)" % key_file)

        try:
            cmd = "openssl s_server -cert %s -key %s -quiet" % (cert_file, key_file)
            self.s = subprocess.Popen(cmd.split(), stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT)
            time.sleep(1)
        except:
            self.haveServer = False
        else:
            # let's try if it is actually up
            try:
                s = socket.socket()
                s.connect(("localhost", 4433))
                s.close()
                if self.s.stdout.readline() != "ERROR\n":
                    raise ValueError
            except:
                self.haveServer = False
            else:
                self.haveServer = True

    def run(self):
        while self.keepServing:
            time.sleep(.5)
            l = self.s.stdout.readline()
            self.s.stdin.write(l)

    def shutdown(self):
        self.keepServing = False
        if not self.s:
            return
        if sys.platform == "win32":
            subprocess.TerminateProcess(int(self.s._handle), -1)
        else:
            os.kill(self.s.pid, 15)

def test_main():
    if not hasattr(socket, "ssl"):
        raise test_support.TestSkipped("socket module has no ssl support")

    tests = [BasicTests]

    if test_support.is_resource_enabled('network'):
        tests.append(ConnectedTests)

    # in these platforms we can kill the openssl process
    if sys.platform in ("sunos5", "darwin", "linux1",
                        "linux2", "win32", "hp-ux11"):

        server = OpenSSLServer()
        if server.haveServer:
            tests.append(OpenSSLTests)
            server.start()
    else:
        server = None

    thread_info = test_support.threading_setup()

    try:
        test_support.run_unittest(*tests)
    finally:
        if server is not None and server.haveServer:
            server.shutdown()

    test_support.threading_cleanup(*thread_info)

if __name__ == "__main__":
    test_main()
<|endoftext|>


<commit_before>{
  "targets": [
    {
      'target_name': 'LDAP',
      'sources': [
        'src/LDAP.cc'
      ],
      'include_dirs': [
        '/usr/local/include'
      ],
      'defines': [
        'LDAP_DEPRECATED'
      ],
      'cflags': [
        '-Wall',
        '--verbose'
      ],
      'ldflags': [
        '-L/usr/local/lib',
        '-lldap',
        '--verbose'
      ],
    }
  ]
}

<commit_msg>Removed verbose.<commit_after>{
  "targets": [
    {
      'target_name': 'LDAP',
      'sources': [
        'src/LDAP.cc'
      ],
      'include_dirs': [
        '/usr/local/include'
      ],
      'defines': [
        'LDAP_DEPRECATED'
      ],
      'cflags': [
        '-Wall',
        '--verbose',
        '-g'
      ],
      'ldflags': [
        '-L/usr/local/lib',
        '-lldap'
      ],
    }
  ]
}

<|endoftext|>


<commit_before>#!/usr/bin/env dials.python
from __future__ import division
from libtbx.phil import parse

"""Utility script to split experiments and reflections from single files into
multiple files with one experiment per output experiment file and one
reflection file per output experiment file.
"""

class Script(object):

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    from libtbx.phil import parse
    import libtbx.load_env

    # The phil scope
    phil_scope = parse('''

      output {
        experiments_prefix = experiments
          .type = str
          .help = "Filename prefix for the split experimental models"

        reflections_prefix = reflections
          .type = str
          .help = "Filename prefix for the split reflections"
      }
    ''', process_includes=True)

    # The script usage
    usage  = "usage: %s [options] [param.phil] " \
             "experiments1.json experiments2.json reflections1.pickle " \
             "reflections2.pickle..." \
             % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      read_reflections=True,
      read_experiments=True,
      check_format=False)

  def run(self):
    '''Execute the script.'''

    from dials.util.options import flatten_reflections, flatten_experiments
    from libtbx.utils import Sorry
    from dials.array_family import flex

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)

    # Try to load the models and data
    if len(params.input.experiments) == 0:
      print "No Experiments found in the input"
      self.parser.print_help()
      return
    if len(params.input.reflections) == 0:
      print "No reflection data found in the input"
      self.parser.print_help()
      return
    try:
      assert len(params.input.reflections) == len(params.input.experiments)
    except AssertionError:
      raise Sorry("The number of input reflections files does not match the "
        "number of input experiments")

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)[0]

    import math
    experiments_template = "%s_%%0%sd.json" %(
      params.output.experiments_prefix,
      int(math.floor(math.log10(len(experiments))) + 1))
    reflections_template = "%s_%%0%sd.pickle" %(
      params.output.reflections_prefix,
      int(math.floor(math.log10(len(experiments))) + 1))

    for i, experiment in enumerate(experiments):
      ref_sel = reflections.select(reflections['id'] == i)
      ref_sel['id'] = flex.int(len(ref_sel), 0)

      from dxtbx.model.experiment.experiment_list import ExperimentList
      from dxtbx.serialize import dump
      experiment_filename = experiments_template %i
      reflections_filename = reflections_template %i
      print 'Saving experiment %d to %s' %(i, experiment_filename)
      dump.experiment_list(ExperimentList([experiment]), experiment_filename)
      print 'Saving reflections for experiment %d to %s' %(i, reflections_filename)
      ref_sel.as_pickle(reflections_filename)

    return

if __name__ == "__main__":
  from dials.util import halraiser
  try:
    script = Script()
    script.run()
  except Exception as e:
    halraiser(e)

<commit_msg>Add help message<commit_after>#!/usr/bin/env dials.python
from __future__ import division
from libtbx.phil import parse

help_message = """

Utility script to split experiments and reflections from single files into
multiple files with one experiment per output experiment file and one
reflection file per output experiment file.

Example::

  dials.split_experiments combined_experiments.json combined_reflections.pickle

"""

class Script(object):

  def __init__(self):
    '''Initialise the script.'''
    from dials.util.options import OptionParser
    from libtbx.phil import parse
    import libtbx.load_env

    # The phil scope
    phil_scope = parse('''

      output {
        experiments_prefix = experiments
          .type = str
          .help = "Filename prefix for the split experimental models"

        reflections_prefix = reflections
          .type = str
          .help = "Filename prefix for the split reflections"
      }
    ''', process_includes=True)

    # The script usage
    usage  = "usage: %s [options] [param.phil] " \
             "experiments1.json experiments2.json reflections1.pickle " \
             "reflections2.pickle..." \
             % libtbx.env.dispatcher_name

    # Create the parser
    self.parser = OptionParser(
      usage=usage,
      phil=phil_scope,
      read_reflections=True,
      read_experiments=True,
      check_format=False,
      epilog=help_message)

  def run(self):
    '''Execute the script.'''

    from dials.util.options import flatten_reflections, flatten_experiments
    from libtbx.utils import Sorry
    from dials.array_family import flex

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=True)

    # Try to load the models and data
    if len(params.input.experiments) == 0:
      print "No Experiments found in the input"
      self.parser.print_help()
      return
    if len(params.input.reflections) == 0:
      print "No reflection data found in the input"
      self.parser.print_help()
      return
    try:
      assert len(params.input.reflections) == len(params.input.experiments)
    except AssertionError:
      raise Sorry("The number of input reflections files does not match the "
        "number of input experiments")

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)[0]

    import math
    experiments_template = "%s_%%0%sd.json" %(
      params.output.experiments_prefix,
      int(math.floor(math.log10(len(experiments))) + 1))
    reflections_template = "%s_%%0%sd.pickle" %(
      params.output.reflections_prefix,
      int(math.floor(math.log10(len(experiments))) + 1))

    for i, experiment in enumerate(experiments):
      ref_sel = reflections.select(reflections['id'] == i)
      ref_sel['id'] = flex.int(len(ref_sel), 0)

      from dxtbx.model.experiment.experiment_list import ExperimentList
      from dxtbx.serialize import dump
      experiment_filename = experiments_template %i
      reflections_filename = reflections_template %i
      print 'Saving experiment %d to %s' %(i, experiment_filename)
      dump.experiment_list(ExperimentList([experiment]), experiment_filename)
      print 'Saving reflections for experiment %d to %s' %(i, reflections_filename)
      ref_sel.as_pickle(reflections_filename)

    return

if __name__ == "__main__":
  from dials.util import halraiser
  try:
    script = Script()
    script.run()
  except Exception as e:
    halraiser(e)

<|endoftext|>


<commit_before><commit_msg>oops<commit_after><|endoftext|>


<commit_before># !/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Stephanie'

from yodatools.converter.Inputs.ExcelInput import ExcelInput
from yodatools.converter.Inputs.yamlInput import yamlInput
from yodatools.converter.Outputs.yamlOutput import yamlOutput
from yodatools.converter.Outputs.dbOutput import dbOutput
from argparse import ArgumentParser
import logging

import time

def run_converter(input_file, yoda_file=None, db_conn = None):
    yoda_output_file_path = yoda_file
    db_output_connection = db_conn

    # Check if it is a yaml, or excel file
    file_type = verify_file_type(input_file)

    if file_type == 'invalid':  # Accept only excel and yaml files
        print "File extension isvalid or no file"
        return
    try:
        if file_type == 'excel':
            excel = ExcelInput(input_file)#, **kwargs)
            excel.parse()
            session = excel.sendODM2Session()
        else:
            # Must be a yoda file

            yoda = yamlInput(input_file)
            yoda.parse(input_file)
            session = yoda.sendODM2Session()

        # Go through each checkbox
        if yoda_output_file_path is not None:
            yaml = yamlOutput()
            yaml.save(session=session, file_path=yoda_output_file_path)

        if db_output_connection is not None:
            db = dbOutput()
            db.save(session=session, file_path=db_output_connection)

        # if 'odm2' in selections:
        #     print 'export to odm2'
        #     """
        #     create connection string
        #     call dboutput and do same as yoda export and send in connection string as filepath
        #     """

        session.close_all()
    except Exception as e:
        print "error parsing values, " + e
        raise e

def verify_file_type(input_file):
    CONST_LEGAL_EXCEL_EXTENSIONS = ('xlsx', 'xlsm')

    if input_file.endswith(CONST_LEGAL_EXCEL_EXTENSIONS):
        file_type = 'excel'
    elif input_file.endswith('yml') or input_file.endswith('yaml') or input_file.endswith('yoda'):
        file_type = 'yaml'
    else:
        file_type = 'invalid'

    return file_type

#command line stuff
# Application entry point.
if __name__ == "__main__":
    parser = ArgumentParser(description="YodaTools")

    parser.add_argument('-i', '--input', dest="inputFile",\
        help="Specify a file in of one of these formats:\n1. A single yodaparser (.yaml) file.\n"
             "2. An Excel YODA template.\n", required=True, action="store")
    # parser.add_argument('-e', '--excel', dest="excelFile",\
    #     help="Specify a single excel path to save the converted data too.", required=False, action="store")
    parser.add_argument('-y', '--yoda', dest="yodaFile",\
        help="Specify a single yoda file path to save the converted data too.", required=False, action="store")
    parser.add_argument('-d', '--database', dest="dbConn",\
                         help="Specify a Database connection. " \
                              "Format: {engine}+{driver}://{user}:{pass}@{address}/{db}\n"" \
                              ""mysql+pymysql://ODM:odm@localhost/odm2\n"" \
                              ""mssql+pyodbc://ODM:123@localhost/odm2\n"" \
                              ""postgresql+psycopg2://ODM:odm@test.uwrl.usu.edu/odm2\n"" \
                              ""sqlite+pysqlite:///path/to/file",
                        required=False, action="store")
    args = parser.parse_args()

    start_time = time.time()
    run_converter(args.inputFile, args.yodaFile, args.dbConn)

    # logger = logging.getLogger('YODA_logger')
    # logger.info("Running time: %s seconds" % (time.time() - start_time))

<commit_msg>update run converter function to include excel file<commit_after># !/usr/bin/env python
# -*- coding: utf-8 -*-

__author__ = 'Stephanie'

from yodatools.converter.Inputs.ExcelInput import ExcelInput
from yodatools.converter.Inputs.yamlInput import yamlInput
from yodatools.converter.Outputs.yamlOutput import yamlOutput
from yodatools.converter.Outputs.dbOutput import dbOutput
from argparse import ArgumentParser
import logging

import time

def run_converter(input_file, yoda_file=None, db_conn = None, excel_file=None):
    yoda_output_file_path = yoda_file
    db_output_connection = db_conn
    excel_output_file_path = excel_file

    # Check if it is a yaml, or excel file
    file_type = verify_file_type(input_file)

    if file_type == 'invalid':  # Accept only excel and yaml files
        print "File extension isvalid or no file"
        return
    try:
        if file_type == 'excel':
            excel = ExcelInput(input_file)#, **kwargs)
            excel.parse()
            session = excel.sendODM2Session()
        else:
            # Must be a yoda file

            yoda = yamlInput(input_file)
            yoda.parse(input_file)
            session = yoda.sendODM2Session()

        # Go through each type
        if yoda_output_file_path is not None:
            yaml = yamlOutput()
            yaml.save(session=session, file_path=yoda_output_file_path)

        if db_output_connection is not None:
            db = dbOutput()
            db.save(session=session, file_path=db_output_connection)

        if excel_output_file_path is not None:
            print "excel output is not yet supported."

        # if 'odm2' in selections:
        #     print 'export to odm2'
        #     """
        #     create connection string
        #     call dboutput and do same as yoda export and send in connection string as filepath
        #     """

        session.close_all()
    except Exception as e:
        print "error parsing values, " + e
        raise e

def verify_file_type(input_file):
    CONST_LEGAL_EXCEL_EXTENSIONS = ('xlsx', 'xlsm')

    if input_file.endswith(CONST_LEGAL_EXCEL_EXTENSIONS):
        file_type = 'excel'
    elif input_file.endswith('yml') or input_file.endswith('yaml') or input_file.endswith('yoda'):
        file_type = 'yaml'
    else:
        file_type = 'invalid'

    return file_type

#command line stuff
# Application entry point.
if __name__ == "__main__":
    parser = ArgumentParser(description="YodaTools")

    parser.add_argument('-i', '--input', dest="inputFile",\
        help="Specify a file in of one of these formats:\n1. A single yodaparser (.yaml) file.\n"
             "2. An Excel YODA template.\n", required=True, action="store")
    # parser.add_argument('-e', '--excel', dest="excelFile",\
    #     help="Specify a single excel path to save the converted data too.", required=False, action="store")
    parser.add_argument('-y', '--yoda', dest="yodaFile",\
        help="Specify a single yoda file path to save the converted data too.", required=False, action="store")
    parser.add_argument('-d', '--database', dest="dbConn",\
                         help="Specify a Database connection. " \
                              "Format: {engine}+{driver}://{user}:{pass}@{address}/{db}\n"" \
                              ""mysql+pymysql://ODM:odm@localhost/odm2\n"" \
                              ""mssql+pyodbc://ODM:123@localhost/odm2\n"" \
                              ""postgresql+psycopg2://ODM:odm@test.uwrl.usu.edu/odm2\n"" \
                              ""sqlite+pysqlite:///path/to/file",
                        required=False, action="store")
    args = parser.parse_args()

    start_time = time.time()
    run_converter(args.inputFile, args.yodaFile, args.dbConn)

    # logger = logging.getLogger('YODA_logger')
    # logger.info("Running time: %s seconds" % (time.time() - start_time))

<|endoftext|>


<commit_before>{
  'targets': [
    {
      'target_name': 'addon',
      'sources': [ 'src/addon.cc' ],
      'cflags': [ '-stdlib=libc++', '-std=c++11' ],
      'ldflags': [ '-std=c++11' ],
      'xcode_settings': {
        'OTHER_CPLUSPLUSFLAGS': [ '-mmacosx-version-min=10.7', '-stdlib=libc++', '-std=c++11' ],
        'OTHER_LDFLAGS': [ '-std=c++11' ],
      },
    }
  ]
}
<commit_msg>move -stdlib directive to mac conditional<commit_after>{
  'targets': [
    {
      'target_name': 'addon',
      'sources': [ 'src/addon.cc' ],
      'cflags': [ '-std=c++11' ],
      'ldflags': [ '-std=c++11' ],
      'conditions': [
        ['OS=="mac"',
          {
            'cflags': [ '-stdlib=libc++' ],
            'xcode_settings': {
              'OTHER_CPLUSPLUSFLAGS': [ '-mmacosx-version-min=10.7', '-stdlib=libc++', '-std=c++11' ],
              'OTHER_LDFLAGS': [ '-std=c++11' ],
            }
          }
        ],
      ]
    }
  ]
}
<|endoftext|>