<commit_before>import pytest

from diofant import (I, O, Rational, Symbol, atanh, conjugate, elliptic_e,
                     elliptic_f, elliptic_k, elliptic_pi, gamma, hyper,
                     meijerg, oo, pi, sin, sqrt, tan, zoo)
from diofant.abc import m, n, z
from diofant.core.function import ArgumentIndexError
from diofant.utilities.randtest import random_complex_number as randcplx
from diofant.utilities.randtest import verify_derivative_numerically as td
from diofant.utilities.randtest import verify_numerically as tn

__all__ = ()

i = Symbol('i', integer=True)
j = Symbol('k', integer=True, positive=True)

def test_elliptic_k():
    assert elliptic_k(0) == pi/2
    assert elliptic_k(Rational(1, 2)) == 8*pi**Rational(3, 2)/gamma(-Rational(1, 4))**2
    assert elliptic_k(1) == zoo
    assert elliptic_k(-1) == gamma(Rational(1, 4))**2/(4*sqrt(2*pi))
    assert elliptic_k(oo) == 0
    assert elliptic_k(-oo) == 0
    assert elliptic_k(I*oo) == 0
    assert elliptic_k(-I*oo) == 0
    assert elliptic_k(zoo) == 0

    assert elliptic_k(z).diff(z) == (elliptic_e(z) - (1 - z)*elliptic_k(z))/(2*z*(1 - z))
    assert td(elliptic_k(z), z)
    pytest.raises(ArgumentIndexError, lambda: elliptic_k(z).fdiff(2))

    zi = Symbol('z', extended_real=False)
    assert elliptic_k(zi).conjugate() == elliptic_k(zi.conjugate())
    zr = Symbol('z', extended_real=True, negative=True)
    assert elliptic_k(zr).conjugate() == elliptic_k(zr)
    assert elliptic_k(z).conjugate() == conjugate(elliptic_k(z), evaluate=False)

    assert elliptic_k(z).rewrite(hyper) == \
        (pi/2)*hyper((Rational(1, 2), Rational(1, 2)), (1,), z)
    assert tn(elliptic_k(z), (pi/2)*hyper((Rational(1, 2), Rational(1, 2)), (1,), z))
    assert elliptic_k(z).rewrite(meijerg) == \
        meijerg(((Rational(1, 2), Rational(1, 2)), []), ((0,), (0,)), -z)/2
    assert tn(elliptic_k(z), meijerg(((Rational(1, 2), Rational(1, 2)), []), ((0,), (0,)), -z)/2)

    assert elliptic_k(z).series(z) == pi/2 + pi*z/8 + 9*pi*z**2/128 + \
        25*pi*z**3/512 + 1225*pi*z**4/32768 + 3969*pi*z**5/131072 + O(z**6)

def test_elliptic_f():
    assert elliptic_f(z, 0) == z
    assert elliptic_f(0, m) == 0
    assert elliptic_f(pi*i/2, m) == i*elliptic_k(m)
    assert elliptic_f(z, oo) == 0
    assert elliptic_f(z, -oo) == 0

    assert elliptic_f(-z, m) == -elliptic_f(z, m)

    assert elliptic_f(z, m).diff(z) == 1/sqrt(1 - m*sin(z)**2)
    assert elliptic_f(z, m).diff(m) == elliptic_e(z, m)/(2*m*(1 - m)) - elliptic_f(z, m)/(2*m) - \
        sin(2*z)/(4*(1 - m)*sqrt(1 - m*sin(z)**2))
    r = randcplx()
    assert td(elliptic_f(z, r), z)
    assert td(elliptic_f(r, m), m)
    pytest.raises(ArgumentIndexError, lambda: elliptic_f(z, m).fdiff(3))

    mi = Symbol('m', extended_real=False)
    assert elliptic_f(z, mi).conjugate() == elliptic_f(z.conjugate(), mi.conjugate())
    mr = Symbol('m', extended_real=True, negative=True)
    assert elliptic_f(z, mr).conjugate() == elliptic_f(z.conjugate(), mr)
    assert elliptic_f(z, m).conjugate() == conjugate(elliptic_f(z, m), evaluate=False)

    assert elliptic_f(z, m).series(z) == \
        z + z**5*(3*m**2/40 - m/30) + m*z**3/6 + O(z**6)

def test_elliptic_e():
    assert elliptic_e(z, 0) == z
    assert elliptic_e(0, m) == 0
    assert elliptic_e(i*pi/2, m) == i*elliptic_e(m)
    assert elliptic_e(z, oo) == zoo
    assert elliptic_e(z, -oo) == zoo
    assert elliptic_e(0) == pi/2
    assert elliptic_e(1) == 1
    assert elliptic_e(oo) == I*oo
    assert elliptic_e(-oo) == oo
    assert elliptic_e(zoo) == zoo

    assert elliptic_e(-z, m) == -elliptic_e(z, m)

    assert elliptic_e(z, m).diff(z) == sqrt(1 - m*sin(z)**2)
    assert elliptic_e(z, m).diff(m) == (elliptic_e(z, m) - elliptic_f(z, m))/(2*m)
    assert elliptic_e(z).diff(z) == (elliptic_e(z) - elliptic_k(z))/(2*z)
    r = randcplx()
    assert td(elliptic_e(r, m), m)
    assert td(elliptic_e(z, r), z)
    assert td(elliptic_e(z), z)
    pytest.raises(ArgumentIndexError, lambda: elliptic_e(z, m).fdiff(3))
    pytest.raises(ArgumentIndexError, lambda: elliptic_e(z).fdiff(2))

    mi = Symbol('m', extended_real=False)
    assert elliptic_e(z, mi).conjugate() == elliptic_e(z.conjugate(), mi.conjugate())
    assert elliptic_e(mi).conjugate() == elliptic_e(mi.conjugate())
    mr = Symbol('m', extended_real=True, negative=True)
    assert elliptic_e(z, mr).conjugate() == elliptic_e(z.conjugate(), mr)
    assert elliptic_e(mr).conjugate() == elliptic_e(mr)
    assert elliptic_e(z, m).conjugate() == conjugate(elliptic_e(z, m))
    assert elliptic_e(z).conjugate() == conjugate(elliptic_e(z))

    assert elliptic_e(z).rewrite(hyper) == (pi/2)*hyper((Rational(-1, 2), Rational(1, 2)), (1,), z)
    assert elliptic_e(z, m).rewrite(hyper) == elliptic_e(z, m)
    assert tn(elliptic_e(z), (pi/2)*hyper((Rational(-1, 2), Rational(1, 2)), (1,), z))
    assert elliptic_e(z).rewrite(meijerg) == \
        -meijerg(((Rational(1, 2), Rational(3, 2)), []), ((0,), (0,)), -z)/4
    assert elliptic_e(z, m).rewrite(meijerg) == elliptic_e(z, m)
    assert tn(elliptic_e(z), -meijerg(((Rational(1, 2), Rational(3, 2)), []), ((0,), (0,)), -z)/4)

    assert elliptic_e(z, m).series(z) == \
        z + z**5*(-m**2/40 + m/30) - m*z**3/6 + O(z**6)
    assert elliptic_e(z).series(z) == pi/2 - pi*z/8 - 3*pi*z**2/128 - \
        5*pi*z**3/512 - 175*pi*z**4/32768 - 441*pi*z**5/131072 + O(z**6)

def test_elliptic_pi():
    assert elliptic_pi(0, z, m) == elliptic_f(z, m)
    assert elliptic_pi(1, z, m) == elliptic_f(z, m) + \
        (sqrt(1 - m*sin(z)**2)*tan(z) - elliptic_e(z, m))/(1 - m)
    assert elliptic_pi(n, i*pi/2, m) == i*elliptic_pi(n, m)
    assert elliptic_pi(n, z, 0) == atanh(sqrt(n - 1)*tan(z))/sqrt(n - 1)
    assert elliptic_pi(n, z, n) == elliptic_f(z, n) - elliptic_pi(1, z, n) + tan(z)/sqrt(1 - n*sin(z)**2)
    assert elliptic_pi(oo, z, m) == 0
    assert elliptic_pi(-oo, z, m) == 0
    assert elliptic_pi(n, z, oo) == 0
    assert elliptic_pi(n, z, -oo) == 0
    assert elliptic_pi(0, m) == elliptic_k(m)
    assert elliptic_pi(1, m) == zoo
    assert elliptic_pi(n, 0) == pi/(2*sqrt(1 - n))
    assert elliptic_pi(2, 1) == -oo
    assert elliptic_pi(-1, 1) == oo
    assert elliptic_pi(n, n) == elliptic_e(n)/(1 - n)
    assert elliptic_pi(oo, m) == 0
    assert elliptic_pi(n, oo) == 0

    assert elliptic_pi(n, -z, m) == -elliptic_pi(n, z, m)

    ni, mi = Symbol('n', extended_real=False), Symbol('m', extended_real=False)
    assert elliptic_pi(ni, z, mi).conjugate() == \
        elliptic_pi(ni.conjugate(), z.conjugate(), mi.conjugate())
    nr, mr = Symbol('n', extended_real=True, negative=True), \
        Symbol('m', extended_real=True, negative=True)
    assert elliptic_pi(nr, z, mr).conjugate() == elliptic_pi(nr, z.conjugate(), mr)
    assert elliptic_pi(n, m).conjugate() == elliptic_pi(n.conjugate(), m.conjugate())
    assert elliptic_pi(n, z, m).conjugate() == conjugate(elliptic_pi(n, z, m))

    assert elliptic_pi(n, z, m).diff(n) == (elliptic_e(z, m) + (m - n)*elliptic_f(z, m)/n +
                                            (n**2 - m)*elliptic_pi(n, z, m)/n - n*sqrt(1 -
                                                                                m*sin(z)**2)*sin(2*z)/(2*(1 - n*sin(z)**2)))/(2*(m - n)*(n - 1))
    assert elliptic_pi(n, z, m).diff(z) == 1/(sqrt(1 - m*sin(z)**2)*(1 - n*sin(z)**2))
    assert elliptic_pi(n, z, m).diff(m) == (elliptic_e(z, m)/(m - 1) + elliptic_pi(n, z, m) -
                                            m*sin(2*z)/(2*(m - 1)*sqrt(1 - m*sin(z)**2)))/(2*(n - m))
    assert elliptic_pi(n, m).diff(n) == (elliptic_e(m) + (m - n)*elliptic_k(m)/n +
                                         (n**2 - m)*elliptic_pi(n, m)/n)/(2*(m - n)*(n - 1))
    assert elliptic_pi(n, m).diff(m) == (elliptic_e(m)/(m - 1) + elliptic_pi(n, m))/(2*(n - m))
    rx, ry = randcplx(), randcplx()
    assert td(elliptic_pi(n, rx, ry), n)
    assert td(elliptic_pi(rx, z, ry), z)
    assert td(elliptic_pi(rx, ry, m), m)
    pytest.raises(ArgumentIndexError, lambda: elliptic_pi(n, z, m).fdiff(4))
    pytest.raises(ArgumentIndexError, lambda: elliptic_pi(n, m).fdiff(3))

    assert elliptic_pi(n, z, m).series(z) == z + z**3*(m/6 + n/3) + \
        z**5*(3*m**2/40 + m*n/10 - m/30 + n**2/5 - n/15) + O(z**6)
<commit_msg>Add workaround for fredrik-johansson/mpmath#571<commit_after>import pytest

from diofant import (I, O, Rational, Symbol, atanh, conjugate, elliptic_e,
                     elliptic_f, elliptic_k, elliptic_pi, gamma, hyper,
                     meijerg, oo, pi, sin, sqrt, tan, zoo)
from diofant.abc import m, n, z
from diofant.core.function import ArgumentIndexError
from diofant.utilities.randtest import random_complex_number as randcplx
from diofant.utilities.randtest import verify_derivative_numerically as td
from diofant.utilities.randtest import verify_numerically as tn

__all__ = ()

i = Symbol('i', integer=True)
j = Symbol('k', integer=True, positive=True)

def test_elliptic_k():
    assert elliptic_k(0) == pi/2
    assert elliptic_k(Rational(1, 2)) == 8*pi**Rational(3, 2)/gamma(-Rational(1, 4))**2
    assert elliptic_k(1) == zoo
    assert elliptic_k(-1) == gamma(Rational(1, 4))**2/(4*sqrt(2*pi))
    assert elliptic_k(oo) == 0
    assert elliptic_k(-oo) == 0
    assert elliptic_k(I*oo) == 0
    assert elliptic_k(-I*oo) == 0
    assert elliptic_k(zoo) == 0

    assert elliptic_k(z).diff(z) == (elliptic_e(z) - (1 - z)*elliptic_k(z))/(2*z*(1 - z))
    assert td(elliptic_k(z), z)
    pytest.raises(ArgumentIndexError, lambda: elliptic_k(z).fdiff(2))

    zi = Symbol('z', extended_real=False)
    assert elliptic_k(zi).conjugate() == elliptic_k(zi.conjugate())
    zr = Symbol('z', extended_real=True, negative=True)
    assert elliptic_k(zr).conjugate() == elliptic_k(zr)
    assert elliptic_k(z).conjugate() == conjugate(elliptic_k(z), evaluate=False)

    assert elliptic_k(z).rewrite(hyper) == \
        (pi/2)*hyper((Rational(1, 2), Rational(1, 2)), (1,), z)
    assert tn(elliptic_k(z), (pi/2)*hyper((Rational(1, 2), Rational(1, 2)), (1,), z))
    assert elliptic_k(z).rewrite(meijerg) == \
        meijerg(((Rational(1, 2), Rational(1, 2)), []), ((0,), (0,)), -z)/2
    assert tn(elliptic_k(z), meijerg(((Rational(1, 2), Rational(1, 2)), []), ((0,), (0,)), -z)/2)

    assert elliptic_k(z).series(z) == pi/2 + pi*z/8 + 9*pi*z**2/128 + \
        25*pi*z**3/512 + 1225*pi*z**4/32768 + 3969*pi*z**5/131072 + O(z**6)

def test_elliptic_f():
    assert elliptic_f(z, 0) == z
    assert elliptic_f(0, m) == 0
    assert elliptic_f(pi*i/2, m) == i*elliptic_k(m)
    assert elliptic_f(z, oo) == 0
    assert elliptic_f(z, -oo) == 0

    assert elliptic_f(-z, m) == -elliptic_f(z, m)

    assert elliptic_f(z, m).diff(z) == 1/sqrt(1 - m*sin(z)**2)
    assert elliptic_f(z, m).diff(m) == elliptic_e(z, m)/(2*m*(1 - m)) - elliptic_f(z, m)/(2*m) - \
        sin(2*z)/(4*(1 - m)*sqrt(1 - m*sin(z)**2))
    r = randcplx()
    assert td(elliptic_f(z, r), z)
    assert td(elliptic_f(r, m), m)
    pytest.raises(ArgumentIndexError, lambda: elliptic_f(z, m).fdiff(3))

    mi = Symbol('m', extended_real=False)
    assert elliptic_f(z, mi).conjugate() == elliptic_f(z.conjugate(), mi.conjugate())
    mr = Symbol('m', extended_real=True, negative=True)
    assert elliptic_f(z, mr).conjugate() == elliptic_f(z.conjugate(), mr)
    assert elliptic_f(z, m).conjugate() == conjugate(elliptic_f(z, m), evaluate=False)

    assert elliptic_f(z, m).series(z) == \
        z + z**5*(3*m**2/40 - m/30) + m*z**3/6 + O(z**6)

def test_elliptic_e():
    assert elliptic_e(z, 0) == z
    assert elliptic_e(0, m) == 0
    assert elliptic_e(i*pi/2, m) == i*elliptic_e(m)
    assert elliptic_e(z, oo) == zoo
    assert elliptic_e(z, -oo) == zoo
    assert elliptic_e(0) == pi/2
    assert elliptic_e(1) == 1
    assert elliptic_e(oo) == I*oo
    assert elliptic_e(-oo) == oo
    assert elliptic_e(zoo) == zoo

    assert elliptic_e(-z, m) == -elliptic_e(z, m)

    assert elliptic_e(z, m).diff(z) == sqrt(1 - m*sin(z)**2)
    assert elliptic_e(z, m).diff(m) == (elliptic_e(z, m) - elliptic_f(z, m))/(2*m)
    assert elliptic_e(z).diff(z) == (elliptic_e(z) - elliptic_k(z))/(2*z)
    r = randcplx()
    assert td(elliptic_e(r, m), m)
    assert td(elliptic_e(z, r), z)
    assert td(elliptic_e(z), z)
    pytest.raises(ArgumentIndexError, lambda: elliptic_e(z, m).fdiff(3))
    pytest.raises(ArgumentIndexError, lambda: elliptic_e(z).fdiff(2))

    mi = Symbol('m', extended_real=False)
    assert elliptic_e(z, mi).conjugate() == elliptic_e(z.conjugate(), mi.conjugate())
    assert elliptic_e(mi).conjugate() == elliptic_e(mi.conjugate())
    mr = Symbol('m', extended_real=True, negative=True)
    assert elliptic_e(z, mr).conjugate() == elliptic_e(z.conjugate(), mr)
    assert elliptic_e(mr).conjugate() == elliptic_e(mr)
    assert elliptic_e(z, m).conjugate() == conjugate(elliptic_e(z, m))
    assert elliptic_e(z).conjugate() == conjugate(elliptic_e(z))

    assert elliptic_e(z).rewrite(hyper) == (pi/2)*hyper((Rational(-1, 2), Rational(1, 2)), (1,), z)
    assert elliptic_e(z, m).rewrite(hyper) == elliptic_e(z, m)
    assert tn(elliptic_e(z), (pi/2)*hyper((Rational(-1, 2), Rational(1, 2)), (1,), z))
    assert elliptic_e(z).rewrite(meijerg) == \
        -meijerg(((Rational(1, 2), Rational(3, 2)), []), ((0,), (0,)), -z)/4
    assert elliptic_e(z, m).rewrite(meijerg) == elliptic_e(z, m)
    assert tn(elliptic_e(z), -meijerg(((Rational(1, 2), Rational(3, 2)), []), ((0,), (0,)), -z)/4)

    assert elliptic_e(z, m).series(z) == \
        z + z**5*(-m**2/40 + m/30) - m*z**3/6 + O(z**6)
    assert elliptic_e(z).series(z) == pi/2 - pi*z/8 - 3*pi*z**2/128 - \
        5*pi*z**3/512 - 175*pi*z**4/32768 - 441*pi*z**5/131072 + O(z**6)

def test_elliptic_pi():
    assert elliptic_pi(0, z, m) == elliptic_f(z, m)
    assert elliptic_pi(1, z, m) == elliptic_f(z, m) + \
        (sqrt(1 - m*sin(z)**2)*tan(z) - elliptic_e(z, m))/(1 - m)
    assert elliptic_pi(n, i*pi/2, m) == i*elliptic_pi(n, m)
    assert elliptic_pi(n, z, 0) == atanh(sqrt(n - 1)*tan(z))/sqrt(n - 1)
    assert elliptic_pi(n, z, n) == elliptic_f(z, n) - elliptic_pi(1, z, n) + tan(z)/sqrt(1 - n*sin(z)**2)
    assert elliptic_pi(oo, z, m) == 0
    assert elliptic_pi(-oo, z, m) == 0
    assert elliptic_pi(n, z, oo) == 0
    assert elliptic_pi(n, z, -oo) == 0
    assert elliptic_pi(0, m) == elliptic_k(m)
    assert elliptic_pi(1, m) == zoo
    assert elliptic_pi(n, 0) == pi/(2*sqrt(1 - n))
    assert elliptic_pi(2, 1) == -oo
    assert elliptic_pi(-1, 1) == oo
    assert elliptic_pi(n, n) == elliptic_e(n)/(1 - n)
    assert elliptic_pi(oo, m) == 0
    assert elliptic_pi(n, oo) == 0

    assert elliptic_pi(n, -z, m) == -elliptic_pi(n, z, m)

    ni, mi = Symbol('n', extended_real=False), Symbol('m', extended_real=False)
    assert elliptic_pi(ni, z, mi).conjugate() == \
        elliptic_pi(ni.conjugate(), z.conjugate(), mi.conjugate())
    nr, mr = Symbol('n', extended_real=True, negative=True), \
        Symbol('m', extended_real=True, negative=True)
    assert elliptic_pi(nr, z, mr).conjugate() == elliptic_pi(nr, z.conjugate(), mr)
    assert elliptic_pi(n, m).conjugate() == elliptic_pi(n.conjugate(), m.conjugate())
    assert elliptic_pi(n, z, m).conjugate() == conjugate(elliptic_pi(n, z, m))

    assert elliptic_pi(n, z, m).diff(n) == (elliptic_e(z, m) + (m - n)*elliptic_f(z, m)/n +
                                            (n**2 - m)*elliptic_pi(n, z, m)/n - n*sqrt(1 -
                                                                                m*sin(z)**2)*sin(2*z)/(2*(1 - n*sin(z)**2)))/(2*(m - n)*(n - 1))
    assert elliptic_pi(n, z, m).diff(z) == 1/(sqrt(1 - m*sin(z)**2)*(1 - n*sin(z)**2))
    assert elliptic_pi(n, z, m).diff(m) == (elliptic_e(z, m)/(m - 1) + elliptic_pi(n, z, m) -
                                            m*sin(2*z)/(2*(m - 1)*sqrt(1 - m*sin(z)**2)))/(2*(n - m))
    assert elliptic_pi(n, m).diff(n) == (elliptic_e(m) + (m - n)*elliptic_k(m)/n +
                                         (n**2 - m)*elliptic_pi(n, m)/n)/(2*(m - n)*(n - 1))
    assert elliptic_pi(n, m).diff(m) == (elliptic_e(m)/(m - 1) + elliptic_pi(n, m))/(2*(n - m))

    # workaround fredrik-johansson/mpmath#571, suggested by Kalevi Suominen
    # in https://github.com/sympy/sympy/issues/20933#issuecomment-779077562
    bounds = {'a': -0.9, 'b': -0.9, 'c': 0.9, 'd': 0.9}
    rx, ry = randcplx(**bounds), randcplx(**bounds)
    assert td(elliptic_pi(n, rx, ry), n, **bounds)
    assert td(elliptic_pi(rx, z, ry), z, **bounds)
    assert td(elliptic_pi(rx, ry, m), m, **bounds)

    pytest.raises(ArgumentIndexError, lambda: elliptic_pi(n, z, m).fdiff(4))
    pytest.raises(ArgumentIndexError, lambda: elliptic_pi(n, m).fdiff(3))

    assert elliptic_pi(n, z, m).series(z) == z + z**3*(m/6 + n/3) + \
        z**5*(3*m**2/40 + m*n/10 - m/30 + n**2/5 - n/15) + O(z**6)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright (c) 2012, 2013 Robert Martens <robert.martens@gmail.com>
# See file COPYING.txt for MIT license terms.

"""
Find floating point vertex coordinates in a Source engine .vmf file.

Looks through a .vmf for any brush planes whose vertices have
non-integer coordinates, printing a list to stdout and optionally
writing out a new file with rounded values. As a side effect of its
brute force regex search, vertalert only checks enabled VisGroups.

"""

import decimal
import os
import re
import sys

def get_max_dev(planes):
    """
    Search a brush's planes for the largest deviation any of its
    vertices' coordinates make from the nearest integer.

    Keyword arguments:
        planes: list of strings representing the planes to check

    """
    floats = []
    for plane in planes:
        # We're only looking for coordinates that were written to the
        # VMF as floating point values, perhaps in scientific notation.
        floats += re.findall(r'-?\d+\.\d+e?-?\d*', plane)
    devs = []
    for coord in floats:
        orig = decimal.Decimal(coord)
        rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
        devs.append(max(rounded, orig) - min(rounded, orig))
    return max(devs)

def fix_plane(plane, regex):
    """
    Use 'regex' pattern to find floating point coordinates in 'plane',
    round to nearest integer, and return corrected plane string.

    Keyword arguments:
        plane: string to search for floats
        regex: Regular Expression pattern to use for search

    """
    floats = re.findall(regex, plane)
    plane_new = plane
    for coord in floats:
        orig = decimal.Decimal(coord)
        rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
        # I replace str(coord) instead of orig here, since
        # that would miss values using scientific notation.
        plane_new = plane_new.replace(str(coord), str(rounded), 1)
    return plane_new

def fix_brushes(brushes, thresh, vmf_in):
    """
    Find and fix brushes with floating point plane vertex coordinates.

    Returns a tuple containing the total number of brushes with floats,
    a list of the greatest deviation any of a brush's coordinates makes
    from the nearest integer, and a fixed version of vmf_in. Corrects
    only deviations less than 'thresh'.

    Keyword arguments:
        brushes: list of brush strings to search
        thresh: threshold below which to ignore/round coordinates
        vmf_in: string containing input VMF contents

    """
    vmf_out = vmf_in
    float_brushes = 0
    percent = len(brushes) / 100.0
    devs = []
    for i, brush in enumerate(brushes):
        brush_id = int(re.search(r'"id"\s"(\d+)"', brush).group(1))
        float_planes = []
        for plane in re.findall(r'"plane"\s".*?"', brush, re.DOTALL):
            if '.' in plane:
                float_planes.append(plane)
        if not float_planes:
            continue
        else:
            float_brushes += 1
        max_dev = get_max_dev(float_planes)
        if max_dev < thresh:
            brush_new = brush
            for plane in float_planes:
                plane_new = fix_plane(plane, r'-?\d+\.\d+e?-?\d*')
                brush_new = brush_new.replace(plane, plane_new)
            vmf_out = vmf_out.replace(brush, brush_new)
        else:
            devs.append((brush_id, max_dev))
        sys.stdout.write('\r%s%% complete' % str(int(i / percent)))
        sys.stdout.flush()
    return (float_brushes, devs, vmf_out)

def print_dev_table(sorted_devs):
    """
    Print, to stdout, a table displaying each brush ID and its
    coordinates' maximum deviation from the nearest integer.

    Keyword arguments:
        sorted_devs: a list of tuples pairing brush id with max deviation

    """
    if sorted_devs:
        max_id_width = len(str(max(sorted_devs[0])))
        left_w = max(max_id_width, len("Suspect id"))
        header = ("Suspect ID").rjust(left_w) + '  ' + "Max dev" + '\n'
        sys.stdout.write(header)
        sys.stdout.write('-' * (len(header) - 1) + '\n')
        for dev in sorted_devs:
            dev_str = str(dev[0]).rjust(left_w) + "  " + str(dev[1]) + "\n"
            sys.stdout.write(dev_str)
            sys.stdout.flush()
        sys.stdout.write('\n')
        sys.stdout.flush()

def vertalert(file_in, fix=False, fixname=None, thresh=None):
    """
    Find, and optionally fix, floating point plane coordinates in a
    Source engine .vmf file.

    Prints a list to stdout of all values greater than or equal to
    thresh, rounds (when using --fix) or ignores all other values.

    Keyword arguments:
        file_in: .vmf file to check
        fix: write out a new file with rounded coordinates (default False)
        fixname: filename when using --fix (default appends _VERTALERT)
        thresh: threshold below which to ignore/round coordinates (default 0.2)

    Please note this function currently only checks enabled VisGroups.

    """
    if not os.path.exists(file_in):
        sys.stderr.write("Could not find " + file_in + "!\n")
        return -1
    if os.path.splitext(file_in)[1] != ".vmf":
        sys.stderr.write("Input must be a .vmf file!\n")
        return -1
    if fixname is None:
        in_name_split = os.path.splitext(file_in)
        fixname = in_name_split[0] + "_VERTALERT" + in_name_split[1]
    if thresh is None:
        thresh = decimal.Context().create_decimal('0.2')

    with open(file_in, 'r') as vmf:
        vmf_in = vmf.read()

    # I no longer read the input in universal line ending mode above,
    # as that changes the type of line endings written to the output
    # when using --fix. Although Hammer can deal with at least Windows
    # and Unix endings, I prefer to write out the same thing I read in.
    # The \r?\n in this regular expression became necessary to ensure
    # Windows, Linux and OS X worked the same way. The pattern is that
    # of the beginning of a solid entry, whose VisGroup is enabled, and
    # breaks down as follows:
    #
    # solid - The word 'solid'.
    # \r?\n - Zero or one carriage return followed by one newline. In
    #         Windows, \n is what \r\n is under Linux and OS X.
    # \t{ - Tab, open curly brace
    # .*? - Zero or more of any character (including newlines, thanks
    #       to my use of re.DOTALL in the call to findall), non-greedy.
    # \r?\n\t} - Zero or one carriage return, one newline, one tab,
    #            closing curly brace.
    brushes = re.findall(r'solid\r?\n\t{.*?\r?\n\t}', vmf_in, re.DOTALL)
    float_brushes, devs, vmf_out = fix_brushes(brushes, thresh, vmf_in)
    sys.stdout.write("\r             \n")
    sys.stdout.flush()

    sorted_devs = sorted(devs, key=lambda dev: dev[-1])
    print_dev_table(sorted_devs)

    if len(sorted_devs) == 1:
        warn_suffix = ""
    else:
        warn_suffix = "es"
    if float_brushes - len(sorted_devs) == 1:
        action_suffix = ""
    else:
        action_suffix = "es"
    if fix:
        action = " automatically rounded"
    else:
        action = " ignored"

    sys.stdout.write(str(float_brushes - len(sorted_devs)) + " brush" +
                     action_suffix + action + '\n')
    sys.stdout.write(str(len(sorted_devs)) + " suspect brush" + warn_suffix +
                     " remaining\n")
    sys.stdout.flush()

    if fix:
        with open(fixname, 'w') as vmf:
            vmf.write(vmf_out)
        sys.stdout.write("\nWrote " + fixname + "!\n")
        sys.stdout.flush()

    return 0

if __name__ == '__main__':
    import argparse
    PARSER = argparse.ArgumentParser(description="VertAlert 0.2.1")
    PARSER.add_argument("input", help="VMF to check")
    PARSER.add_argument(
        "-f", "--fix", help="write fixed up VMF", action="store_true")
    PARSER.add_argument(
        "-fn", "--fixname",
        help="filename to use with --fix, default appends _VERTALERT suffix")
    PARSER.add_argument(
        "-t", "--thresh",
        type=decimal.Decimal,
        help="threshold below which to ignore/round coordinates (default 0.2)")
    ARGS = PARSER.parse_args()

    vertalert(ARGS.input, ARGS.fix, ARGS.fixname, ARGS.thresh)
<commit_msg>Break deviation calc out into get_dev function<commit_after>#!/usr/bin/env python
#
# Copyright (c) 2012, 2013 Robert Martens <robert.martens@gmail.com>
# See file COPYING.txt for MIT license terms.

"""
Find floating point vertex coordinates in a Source engine .vmf file.

Looks through a .vmf for any brush planes whose vertices have
non-integer coordinates, printing a list to stdout and optionally
writing out a new file with rounded values. As a side effect of its
brute force regex search, vertalert only checks enabled VisGroups.

"""

import decimal
import os
import re
import sys

def get_dev(coord):
    orig = decimal.Decimal(coord)
    rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
    return max(rounded, orig) - min(rounded, orig)

def get_max_dev(planes):
    """
    Search a brush's planes for the largest deviation any of its
    vertices' coordinates make from the nearest integer.

    Keyword arguments:
        planes: list of strings representing the planes to check

    """
    floats = []
    for plane in planes:
        # We're only looking for coordinates that were written to the
        # VMF as floating point values, perhaps in scientific notation.
        floats += re.findall(r'-?\d+\.\d+e?-?\d*', plane)
    devs = []
    for coord in floats:
        devs.append(get_dev(coord))
    return max(devs)

def fix_plane(plane, regex):
    """
    Use 'regex' pattern to find floating point coordinates in 'plane',
    round to nearest integer, and return corrected plane string.

    Keyword arguments:
        plane: string to search for floats
        regex: Regular Expression pattern to use for search

    """
    floats = re.findall(regex, plane)
    plane_new = plane
    for coord in floats:
        orig = decimal.Decimal(coord)
        rounded = orig.quantize(1, decimal.ROUND_HALF_EVEN)
        # I replace str(coord) instead of orig here, since
        # that would miss values using scientific notation.
        plane_new = plane_new.replace(str(coord), str(rounded), 1)
    return plane_new

def fix_brushes(brushes, thresh, vmf_in):
    """
    Find and fix brushes with floating point plane vertex coordinates.

    Returns a tuple containing the total number of brushes with floats,
    a list of the greatest deviation any of a brush's coordinates makes
    from the nearest integer, and a fixed version of vmf_in. Corrects
    only deviations less than 'thresh'.

    Keyword arguments:
        brushes: list of brush strings to search
        thresh: threshold below which to ignore/round coordinates
        vmf_in: string containing input VMF contents

    """
    vmf_out = vmf_in
    float_brushes = 0
    percent = len(brushes) / 100.0
    devs = []
    for i, brush in enumerate(brushes):
        brush_id = int(re.search(r'"id"\s"(\d+)"', brush).group(1))
        float_planes = []
        for plane in re.findall(r'"plane"\s".*?"', brush, re.DOTALL):
            if '.' in plane:
                float_planes.append(plane)
        if not float_planes:
            continue
        else:
            float_brushes += 1
        max_dev = get_max_dev(float_planes)
        if max_dev < thresh:
            brush_new = brush
            for plane in float_planes:
                plane_new = fix_plane(plane, r'-?\d+\.\d+e?-?\d*')
                brush_new = brush_new.replace(plane, plane_new)
            vmf_out = vmf_out.replace(brush, brush_new)
        else:
            devs.append((brush_id, max_dev))
        sys.stdout.write('\r%s%% complete' % str(int(i / percent)))
        sys.stdout.flush()
    return (float_brushes, devs, vmf_out)

def print_dev_table(sorted_devs):
    """
    Print, to stdout, a table displaying each brush ID and its
    coordinates' maximum deviation from the nearest integer.

    Keyword arguments:
        sorted_devs: a list of tuples pairing brush id with max deviation

    """
    if sorted_devs:
        max_id_width = len(str(max(sorted_devs[0])))
        left_w = max(max_id_width, len("Suspect id"))
        header = ("Suspect ID").rjust(left_w) + '  ' + "Max dev" + '\n'
        sys.stdout.write(header)
        sys.stdout.write('-' * (len(header) - 1) + '\n')
        for dev in sorted_devs:
            dev_str = str(dev[0]).rjust(left_w) + "  " + str(dev[1]) + "\n"
            sys.stdout.write(dev_str)
            sys.stdout.flush()
        sys.stdout.write('\n')
        sys.stdout.flush()

def vertalert(file_in, fix=False, fixname=None, thresh=None):
    """
    Find, and optionally fix, floating point plane coordinates in a
    Source engine .vmf file.

    Prints a list to stdout of all values greater than or equal to
    thresh, rounds (when using --fix) or ignores all other values.

    Keyword arguments:
        file_in: .vmf file to check
        fix: write out a new file with rounded coordinates (default False)
        fixname: filename when using --fix (default appends _VERTALERT)
        thresh: threshold below which to ignore/round coordinates (default 0.2)

    Please note this function currently only checks enabled VisGroups.

    """
    if not os.path.exists(file_in):
        sys.stderr.write("Could not find " + file_in + "!\n")
        return -1
    if os.path.splitext(file_in)[1] != ".vmf":
        sys.stderr.write("Input must be a .vmf file!\n")
        return -1
    if fixname is None:
        in_name_split = os.path.splitext(file_in)
        fixname = in_name_split[0] + "_VERTALERT" + in_name_split[1]
    if thresh is None:
        thresh = decimal.Context().create_decimal('0.2')

    with open(file_in, 'r') as vmf:
        vmf_in = vmf.read()

    # I no longer read the input in universal line ending mode above,
    # as that changes the type of line endings written to the output
    # when using --fix. Although Hammer can deal with at least Windows
    # and Unix endings, I prefer to write out the same thing I read in.
    # The \r?\n in this regular expression became necessary to ensure
    # Windows, Linux and OS X worked the same way. The pattern is that
    # of the beginning of a solid entry, whose VisGroup is enabled, and
    # breaks down as follows:
    #
    # solid - The word 'solid'.
    # \r?\n - Zero or one carriage return followed by one newline. In
    #         Windows, \n is what \r\n is under Linux and OS X.
    # \t{ - Tab, open curly brace
    # .*? - Zero or more of any character (including newlines, thanks
    #       to my use of re.DOTALL in the call to findall), non-greedy.
    # \r?\n\t} - Zero or one carriage return, one newline, one tab,
    #            closing curly brace.
    brushes = re.findall(r'solid\r?\n\t{.*?\r?\n\t}', vmf_in, re.DOTALL)
    float_brushes, devs, vmf_out = fix_brushes(brushes, thresh, vmf_in)
    sys.stdout.write("\r             \n")
    sys.stdout.flush()

    sorted_devs = sorted(devs, key=lambda dev: dev[-1])
    print_dev_table(sorted_devs)

    if len(sorted_devs) == 1:
        warn_suffix = ""
    else:
        warn_suffix = "es"
    if float_brushes - len(sorted_devs) == 1:
        action_suffix = ""
    else:
        action_suffix = "es"
    if fix:
        action = " automatically rounded"
    else:
        action = " ignored"

    sys.stdout.write(str(float_brushes - len(sorted_devs)) + " brush" +
                     action_suffix + action + '\n')
    sys.stdout.write(str(len(sorted_devs)) + " suspect brush" + warn_suffix +
                     " remaining\n")
    sys.stdout.flush()

    if fix:
        with open(fixname, 'w') as vmf:
            vmf.write(vmf_out)
        sys.stdout.write("\nWrote " + fixname + "!\n")
        sys.stdout.flush()

    return 0

if __name__ == '__main__':
    import argparse
    PARSER = argparse.ArgumentParser(description="VertAlert 0.2.1")
    PARSER.add_argument("input", help="VMF to check")
    PARSER.add_argument(
        "-f", "--fix", help="write fixed up VMF", action="store_true")
    PARSER.add_argument(
        "-fn", "--fixname",
        help="filename to use with --fix, default appends _VERTALERT suffix")
    PARSER.add_argument(
        "-t", "--thresh",
        type=decimal.Decimal,
        help="threshold below which to ignore/round coordinates (default 0.2)")
    ARGS = PARSER.parse_args()

    vertalert(ARGS.input, ARGS.fix, ARGS.fixname, ARGS.thresh)
<|endoftext|>


<commit_before>#!/usr/bin/python
#- Copyright 2014 GOTO 10.
#- Licensed under the Apache License, Version 2.0 (see LICENSE).

## Main entry-point. Dispatches to other scripts.

from command import Command, shell_escape
import argparse
import logging
import node
import os
import os.path
import platform
import re
import subprocess
import sys

# Current version of the init script. Bump this to force build scripts to
# regenerate.
_VERSION = 2

# Returns the default value to use for the language.
def get_default_shell():
  system = platform.system()
  if system in ["Linux", "Darwin"]:
    return "sh"
  elif system == "Windows":
    return "bat"
  else:
    return None

_DEFAULT_SYSTEMS = {
  "Windows": "windows",
  "Darwin": "mac"
}
def get_default_system():
  return _DEFAULT_SYSTEMS.get(platform.system(), "posix")

# The main entry-point class.
class MkMk(object):

  def __init__(self, args):
    parser = self.build_option_parser()
    if "--" in args:
      index = args.index("--")
      self.extras = args[index+1:]
      args = args[0:index]
    else:
      self.extras = []
    (self.options, self.unknown) = parser.parse_known_args(args)

  # Parse the options shared between all the different handers.
  def build_option_parser(self):
    parser = argparse.ArgumentParser()
    commands = sorted(self.get_handlers().keys())
    parser.add_argument('command', choices=commands)
    parser.add_argument('--config', default=None, help='The root configuration')
    parser.add_argument('--makefile', default=None,
      help='Name of the makefile to generate')
    parser.add_argument('--bindir', default='out',
      help='The location to store generated files in')
    parser.add_argument('--buildflags', default=None,
      help='Flags to pass through to the build process')
    parser.add_argument('--extension', default=[], action='append',
      help='Specify a build extension to enable')
    parser.add_argument('--noisy', default=False, action='store_true',
      help='Echo all commands being executed')
    parser.add_argument('--shell', default=get_default_shell(),
      help='Which shell to generate a build script for')
    parser.add_argument('--script', default=None,
      help='Name of the build script to generate')
    parser.add_argument('--before', default=None,
      help='Version to compare with when running has_changed')
    parser.add_argument('--system', default=get_default_system(),
      help='The system/os we\'re building on')
    return parser

  # Returns a map from handler names to handlers.
  def get_handlers(self):
    result = {}
    for (name, value) in MkMk.__dict__.items():
      match = re.match(r'handle_(.*)', name)
      if match is None:
        continue
      action = match.group(1)
      result[action] = value
    return result

  # Execute the makefile command.
  def handle_makefile(self):
    self.ensure_no_unknown()
    import makefile
    runner = makefile.MkMkMakefile(self.options)
    runner.run()

  def handle_init(self):
    import init
    mkmk = sys.argv[0]
    return init.generate_build_script(_VERSION, mkmk, self.options, self.unknown)

  def handle_run(self):
    script = self.handle_init()
    command = [script] + self.extras
    return subprocess.check_call(command)

  # Checks whether the contents of this script still MD5-hashes to the given
  # value.
  def handle_has_changed(self):
    if self.options.before == str(_VERSION):
      print("Same")
    else:
      print("Changed")

  # Checks that there were no unknown flags, otherwise dies with an error.
  def ensure_no_unknown(self):
    if len(self.unknown) > 0:
      print "Unknown flags: %s." % " ".join(self.unknown)
      sys.exit(1)

  # Dispatches to the appropriate command.
  def run(self):
    handlers = self.get_handlers()
    command = self.options.command
    handler = handlers[command]
    return handler(self)

def main():
  mkmk = MkMk(sys.argv[1:])
  try:
    mkmk.run()
  except KeyboardInterrupt, ki:
    logging.error("Interrupted; exiting.")
    sys.exit(1)
  except subprocess.CalledProcessError, ce:
    logging.error("%s", ce)
    sys.exit(1)

if __name__ == "__main__":
  main()
<commit_msg>Added --self<commit_after>#!/usr/bin/python
#- Copyright 2014 GOTO 10.
#- Licensed under the Apache License, Version 2.0 (see LICENSE).

## Main entry-point. Dispatches to other scripts.

from command import Command, shell_escape
import argparse
import logging
import node
import os
import os.path
import platform
import re
import subprocess
import sys

# Current version of the init script. Bump this to force build scripts to
# regenerate.
_VERSION = 2

# Returns the default value to use for the language.
def get_default_shell():
  system = platform.system()
  if system in ["Linux", "Darwin"]:
    return "sh"
  elif system == "Windows":
    return "bat"
  else:
    return None

_DEFAULT_SYSTEMS = {
  "Windows": "windows",
  "Darwin": "mac"
}
def get_default_system():
  return _DEFAULT_SYSTEMS.get(platform.system(), "posix")

# The main entry-point class.
class MkMk(object):

  def __init__(self, args):
    parser = self.build_option_parser()
    if "--" in args:
      index = args.index("--")
      self.extras = args[index+1:]
      args = args[0:index]
    else:
      self.extras = []
    (self.options, self.unknown) = parser.parse_known_args(args)

  # Parse the options shared between all the different handers.
  def build_option_parser(self):
    parser = argparse.ArgumentParser()
    commands = sorted(self.get_handlers().keys())
    parser.add_argument('command', choices=commands)
    parser.add_argument('--config', default=None, help='The root configuration')
    parser.add_argument('--makefile', default=None,
      help='Name of the makefile to generate')
    parser.add_argument('--bindir', default='out',
      help='The location to store generated files in')
    parser.add_argument('--buildflags', default=None,
      help='Flags to pass through to the build process')
    parser.add_argument('--extension', default=[], action='append',
      help='Specify a build extension to enable')
    parser.add_argument('--noisy', default=False, action='store_true',
      help='Echo all commands being executed')
    parser.add_argument('--shell', default=get_default_shell(),
      help='Which shell to generate a build script for')
    parser.add_argument('--script', default=None,
      help='Name of the build script to generate')
    parser.add_argument('--before', default=None,
      help='Version to compare with when running has_changed')
    parser.add_argument('--system', default=get_default_system(),
      help='The system/os we\'re building on')
    parser.add_argument('--self', default=None,
      help='Optional argument specifying how to run mkmk.')
    return parser

  # Returns a map from handler names to handlers.
  def get_handlers(self):
    result = {}
    for (name, value) in MkMk.__dict__.items():
      match = re.match(r'handle_(.*)', name)
      if match is None:
        continue
      action = match.group(1)
      result[action] = value
    return result

  # Execute the makefile command.
  def handle_makefile(self):
    self.ensure_no_unknown()
    import makefile
    runner = makefile.MkMkMakefile(self.options)
    runner.run()

  def handle_init(self):
    import init
    mkmk = self.options.self or sys.argv[0]
    return init.generate_build_script(_VERSION, mkmk, self.options, self.unknown)

  def handle_run(self):
    script = self.handle_init()
    command = [script] + self.extras
    return subprocess.check_call(command)

  # Checks whether the contents of this script still MD5-hashes to the given
  # value.
  def handle_has_changed(self):
    if self.options.before == str(_VERSION):
      print("Same")
    else:
      print("Changed")

  # Checks that there were no unknown flags, otherwise dies with an error.
  def ensure_no_unknown(self):
    if len(self.unknown) > 0:
      print "Unknown flags: %s." % " ".join(self.unknown)
      sys.exit(1)

  # Dispatches to the appropriate command.
  def run(self):
    handlers = self.get_handlers()
    command = self.options.command
    handler = handlers[command]
    return handler(self)

def main():
  mkmk = MkMk(sys.argv[1:])
  try:
    mkmk.run()
  except KeyboardInterrupt, ki:
    logging.error("Interrupted; exiting.")
    sys.exit(1)
  except subprocess.CalledProcessError, ce:
    logging.error("%s", ce)
    sys.exit(1)

if __name__ == "__main__":
  main()
<|endoftext|>


<commit_before>import random

from StringIO import StringIO
from datetime import datetime
from dateutil.relativedelta import relativedelta

from django.template.defaultfilters import slugify
from django.core.management.base import BaseCommand, CommandError
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.contrib.auth.models import User
from django.conf import settings

from challenges.models import Submission, Category, Phase, Challenge, Project
import identicon

# Expected constants in the DB
IGNITE_PROJECT_SLUG = 'us-ignite'
IGNITE_CHALLENGE_SLUG = 'ignite-challenge'
IGNITE_IDEATION_NAME = 'Ideation'
IGNITE_DEVELOPMENT_NAME = 'Development'

def update_phase_date(phase, start_date, end_date, order):
    phase.start_date = start_date
    phase.end_date = end_date
    phase.order = order
    phase.save()
    return phase

# wording
d = open('/usr/share/dict/words', "r").readlines()
_random_words = lambda n: " ".join([random.choice(d).lower().rstrip() for i in range(n)])
_random_paragraph = lambda: _random_words(30).capitalize()
_random_paragraphs = lambda n: ".\n\n".join([_random_paragraph() for i in range(n)])

def _random_image(name):
    name = "dummy/dummy_%s.png" % name
    icon = identicon.render_identicon(random.randint(5 ** 5, 10 ** 10),
                                      random.randint(64, 250))
    # using storage
    tmp_file = StringIO()
    icon.save(tmp_file, 'PNG')
    return default_storage.save(name, ContentFile(tmp_file.getvalue()))

def create_user(handle):
    """Helper to create Users with a profile"""
    email = '%s@%s.com' % (handle, handle)
    user = User.objects.create_user(handle, email, handle)
    profile = user.get_profile()
    # middleware needs a name if not will redirect to edit
    profile.name = handle.title()
    profile.save()
    return profile

class Command(BaseCommand):
    help = """Bootstraps Mozilla Ignite data in the ideation phase, without the
    development phase changes. Used to simulate the current content on the site.
    This is only suitable to be run on a Development environment.
    """

    USERS = 50
    POST_PER_USER = 3

    def handle(self, *args, **kwargs):
        if not settings.DEBUG:
            raise CommandError("IMPORTANT: Make sure you only run this on a "
                               "Development environment ")
        print("Make sure, the project code base has NOT the development phase "
              "changes, before running this")
        answer = raw_input("Does the project codebase ONLY contain the "
                           "Ideation codebase (yes/no)? ")
        if answer != 'yes':
            raise CommandError("Make sure your codebase .")
        answer = raw_input("This will IRREVERSIBLY add TEST DATA to your "
                           "database. Proceed (yes/no)? ")
        if answer != 'yes':
            raise CommandError("Phew. Import canceled.")
        now = datetime.utcnow()
        project = Project.objects.get(slug=IGNITE_PROJECT_SLUG)
        challenge = Challenge.objects.get(slug=IGNITE_CHALLENGE_SLUG)
        ideation = Phase.objects.get(name=IGNITE_IDEATION_NAME)
        category = Category.objects.all()[0]
        try:
            development = Phase.objects.get(name=IGNITE_DEVELOPMENT_NAME)
        except Phase.DoesNotExist:
            development = None
        # Open ideation phase
        ideation = update_phase_date(ideation,
                                     now - relativedelta(days=3),
                                     now + relativedelta(days=30), 1)
        if development:
            development = update_phase_date(development,
                                            now + relativedelta(days=50),
                                            now + relativedelta(days=80), 2)
        for i in range(self.USERS):
            profile = create_user(slugify(_random_words(1)))
            for p in range(self.POST_PER_USER):
                data = {
                    'title': _random_words(4).title(),
                    'brief_description': _random_words(15),
                    'description': _random_paragraphs(5),
                    'category': category,
                    'life_improvements': _random_paragraphs(2),
                    'interest_making': _random_paragraphs(2),
                    'phase': ideation,
                    'created_by': profile,
                    'sketh_note': _random_image(_random_words(1)),
                    }
                submission = Submission.objects.create(**data)
        print "DONE!"
<commit_msg>Slugify of the thumbnail<commit_after>import random

from StringIO import StringIO
from datetime import datetime
from dateutil.relativedelta import relativedelta

from django.template.defaultfilters import slugify
from django.core.management.base import BaseCommand, CommandError
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.contrib.auth.models import User
from django.conf import settings

from challenges.models import Submission, Category, Phase, Challenge, Project
import identicon

# Expected constants in the DB
IGNITE_PROJECT_SLUG = 'us-ignite'
IGNITE_CHALLENGE_SLUG = 'ignite-challenge'
IGNITE_IDEATION_NAME = 'Ideation'
IGNITE_DEVELOPMENT_NAME = 'Development'

def update_phase_date(phase, start_date, end_date, order):
    phase.start_date = start_date
    phase.end_date = end_date
    phase.order = order
    phase.save()
    return phase

# wording
d = open('/usr/share/dict/words', "r").readlines()
_random_words = lambda n: " ".join([random.choice(d).lower().rstrip() for i in range(n)])
_random_paragraph = lambda: _random_words(30).capitalize()
_random_paragraphs = lambda n: ".\n\n".join([_random_paragraph() for i in range(n)])

def _random_image(name):
    name = "dummy/dummy_%s.png" % name
    icon = identicon.render_identicon(random.randint(5 ** 5, 10 ** 10),
                                      random.randint(64, 250))
    # using storage
    tmp_file = StringIO()
    icon.save(tmp_file, 'PNG')
    return default_storage.save(name, ContentFile(tmp_file.getvalue()))

def create_user(handle):
    """Helper to create Users with a profile"""
    email = '%s@%s.com' % (handle, handle)
    user = User.objects.create_user(handle, email, handle)
    profile = user.get_profile()
    # middleware needs a name if not will redirect to edit
    profile.name = handle.title()
    profile.save()
    return profile

class Command(BaseCommand):
    help = """Bootstraps Mozilla Ignite data in the ideation phase, without the
    development phase changes. Used to simulate the current content on the site.
    This is only suitable to be run on a Development environment.
    """

    USERS = 50
    POST_PER_USER = 3

    def handle(self, *args, **kwargs):
        if not settings.DEBUG:
            raise CommandError("IMPORTANT: Make sure you only run this on a "
                               "Development environment ")
        print("Make sure, the project code base has NOT the development phase "
              "changes, before running this")
        answer = raw_input("Does the project codebase ONLY contain the "
                           "Ideation codebase (yes/no)? ")
        if answer != 'yes':
            raise CommandError("Make sure your codebase .")
        answer = raw_input("This will IRREVERSIBLY add TEST DATA to your "
                           "database. Proceed (yes/no)? ")
        if answer != 'yes':
            raise CommandError("Phew. Import canceled.")
        now = datetime.utcnow()
        project = Project.objects.get(slug=IGNITE_PROJECT_SLUG)
        challenge = Challenge.objects.get(slug=IGNITE_CHALLENGE_SLUG)
        ideation = Phase.objects.get(name=IGNITE_IDEATION_NAME)
        category = Category.objects.all()[0]
        try:
            development = Phase.objects.get(name=IGNITE_DEVELOPMENT_NAME)
        except Phase.DoesNotExist:
            development = None
        # Open ideation phase
        ideation = update_phase_date(ideation,
                                     now - relativedelta(days=3),
                                     now + relativedelta(days=30), 1)
        if development:
            development = update_phase_date(development,
                                            now + relativedelta(days=50),
                                            now + relativedelta(days=80), 2)
        for i in range(self.USERS):
            profile = create_user(slugify(_random_words(1)))
            for p in range(self.POST_PER_USER):
                data = {
                    'title': _random_words(4).title(),
                    'brief_description': _random_words(15),
                    'description': _random_paragraphs(5),
                    'category': category,
                    'life_improvements': _random_paragraphs(2),
                    'interest_making': _random_paragraphs(2),
                    'phase': ideation,
                    'created_by': profile,
                    'sketh_note': _random_image(slugify(_random_words(1))),
                    }
                submission = Submission.objects.create(**data)
        print "DONE!"
<|endoftext|>


<commit_before>#!/usr/bin/env python

from datetime import datetime
import traceback
import functools
import multiprocessing
from multiprocessing import Pool

import pytz
import time
import os
import sys
from pathlib import Path

import argparse

import numpy as np
import scipy
import pandas as pd

import matplotlib
from matplotlib import style
style.use('ggplot')
import matplotlib.pyplot as plt

from astropy import stats
from astropy.io import fits, ascii
from astropy.table import Column
from astropy.time import Time

import astropy.units as u
from astropy.io import fits
from mmtwfs.wfs import WFSFactory

import logging

log = logging.getLogger('WFS Reanalyze')
log.setLevel(logging.INFO)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

tz = pytz.timezone("America/Phoenix")

# instantiate all of the WFS systems...
wfs_keys = ['f9', 'newf9', 'f5', 'mmirs', 'binospec']
wfs_systems = {}
wfs_names = {}
for w in wfs_keys:
    wfs_systems[w] = WFSFactory(wfs=w)
    wfs_names[w] = wfs_systems[w].name

# give mmirs a default
wfs_systems['mmirs'].default_mode = 'mmirs1'

# map f9 to oldf9
wfs_systems['oldf9'] = wfs_systems['f9']

# loosen mmirs centering tolerance to deal with past camera misalignments
wfs_systems['mmirs'].cen_tol = 120.
wfs_systems['binospec'].cen_tol = 120.

def check_image(f, wfskey=None):
    hdr = {}
    with fits.open(f, output_verify="ignore") as hdulist:
        for h in hdulist:
            hdr.update(h.header)
        data = hdulist[-1].data

    if 'CURR_TEMP' in hdr:
        hdr['OSSTEMP'] = hdr['CURR_TEMP']

    # if wfskey is None, figure out which WFS from the header info...
    if wfskey is None:
        # check for MMIRS
        if 'WFSNAME' in hdr:
            if 'mmirs' in hdr['WFSNAME']:
                wfskey = 'mmirs'
        if 'mmirs' in f.name:
            wfskey = 'mmirs'

        # check for binospec
        if 'bino' in f.name or 'wfs_ff_cal_img' in f.name:
            wfskey = 'binospec'
        if 'ORIGIN' in hdr:
            if 'Binospec' in hdr['ORIGIN']:
                wfskey = 'binospec'

        # check for new F/9
        if 'f9wfs' in f.name:
            wfskey = 'newf9'
        if 'OBSERVER' in hdr:
            if 'F/9 WFS' in hdr['OBSERVER']:
                wfskey = 'newf9'
        if wfskey is None and 'CAMERA' in hdr:
            if 'F/9 WFS' in hdr['CAMERA']:
                wfskey = 'newf9'

        # check for old F/9
        if 'INSTRUME' in hdr:
            if 'Apogee' in hdr['INSTRUME']:
                wfskey = 'oldf9'
        if 'DETECTOR' in hdr:
            if 'Apogee' in hdr['DETECTOR']:
                wfskey = 'oldf9'

        # check for F/5 (hecto)
        if wfskey is None and 'SEC' in hdr:  # mmirs has SEC in header as well and is caught above
            if 'F5' in hdr['SEC']:
                wfskey = 'f5'

        # some early F/5 data had no real id in their headers...
        if wfskey is None and Path(f.parent / "F5").exists():
            wfskey = 'f5'

    if wfskey is None:
        # if wfskey is still None at this point, whinge.
        log.error(f"Can't determine WFS for {f.name}...")

    if 'AIRMASS' not in hdr:
        if 'SECZ' in hdr:
            hdr['AIRMASS'] = hdr['SECZ']
        else:
            hdr['AIRMASS'] = np.nan

    if 'EXPTIME' not in hdr:
        hdr['EXPTIME'] = np.nan

    # we need to fix the headers in all cases to have a proper DATE-OBS entry with
    # properly formatted FITS timestamp.  in the meantime, this hack gets us what we need
    # for analysis in pandas.
    dtime = None
    if 'DATEOBS' in hdr:
        dateobs = hdr['DATEOBS']
        if 'UT' in hdr:
            ut = hdr['UT'].strip()
        elif 'TIME-OBS' in hdr:
            ut = hdr['TIME-OBS']
        else:
            ut = "07:00:00"  # midnight
        timestring = dateobs + " " + ut + " UTC"
        if '-' in timestring:
            dtime = datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S %Z")
        else:
            dtime = datetime.strptime(timestring, "%a %b %d %Y %H:%M:%S %Z")

    else:
        if wfskey == "oldf9":
            d = hdr['DATE-OBS']
            if '/' in d:
                day, month, year = d.split('/')
                year = str(int(year) + 1900)
                timestring = year + "-" + month + "-" + day + " " + hdr['TIME-OBS'] + " UTC"
            else:
                timestring = d + " " + hdr['TIME-OBS'] + " UTC"
            dtime = datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S %Z")
        else:
            if 'DATE-OBS' in hdr:
                timestring = hdr['DATE-OBS'] + " UTC"
                try:
                    dtime = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S.%f %Z")
                except:
                    dtime = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S %Z")
                # mmirs uses local time in this header pre-2019
                if wfskey == 'mmirs' and dtime < datetime.fromisoformat("2019-01-01T12:00:00"):
                    local_dt = tz.localize(dtime)
                    dtime = local_dt.astimezone(pytz.utc)
            else:
                if 'MJD' in hdr:
                    dtime = Time(hdr['MJD'], format='mjd').to_datetime()
                else:
                    dt = datetime.fromtimestamp(f.stat().st_ctime)
                    local_dt = tz.localize(dt)
                    dtime = local_dt.astimezone(pytz.utc)

    if dtime is None:
        log.error(f"No valid timestamp in header for {f.name}...")
        obstime = None
    else:
        obstime = dtime.isoformat().replace('+00:00', '')

    hdr['WFSKEY'] = wfskey
    hdr['OBS-TIME'] = obstime
    return data, hdr

def process_image(f, force=False):
    """
    Process FITS file, f, to get info we want from the header and then analyse it with the
    appropriate WFS instance. Return results in a comma-separated line that will be collected
    and saved in a CSV file.
    """
    if "Ref" in str(f) or "sog" in str(f):
        return None

    outfile = f.parent / (f.stem + ".output")
    if not force and Path.exists(outfile):
        log.info(f"Already processed {f.name}, loading previous data...")
        with open(outfile, 'r') as fp:
            line = fp.readlines()[0]
            return line

    try:
        data, hdr = check_image(f)
    except Exception as e:
        log.error(f"Problem checking {f}: {e}")
        return None

    wfskey = hdr['WFSKEY']
    obstime = hdr['OBS-TIME']
    airmass = hdr['AIRMASS']
    exptime = hdr['EXPTIME']
    az = hdr.get('AZ', np.nan)
    el = hdr.get('EL', np.nan)
    tiltx = hdr.get('TILTX', np.nan)
    tilty = hdr.get('TILTY', np.nan)
    transx = hdr.get('TRANSX', np.nan)
    transy = hdr.get('TRANSY', np.nan)
    focus = hdr.get('FOCUS', np.nan)
    if np.isnan(focus) and 'TRANSZ' in hdr:
        focus = hdr.get('TRANSZ', np.nan)
    osst = hdr.get('OSSTEMP', np.nan)
    if 'OUT_T' in hdr:
        outt = hdr.get('OUT_T', np.nan)
    else:
        outt = hdr.get('T_OUT', np.nan)
    if 'CHAM_T' in hdr:
        chamt = hdr.get('CHAM_T', np.nan)
    else:
        chamt = hdr.get('T_CHAM', np.nan)

    # being conservative here and only using data that has proper slope determination
    # and wavefront solution. also want to get statistics on the quality of the wavefront fits.
    try:
        results = wfs_systems[wfskey].measure_slopes(str(f), plot=False)
    except:
        log.error(f"Problem analyzing {f.name}...")
        results = {}
        results['slopes'] = None

    if results['slopes'] is not None:
        try:
            zresults = wfs_systems[wfskey].fit_wavefront(results, plot=False)
            zv = zresults['zernike']
            focerr = wfs_systems[wfskey].calculate_focus(zv)
            cc_x_err, cc_y_err = wfs_systems[wfskey].calculate_cc(zv)
            line = f"{obstime},{wfskey},{f.name},{exptime},{airmass},{az},{el},{osst},{outt},{chamt},{tiltx},{tilty},{transx},{transy},{focus},{focerr.value},{cc_x_err.value},{cc_y_err.value},{results['xcen']},{results['ycen']},{results['seeing'].value},{results['raw_seeing'].value},{results['fwhm']},{zresults['zernike_rms'].value},{zresults['residual_rms'].value}\n"
            zfile = f.parent / (f.stem + ".reanalyze.zernike")
            zresults['zernike'].save(filename=zfile)
            spotfile = f.parent / (f.stem + ".spots.csv")
            results['spots'].write(spotfile, overwrite=True)
            with open(outfile, 'w') as fp:
                fp.write(line)
            return line
        except Exception as e:
            log.error(f"Problem fitting wavefront for {f.name}: {e}")
            return None
    else:
        return None

def main():
    """
    Take directories as argument and go through each one to process files in parallel using Pool.map()
    """
    parser = argparse.ArgumentParser(description='Utility for parallelized batch reprocessing of WFS data')

    parser.add_argument(
        '-r', '--rootdir',
        metavar="<Root directory containing WFS data>",
        default="."
    )

    parser.add_argument(
        '-d', '--dirs',
        metavar="<Glob of directories to process>",
        default="*"
    )

    parser.add_argument(
        '--forcedir',
        metavar="<Force rebuild of CSV for a directory>",
        action='store_true'
    )

    parser.add_argument(
        '--force',
        metavar="<Force reanalysis of individual data files>",
        action="store_true"
    )

    parser.add_argument(
        '-n', '--nproc',
        metavar="<Number of parallel processes to use>",
        default=int(multiprocessing.cpu_count()/2)  # MKL uses a lot of threads so best to limit Pool to half available cores
    )

    args = parser.parse_args()

    rootdir = Path(args.rootdir)

    log.info(f"Using {args.nproc} cores...")

    dirs = sorted(list(rootdir.glob(args.dirs)))  # pathlib, where have you been all my life!
    csv_header = "time,wfs,file,exptime,airmass,az,el,osst,outt,chamt,tiltx,tilty,transx,transy,focus,focerr,cc_x_err,cc_y_err,xcen,ycen,seeing,raw_seeing,fwhm,wavefront_rms,residual_rms\n"
    slow = False
    for d in dirs:
        if d.is_dir():
            if not args.forcedir and Path.exists(d / "reanalyze_results.csv"):
                log.info(f"Already processed {d.name}...")
            else:
                try:
                    lines = []
                    lines.append(csv_header)
                    night = int(d.name)  # valid WFS directories are ints of the form YYYYMMDD. if not this form, int barfs
                    fitsfiles = d.glob("*.fits")
                    log.info(f"Processing {d}...")
                    if slow:
                        plines = []
                        for f in fitsfiles:
                            log.debug(f"Processing {f}...")
                            l = process_image(f, force=args.force)
                            plines.append(l)
                    else:
                        with Pool(processes=args.nproc) as pool:
                            plines = pool.map(process_image, fitsfiles)  # plines comes out in same order as fitslines!

                    plines = list(filter(None.__ne__, plines))  # trim out any None entries
                    if len(plines) > 0:
                        lines.extend(plines)
                        with open(d / "reanalyze_results.csv", "w") as f:
                            f.writelines(lines)

                except ValueError as e:  # this means running int(d.name) failed so it's not a valid directory...
                    log.warn(f"Skipping %s... ({e})" % d.name)
<commit_msg>fix errors in argparse setup<commit_after>#!/usr/bin/env python

from datetime import datetime
import traceback
import functools
import multiprocessing
from multiprocessing import Pool

import pytz
import time
import os
import sys
from pathlib import Path

import argparse

import numpy as np
import scipy
import pandas as pd

import matplotlib
from matplotlib import style
style.use('ggplot')
import matplotlib.pyplot as plt

from astropy import stats
from astropy.io import fits, ascii
from astropy.table import Column
from astropy.time import Time

import astropy.units as u
from astropy.io import fits
from mmtwfs.wfs import WFSFactory

import logging

log = logging.getLogger('WFS Reanalyze')
log.setLevel(logging.INFO)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(levelname)s - %(message)s')
ch.setFormatter(formatter)
log.addHandler(ch)

tz = pytz.timezone("America/Phoenix")

# instantiate all of the WFS systems...
wfs_keys = ['f9', 'newf9', 'f5', 'mmirs', 'binospec']
wfs_systems = {}
wfs_names = {}
for w in wfs_keys:
    wfs_systems[w] = WFSFactory(wfs=w)
    wfs_names[w] = wfs_systems[w].name

# give mmirs a default
wfs_systems['mmirs'].default_mode = 'mmirs1'

# map f9 to oldf9
wfs_systems['oldf9'] = wfs_systems['f9']

# loosen mmirs centering tolerance to deal with past camera misalignments
wfs_systems['mmirs'].cen_tol = 120.
wfs_systems['binospec'].cen_tol = 120.

def check_image(f, wfskey=None):
    hdr = {}
    with fits.open(f, output_verify="ignore") as hdulist:
        for h in hdulist:
            hdr.update(h.header)
        data = hdulist[-1].data

    if 'CURR_TEMP' in hdr:
        hdr['OSSTEMP'] = hdr['CURR_TEMP']

    # if wfskey is None, figure out which WFS from the header info...
    if wfskey is None:
        # check for MMIRS
        if 'WFSNAME' in hdr:
            if 'mmirs' in hdr['WFSNAME']:
                wfskey = 'mmirs'
        if 'mmirs' in f.name:
            wfskey = 'mmirs'

        # check for binospec
        if 'bino' in f.name or 'wfs_ff_cal_img' in f.name:
            wfskey = 'binospec'
        if 'ORIGIN' in hdr:
            if 'Binospec' in hdr['ORIGIN']:
                wfskey = 'binospec'

        # check for new F/9
        if 'f9wfs' in f.name:
            wfskey = 'newf9'
        if 'OBSERVER' in hdr:
            if 'F/9 WFS' in hdr['OBSERVER']:
                wfskey = 'newf9'
        if wfskey is None and 'CAMERA' in hdr:
            if 'F/9 WFS' in hdr['CAMERA']:
                wfskey = 'newf9'

        # check for old F/9
        if 'INSTRUME' in hdr:
            if 'Apogee' in hdr['INSTRUME']:
                wfskey = 'oldf9'
        if 'DETECTOR' in hdr:
            if 'Apogee' in hdr['DETECTOR']:
                wfskey = 'oldf9'

        # check for F/5 (hecto)
        if wfskey is None and 'SEC' in hdr:  # mmirs has SEC in header as well and is caught above
            if 'F5' in hdr['SEC']:
                wfskey = 'f5'

        # some early F/5 data had no real id in their headers...
        if wfskey is None and Path(f.parent / "F5").exists():
            wfskey = 'f5'

    if wfskey is None:
        # if wfskey is still None at this point, whinge.
        log.error(f"Can't determine WFS for {f.name}...")

    if 'AIRMASS' not in hdr:
        if 'SECZ' in hdr:
            hdr['AIRMASS'] = hdr['SECZ']
        else:
            hdr['AIRMASS'] = np.nan

    if 'EXPTIME' not in hdr:
        hdr['EXPTIME'] = np.nan

    # we need to fix the headers in all cases to have a proper DATE-OBS entry with
    # properly formatted FITS timestamp.  in the meantime, this hack gets us what we need
    # for analysis in pandas.
    dtime = None
    if 'DATEOBS' in hdr:
        dateobs = hdr['DATEOBS']
        if 'UT' in hdr:
            ut = hdr['UT'].strip()
        elif 'TIME-OBS' in hdr:
            ut = hdr['TIME-OBS']
        else:
            ut = "07:00:00"  # midnight
        timestring = dateobs + " " + ut + " UTC"
        if '-' in timestring:
            dtime = datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S %Z")
        else:
            dtime = datetime.strptime(timestring, "%a %b %d %Y %H:%M:%S %Z")

    else:
        if wfskey == "oldf9":
            d = hdr['DATE-OBS']
            if '/' in d:
                day, month, year = d.split('/')
                year = str(int(year) + 1900)
                timestring = year + "-" + month + "-" + day + " " + hdr['TIME-OBS'] + " UTC"
            else:
                timestring = d + " " + hdr['TIME-OBS'] + " UTC"
            dtime = datetime.strptime(timestring, "%Y-%m-%d %H:%M:%S %Z")
        else:
            if 'DATE-OBS' in hdr:
                timestring = hdr['DATE-OBS'] + " UTC"
                try:
                    dtime = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S.%f %Z")
                except:
                    dtime = datetime.strptime(timestring, "%Y-%m-%dT%H:%M:%S %Z")
                # mmirs uses local time in this header pre-2019
                if wfskey == 'mmirs' and dtime < datetime.fromisoformat("2019-01-01T12:00:00"):
                    local_dt = tz.localize(dtime)
                    dtime = local_dt.astimezone(pytz.utc)
            else:
                if 'MJD' in hdr:
                    dtime = Time(hdr['MJD'], format='mjd').to_datetime()
                else:
                    dt = datetime.fromtimestamp(f.stat().st_ctime)
                    local_dt = tz.localize(dt)
                    dtime = local_dt.astimezone(pytz.utc)

    if dtime is None:
        log.error(f"No valid timestamp in header for {f.name}...")
        obstime = None
    else:
        obstime = dtime.isoformat().replace('+00:00', '')

    hdr['WFSKEY'] = wfskey
    hdr['OBS-TIME'] = obstime
    return data, hdr

def process_image(f, force=False):
    """
    Process FITS file, f, to get info we want from the header and then analyse it with the
    appropriate WFS instance. Return results in a comma-separated line that will be collected
    and saved in a CSV file.
    """
    if "Ref" in str(f) or "sog" in str(f):
        return None

    outfile = f.parent / (f.stem + ".output")
    if not force and Path.exists(outfile):
        log.info(f"Already processed {f.name}, loading previous data...")
        with open(outfile, 'r') as fp:
            line = fp.readlines()[0]
            return line

    try:
        data, hdr = check_image(f)
    except Exception as e:
        log.error(f"Problem checking {f}: {e}")
        return None

    wfskey = hdr['WFSKEY']
    obstime = hdr['OBS-TIME']
    airmass = hdr['AIRMASS']
    exptime = hdr['EXPTIME']
    az = hdr.get('AZ', np.nan)
    el = hdr.get('EL', np.nan)
    tiltx = hdr.get('TILTX', np.nan)
    tilty = hdr.get('TILTY', np.nan)
    transx = hdr.get('TRANSX', np.nan)
    transy = hdr.get('TRANSY', np.nan)
    focus = hdr.get('FOCUS', np.nan)
    if np.isnan(focus) and 'TRANSZ' in hdr:
        focus = hdr.get('TRANSZ', np.nan)
    osst = hdr.get('OSSTEMP', np.nan)
    if 'OUT_T' in hdr:
        outt = hdr.get('OUT_T', np.nan)
    else:
        outt = hdr.get('T_OUT', np.nan)
    if 'CHAM_T' in hdr:
        chamt = hdr.get('CHAM_T', np.nan)
    else:
        chamt = hdr.get('T_CHAM', np.nan)

    # being conservative here and only using data that has proper slope determination
    # and wavefront solution. also want to get statistics on the quality of the wavefront fits.
    try:
        results = wfs_systems[wfskey].measure_slopes(str(f), plot=False)
    except:
        log.error(f"Problem analyzing {f.name}...")
        results = {}
        results['slopes'] = None

    if results['slopes'] is not None:
        try:
            zresults = wfs_systems[wfskey].fit_wavefront(results, plot=False)
            zv = zresults['zernike']
            focerr = wfs_systems[wfskey].calculate_focus(zv)
            cc_x_err, cc_y_err = wfs_systems[wfskey].calculate_cc(zv)
            line = f"{obstime},{wfskey},{f.name},{exptime},{airmass},{az},{el},{osst},{outt},{chamt},{tiltx},{tilty},{transx},{transy},{focus},{focerr.value},{cc_x_err.value},{cc_y_err.value},{results['xcen']},{results['ycen']},{results['seeing'].value},{results['raw_seeing'].value},{results['fwhm']},{zresults['zernike_rms'].value},{zresults['residual_rms'].value}\n"
            zfile = f.parent / (f.stem + ".reanalyze.zernike")
            zresults['zernike'].save(filename=zfile)
            spotfile = f.parent / (f.stem + ".spots.csv")
            results['spots'].write(spotfile, overwrite=True)
            with open(outfile, 'w') as fp:
                fp.write(line)
            return line
        except Exception as e:
            log.error(f"Problem fitting wavefront for {f.name}: {e}")
            return None
    else:
        return None

def main():
    """
    Take directories as argument and go through each one to process files in parallel using Pool.map()
    """
    parser = argparse.ArgumentParser(description='Utility for parallelized batch reprocessing of WFS data')

    parser.add_argument(
        '-r', '--rootdir',
        metavar="<root dir>",
        help="Directory containing WFS data. Defaults to current working directory.",
        default="."
    )

    parser.add_argument(
        '-d', '--dirs',
        metavar="<glob>",
        help="Glob of directories to process. Defaults to *.",
        default="*"
    )

    parser.add_argument(
        '--forcedir',
        help="Force rebuild of CSV for a directory",
        action='store_true'
    )

    parser.add_argument(
        '--force',
        help="Force reanalysis of individual data files",
        action="store_true"
    )

    parser.add_argument(
        '-n', '--nproc',
        metavar="<# processes>",
        help="Number of parallel processes. Defaults to half number of available cores.",
        default=int(multiprocessing.cpu_count()/2)  # MKL uses a lot of threads so best to limit Pool to half available cores
    )

    args = parser.parse_args()

    rootdir = Path(args.rootdir)

    log.info(f"Using {args.nproc} cores...")

    dirs = sorted(list(rootdir.glob(args.dirs)))  # pathlib, where have you been all my life!
    csv_header = "time,wfs,file,exptime,airmass,az,el,osst,outt,chamt,tiltx,tilty,transx,transy,focus,focerr,cc_x_err,cc_y_err,xcen,ycen,seeing,raw_seeing,fwhm,wavefront_rms,residual_rms\n"
    slow = False
    for d in dirs:
        if d.is_dir():
            if not args.forcedir and Path.exists(d / "reanalyze_results.csv"):
                log.info(f"Already processed {d.name}...")
            else:
                try:
                    lines = []
                    lines.append(csv_header)
                    night = int(d.name)  # valid WFS directories are ints of the form YYYYMMDD. if not this form, int barfs
                    fitsfiles = d.glob("*.fits")
                    log.info(f"Processing {d}...")
                    if slow:
                        plines = []
                        for f in fitsfiles:
                            log.debug(f"Processing {f}...")
                            l = process_image(f, force=args.force)
                            plines.append(l)
                    else:
                        with Pool(processes=args.nproc) as pool:
                            plines = pool.map(process_image, fitsfiles)  # plines comes out in same order as fitslines!

                    plines = list(filter(None.__ne__, plines))  # trim out any None entries
                    if len(plines) > 0:
                        lines.extend(plines)
                        with open(d / "reanalyze_results.csv", "w") as f:
                            f.writelines(lines)

                except ValueError as e:  # this means running int(d.name) failed so it's not a valid directory...
                    log.warn(f"Skipping %s... ({e})" % d.name)
<|endoftext|>


<commit_before>#! /usr/bin/env python

import os, sys

src = os.path.basename(sys.argv[0])
print src

xxxxxx

tar = src + ".tar"
gz = tar + ".gz"
tgz = src + ".tgz"

os.system('tar --exclude="CVS|mktarball.py" -cvf %s %s' % (tar, src))
os.system('gzip -9v %s' % tar)
os.rename(gz, tgz)
<commit_msg>working version<commit_after>#! /usr/bin/env python

import os, sys

src, self = os.path.split(sys.argv[0])
tar = src + ".tar"
gz = tar + ".gz"
tgz = src + ".tgz"

print "source:", src
print "dest:", tgz

os.system('tar --exclude=CVS --exclude=%s -cf %s %s' % (self, tar, src))
os.system('gzip -9v %s' % tar)
os.rename(gz, tgz)
<|endoftext|>


<commit_before>import unittest

from pymel import *
from pymel.tools.pymelControlPanel import getApiClassName, getClassHierarchy

def getFundamentalTypes():
    classList = sorted( list( set( [ key[0] for key in api.apiToMelData.keys()] ) ) )
    leaves = [ util.capitalize(x.key) for x in factories.nodeHierarchy.leaves() ]
    return sorted( set(classList).intersection(leaves) )

EXCEPTIONS = ['MotionPath','OldBlindDataBase', 'TextureToGeom']

class testCase_attribs(unittest.TestCase):
    def setUp(self):
        self.sphere1, hist = polySphere()

        class AttributeData(object):
            node = self.sphere1

            def __init__(self, name, **initArgs):
                self.name = name
                self.initArgs = initArgs

            def add(self):
                addAttr(self.node, longName=self.name, **self.initArgs)

        self.newAttrs = [
                        AttributeData('multiByte', multi=True, attributeType='byte'),
                        AttributeData('compound', attributeType='compound', numberOfChildren=3),
                        AttributeData('compound_multiFloat', attributeType='float', multi=True, parent='compound'),
                        AttributeData('compound_double', attributeType='double', parent='compound'),
                        AttributeData('compound_compound', attributeType='compound', numberOfChildren=2, parent='compound'),
                        AttributeData('compound_compound_matrix', attributeType='matrix', parent='compound_compound'),
                        AttributeData('compound_compound_long', attributeType='long', parent='compound_compound'),
                        ]

        for attr in self.newAttrs:
            attr.add()

        self.newAttrs = dict([(newAttr.name, Attribute(self.sphere1 + "." + newAttr.name)) for newAttr in self.newAttrs ])

        self.setMultiElement = self.newAttrs['multiByte'][1]
        self.setMultiElement.set(1)

        self.unsetMultiElement = self.newAttrs['multiByte'][3]

    def tearDown(self):
        delete(self.sphere1)

    def test_newAttrsExists(self):
        for attr in self.newAttrs.itervalues():
#            print "Testing existence of:", attr.name()
            self.assertTrue(attr.exists())

    def test_setMultiElementExists(self):
        self.assertTrue(self.setMultiElement.exists())

    def test_unsetMultiElementExists(self):
        self.assertFalse(self.unsetMultiElement.exists())

    def test_getParent(self):
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(), self.newAttrs['compound_compound'])

        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(0), self.newAttrs['compound_compound_long'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=1), self.newAttrs['compound_compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(2), self.newAttrs['compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=3), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-1), self.newAttrs['compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=-2), self.newAttrs['compound_compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-3), self.newAttrs['compound_compound_long'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=-4), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-5), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=4), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-63), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=32), None)

def testInvertibles():
    classList = getFundamentalTypes()
    print classList
    for pynodeName in classList:
        try:
            pynode = getattr( core.nodetypes, pynodeName )
        except AttributeError:
            print "could not find", pynodeName
            continue

        if not issubclass( pynode, PyNode ) or pynodeName in EXCEPTIONS:
            continue

        if issubclass(pynode, GeometryShape):
            if pynode == Mesh :
                obj = polyCube()[0].getShape()
                obj.createColorSet( 'thingie' )
            elif pynode == Subdiv:
                obj = polyToSubdiv( polyCube()[0].getShape())[0].getShape()
            elif pynode == NurbsSurface:
                obj = sphere()[0].getShape()
            elif pynode == NurbsCurve:
                obj = circle()[0].getShape()
            else:
                print "skipping shape", pynode
                continue
        else:
            obj = createNode( util.uncapitalize(pynodeName) )

        print repr(obj)

        for className, apiClassName in getClassHierarchy(pynodeName):

            if apiClassName not in api.apiClassInfo:
                continue

            #print className, apiClassName

            classInfo = api.apiClassInfo[apiClassName]
            invertibles = classInfo['invertibles']
            #print invertibles

            for setMethod, getMethod in invertibles:
                info = classInfo['methods'][setMethod]
                try:
                    setMethod = info[0]['pymelName']
                except KeyError: pass

                setMethod, data = factories._getApiOverrideNameAndData( className, setMethod )
                try:
                    overloadIndex = data['overloadIndex']
                    info = info[overloadIndex]
                except (KeyError, TypeError): pass
                else:
                    try:
                        setter = getattr( pynode, setMethod )
                    except AttributeError: pass
                    else:
                        def getType(type):
                            typeMap = {   'bool' : True,
                                'double' : 2.5, # min required for setFocalLength
                                'double3' : ( 1.0, 2.0, 3.0),
                                'MEulerRotation' : ( 1.0, 2.0, 3.0),
                                'float': 2.5,
                                'MFloatArray': [1.1, 2.2, 3.3],
                                'MString': 'thingie',
                                'float2': (.1, .2),
                                'MPoint' : [1,2,3],
                                'short': 1,
                                'MColor' : [1,0,0],
                                'MColorArray': ( [1.0,0.0,0.0], [0.0,1.0,0.0], [0.0,0.0,1.0] ),
                                'MVector' : [1,0,0],
                                'MVectorArray': ( [1.0,0.0,0.0], [0.0,1.0,0.0], [0.0,0.0,1.0] ),
                                'int' : 1,
                                'MIntArray': [1,2,3],
                                'MSpace.Space' : 'world'
                            }
                            if '.' in type:
                                return 1 # take a dumb guess at an enum
                            else:
                                return typeMap[type]

                        inArgs = [ arg for arg in info['inArgs'] if arg not in info['defaults'] ]
                        types = [ str(info['types'][arg]) for arg in inArgs ]
                        try:
                            if apiClassName == 'MFnMesh' and setMethod == 'setUVs':
                                args = [ [.1]*obj.numUVs(), [.2]*obj.numUVs() ]
                            elif apiClassName == 'MFnMesh' and setMethod == 'setColors':
                                args = [ [ [.5,.5,.5] ]*obj.numColors() ]
                            elif apiClassName == 'MFnNurbsCurve' and setMethod == 'setKnot':
                                args = [ 0, 2.5 ]
                            else:
                                args = [ getType(typ) for typ in types ]
                            descr =  '%s.%s(%s)' % ( pynodeName, setMethod, ', '.join( [ repr(x) for x in args] ) )
    #                            try:
    #                                setter( obj, *args )
    #                            except Exception, e:
    #                                print str(e)
                            args = [obj] + args
                            def checkSetter( setter, args ):
                                setter( *args )
                                if Version.current > Version.v85sp1:
                                    mel.undo()
                            checkSetter.description = descr
                            yield checkSetter, setter, args
                        except KeyError, msg:
                            print str(msg)
        try:
            delete( obj )
        except:
            pass

                    #print types
<commit_msg>improved nodetypes test<commit_after>import unittest

from pymel import *
from pymel.tools.pymelControlPanel import getApiClassName, getClassHierarchy

def getFundamentalTypes():
    classList = sorted( list( set( [ key[0] for key in api.apiToMelData.keys()] ) ) )
    leaves = [ util.capitalize(x.key) for x in factories.nodeHierarchy.leaves() ]
    return sorted( set(classList).intersection(leaves) )

EXCEPTIONS = ['MotionPath','OldBlindDataBase', 'TextureToGeom']

class testCase_attribs(unittest.TestCase):
    def setUp(self):
        self.sphere1, hist = polySphere()

        class AttributeData(object):
            node = self.sphere1

            def __init__(self, name, **initArgs):
                self.name = name
                self.initArgs = initArgs

            def add(self):
                addAttr(self.node, longName=self.name, **self.initArgs)

        self.newAttrs = [
                        AttributeData('multiByte', multi=True, attributeType='byte'),
                        AttributeData('compound', attributeType='compound', numberOfChildren=3),
                        AttributeData('compound_multiFloat', attributeType='float', multi=True, parent='compound'),
                        AttributeData('compound_double', attributeType='double', parent='compound'),
                        AttributeData('compound_compound', attributeType='compound', numberOfChildren=2, parent='compound'),
                        AttributeData('compound_compound_matrix', attributeType='matrix', parent='compound_compound'),
                        AttributeData('compound_compound_long', attributeType='long', parent='compound_compound'),
                        ]

        for attr in self.newAttrs:
            attr.add()

        self.newAttrs = dict([(newAttr.name, Attribute(self.sphere1 + "." + newAttr.name)) for newAttr in self.newAttrs ])

        self.setMultiElement = self.newAttrs['multiByte'][1]
        self.setMultiElement.set(1)

        self.unsetMultiElement = self.newAttrs['multiByte'][3]

    def tearDown(self):
        delete(self.sphere1)

    def test_newAttrsExists(self):
        for attr in self.newAttrs.itervalues():
#            print "Testing existence of:", attr.name()
            self.assertTrue(attr.exists())

    def test_setMultiElementExists(self):
        self.assertTrue(self.setMultiElement.exists())

    def test_unsetMultiElementExists(self):
        self.assertFalse(self.unsetMultiElement.exists())

    def test_getParent(self):
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(), self.newAttrs['compound_compound'])

        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(0), self.newAttrs['compound_compound_long'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=1), self.newAttrs['compound_compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(2), self.newAttrs['compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=3), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-1), self.newAttrs['compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=-2), self.newAttrs['compound_compound'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-3), self.newAttrs['compound_compound_long'])
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=-4), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-5), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=4), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(-63), None)
        self.assertEqual(self.newAttrs['compound_compound_long'].getParent(generations=32), None)

def testInvertibles():
    classList = getFundamentalTypes()
    print classList
    for pynodeName in classList:
        try:
            pynode = getattr( core.nodetypes, pynodeName )
        except AttributeError:
            print "could not find", pynodeName
            continue

        if not issubclass( pynode, PyNode ) or pynodeName in EXCEPTIONS:
            continue

        if issubclass(pynode, GeometryShape):
            if pynode == Mesh :
                obj = polyCube()[0].getShape()
                obj.createColorSet( 'thingie' )
            elif pynode == Subdiv:
                obj = polyToSubdiv( polyCube()[0].getShape())[0].getShape()
            elif pynode == NurbsSurface:
                obj = sphere()[0].getShape()
            elif pynode == NurbsCurve:
                obj = circle()[0].getShape()
            else:
                print "skipping shape", pynode
                continue
        else:
            obj = createNode( util.uncapitalize(pynodeName) )

        print repr(obj)

        for className, apiClassName in getClassHierarchy(pynodeName):

            if apiClassName not in api.apiClassInfo:
                continue

            #print className, apiClassName

            classInfo = api.apiClassInfo[apiClassName]
            invertibles = classInfo['invertibles']
            #print invertibles

            for setMethod, getMethod in invertibles:
                info = classInfo['methods'][setMethod]
                try:
                    setMethod = info[0]['pymelName']
                except KeyError: pass

                setMethod, data = factories._getApiOverrideNameAndData( className, setMethod )
                try:
                    overloadIndex = data['overloadIndex']
                    info = info[overloadIndex]
                except (KeyError, TypeError): pass
                else:
                    # test if this invertible has been broken in pymelControlPanel
                    if not info['inverse']:
                        continue

                    try:
                        setter = getattr( pynode, setMethod )
                    except AttributeError: pass
                    else:
                        def getType(type):
                            typeMap = {   'bool' : True,
                                'double' : 2.5, # min required for setFocalLength
                                'double3' : ( 1.0, 2.0, 3.0),
                                'MEulerRotation' : ( 1.0, 2.0, 3.0),
                                'float': 2.5,
                                'MFloatArray': [1.1, 2.2, 3.3],
                                'MString': 'thingie',
                                'float2': (.1, .2),
                                'MPoint' : [1,2,3],
                                'short': 1,
                                'MColor' : [1,0,0],
                                'MColorArray': ( [1.0,0.0,0.0], [0.0,1.0,0.0], [0.0,0.0,1.0] ),
                                'MVector' : [1,0,0],
                                'MVectorArray': ( [1.0,0.0,0.0], [0.0,1.0,0.0], [0.0,0.0,1.0] ),
                                'int' : 1,
                                'MIntArray': [1,2,3],
                                'MSpace.Space' : 'world'
                            }
                            if '.' in type:
                                return 1 # take a dumb guess at an enum
                            else:
                                return typeMap[type]

                        inArgs = [ arg for arg in info['inArgs'] if arg not in info['defaults'] ]
                        types = [ str(info['types'][arg]) for arg in inArgs ]
                        try:
                            if apiClassName == 'MFnMesh' and setMethod == 'setUVs':
                                args = [ [.1]*obj.numUVs(), [.2]*obj.numUVs() ]
                            elif apiClassName == 'MFnMesh' and setMethod == 'setColors':
                                args = [ [ [.5,.5,.5] ]*obj.numColors() ]
                            elif apiClassName == 'MFnNurbsCurve' and setMethod == 'setKnot':
                                args = [ 6, 4.5 ]
                            else:
                                args = [ getType(typ) for typ in types ]
                            descr =  '%s.%s(%s)' % ( pynodeName, setMethod, ', '.join( [ repr(x) for x in args] ) )
    #                            try:
    #                                setter( obj, *args )
    #                            except Exception, e:
    #                                print str(e)
                            args = [obj] + args
                            def checkSetter( setter, args ):
                                setter( *args )
                                if Version.current > Version.v85sp1:
                                    mel.undo()
                            checkSetter.description = descr
                            yield checkSetter, setter, args
                        except KeyError, msg:
                            print str(msg)
        try:
            delete( obj )
        except:
            pass

                    #print types
<|endoftext|>


<commit_before>#
# Copyright (c) 2014-2015 Christopher L. Felton
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import

from myhdl import *
from mn.system import Clock
from mn.system import Reset
from mn.system.memmap._memmap import MemMap

class AvalonMM(MemMap):
    name = 'avalon'

    def __init__(self, glbl=None, data_width=8, address_width=16,
                 name=None):
        """
        Parameters (kwargs):
        --------------------
        :param glbl: system clock and reset
        :param data_width: data bus width
        :param address_width: address bus width
        :param name: name for the bus
        """
        super(AvalonMM, self).__init__(data_width=data_width,
                                       address_width=address_width)
        if glbl is None:
            self.clk = Clock(0)
        else:
            self.clk = glbl.clock

        if glbl.reset is None:
            self.reset = Reset(0, active=1, async=False)
        else:
            self.reset = glbl.reset

        self.address = Signal(intbv(0)[address_width:])
        self.byteenable = Signal(intbv(0)[2:])
        self.read = Signal(bool(0))
        self.write = Signal(bool(0))
        self.waitrequest = Signal(bool(0))
        self.readdatavalid = Signal(bool(0))
        self.readdata = Signal(intbv(0)[data_width:])
        self.writedata = Signal(intbv(0)[data_width:])
        self.response = Signal(intbv(0)[2:])

    def add_output_bus(self, name, readdata, readdatavalid, waitrequest):
        self._readdata.append(readdata)
        self._readatavalid.append(readatavalid)
        self._waitrequest.append(waitrequest)

    def m_per_outputs(self):
        """ combine all the peripheral outputs
        """
        assert len(self._readata) == len(self._readatavalid)
        ndevs = len(self._readdata)

        av = self

        @always_seq(self.clk.posedge, reset=self.reset)
        def rtl_or_combine():
            rddats = 0
            valids = 0
            for ii in range(ndevs):
                rddats = rddats | av._readdata[ii]
                valids = valids | av._readdatavalid[ii]
            av.readdata.next = rddats
            av.readdatavalid = valids

        return rtl_or_combine

    def m_per_interface(self, glbl, ):<commit_msg>removed unused file<commit_after><|endoftext|>