<commit_before># This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

# Author: Anne Mulhern <mulhern@cs.wisc.edu>

""" Configuration of the justbytes package. """

from fractions import Fraction

from ._approx_display import ApproxPrefix

class BaseConfig(object):
    """
    Whether and how to show the base.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "use_prefix=%(use_prefix)s",
       "use_subscript=%(use_subscript)s"
    ])

    def __init__(self, use_prefix=False, use_subscript=True):
        """
        Initializer.

        :param bool use_prefix: show base using prefix, e.g., 0x, 0
        :param bool use_subscript: show base using subscript
        """
        self.use_prefix = use_prefix
        self.use_subscript = use_subscript

    def __str__(self): # pragma: no cover
        values = {
           'use_prefix' : self.use_prefix,
           'use_subscript' : self.use_subscript
        }
        return "BaseConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class StripConfig(object):
    """
    Stripping trailing zeros.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "strip=%(strip)s",
       "strip_exact=%(strip_exact)s",
       "strip_whole=%(strip_whole)s"
    ])

    def __init__(self, strip=False, strip_exact=False, strip_whole=True):
        """
        Initializer.

        :param bool strip: strip all trailing zeros
        :param bool strip_exact: strip if value is exact
        :param bool strip_whole: strip if value is exact and non-fractional

        strip is stronger than strip_exact which is stronger than strip_whole
        """
        self.strip = strip
        self.strip_exact = strip_exact
        self.strip_whole = strip_whole

    def __str__(self): # pragma: no cover
        values = {
           'strip' : self.strip,
           'strip_exact' : self.strip_exact,
           'strip_whole' : self.strip_whole
        }
        return "StripConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class DigitsConfig(object):
    """
    How to display digits.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "separator=%(separator)s",
       "use_caps=%(use_caps)s",
       "use_letters=%(use_letters)s"
    ])

    def __init__(
       self,
       separator='~',
       use_caps=False,
       use_letters=True
    ):
        """
        Initializer.

        :param str separator: separate for digits
        :param bool use_caps: if set, use capital letters
        :param bool use_letters: if set, use letters

        If digits in this base require more than one character.
        """
        self.separator = separator
        self.use_caps = use_caps
        self.use_letters = use_letters

    def __str__(self): # pragma: no cover
        values = {
           'separator' : self.separator,
           'use_caps' : self.use_caps,
           'use_letters' : self.use_letters
        }
        return "DigitsConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class ApproxConfig(object):
    """
    Whether and how to show approximation information.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "xform=%(xform)s"
    ])

    def __init__(
       self,
       xform=ApproxPrefix([Fraction(1, 2)]).xform
    ):
        """
        Initializer.

        :param xform: a transformer for a number, to show approximation info
        :type xform: (Rational * str) -> str
        """
        self.xform = xform

    def __str__(self): # pragma: no cover
        values = {
           'xform' : self.xform
        }
        return "ApproxConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class DisplayConfig(object):
    """
    Superficial aspects of display.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "approx_config=%(approx_config)s",
       "base_config=%(base_config)s",
       "digits_config=%(digits_config)s",
       "strip_config-%(strip_config)s"
    ])

    def __init__(
       self,
       approx_config=ApproxConfig(),
       base_config=BaseConfig(),
       digits_config=DigitsConfig(),
       strip_config=StripConfig()
    ):
        """
        Initializer.

        :param ApproxConfig approx_config: how to show approximation information
        :param BaseConfig base_config: how to show base
        :param DigitsConfig digits_config: how to display digits
        :param StripConfig strip_config: how to strip zeros

        There are only two base prefixes acknowledged, 0 for octal and 0x for
        hexadecimal.
        """
        self.approx_config = approx_config
        self.base_config = base_config
        self.digits_config = digits_config
        self.strip_config = strip_config

    def __str__(self): # pragma: no cover
        values = {
           'approx_config' : self.approx_config,
           'base_config' : self.base_config,
           'digits_config' : self.digits_config,
           'strip_config' : self.strip_config
        }
        return "DisplayConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class BasesConfig(object):
    """
    Configuration class for any bases things.
    """
    # pylint: disable=too-few-public-methods

    DISPLAY_CONFIG = DisplayConfig()

    @classmethod
    def set_display_config(cls, config): # pragma: no cover
        """
        Set configuration for superficial aspects of display.

        :param DisplayConfig config: a configuration object
        """
        cls.DISPLAY_CONFIG = DisplayConfig(
            approx_config=config.approx_config,
            base_config=config.base_config,
            digits_config=config.digits_config,
            strip_config=config.strip_config
        )
<commit_msg>Revert "Do not use a justbytes specific approximation display method."<commit_after># This copyrighted material is made available to anyone wishing to use,
# modify, copy, or redistribute it subject to the terms and conditions of
# the GNU General Public License v.2, or (at your option) any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY expressed or implied, including the implied warranties of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.  You should have received a copy of the
# GNU General Public License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

# Author: Anne Mulhern <mulhern@cs.wisc.edu>

""" Configuration of the justbytes package. """

from fractions import Fraction

from ._approx_display import ApproxPrefix

class BaseConfig(object):
    """
    Whether and how to show the base.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "use_prefix=%(use_prefix)s",
       "use_subscript=%(use_subscript)s"
    ])

    def __init__(self, use_prefix=False, use_subscript=True):
        """
        Initializer.

        :param bool use_prefix: show base using prefix, e.g., 0x, 0
        :param bool use_subscript: show base using subscript
        """
        self.use_prefix = use_prefix
        self.use_subscript = use_subscript

    def __str__(self): # pragma: no cover
        values = {
           'use_prefix' : self.use_prefix,
           'use_subscript' : self.use_subscript
        }
        return "BaseConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class StripConfig(object):
    """
    Stripping trailing zeros.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "strip=%(strip)s",
       "strip_exact=%(strip_exact)s",
       "strip_whole=%(strip_whole)s"
    ])

    def __init__(self, strip=False, strip_exact=False, strip_whole=True):
        """
        Initializer.

        :param bool strip: strip all trailing zeros
        :param bool strip_exact: strip if value is exact
        :param bool strip_whole: strip if value is exact and non-fractional

        strip is stronger than strip_exact which is stronger than strip_whole
        """
        self.strip = strip
        self.strip_exact = strip_exact
        self.strip_whole = strip_whole

    def __str__(self): # pragma: no cover
        values = {
           'strip' : self.strip,
           'strip_exact' : self.strip_exact,
           'strip_whole' : self.strip_whole
        }
        return "StripConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class DigitsConfig(object):
    """
    How to display digits.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "separator=%(separator)s",
       "use_caps=%(use_caps)s",
       "use_letters=%(use_letters)s"
    ])

    def __init__(
       self,
       separator='~',
       use_caps=False,
       use_letters=True
    ):
        """
        Initializer.

        :param str separator: separate for digits
        :param bool use_caps: if set, use capital letters
        :param bool use_letters: if set, use letters

        If digits in this base require more than one character.
        """
        self.separator = separator
        self.use_caps = use_caps
        self.use_letters = use_letters

    def __str__(self): # pragma: no cover
        values = {
           'separator' : self.separator,
           'use_caps' : self.use_caps,
           'use_letters' : self.use_letters
        }
        return "DigitsConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class ApproxConfig(object):
    """
    Whether and how to show approximation information.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "xform=%(xform)s"
    ])

    def __init__(
       self,
       xform=ApproxPrefix([Fraction(1, 1024), Fraction(1, 2)]).xform
    ):
        """
        Initializer.

        :param xform: a transformer for a number, to show approximation info
        :type xform: (Rational * str) -> str
        """
        self.xform = xform

    def __str__(self): # pragma: no cover
        values = {
           'xform' : self.xform
        }
        return "ApproxConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class DisplayConfig(object):
    """
    Superficial aspects of display.
    """
    # pylint: disable=too-few-public-methods

    _FMT_STR = ", ".join([
       "approx_config=%(approx_config)s",
       "base_config=%(base_config)s",
       "digits_config=%(digits_config)s",
       "strip_config-%(strip_config)s"
    ])

    def __init__(
       self,
       approx_config=ApproxConfig(),
       base_config=BaseConfig(),
       digits_config=DigitsConfig(),
       strip_config=StripConfig()
    ):
        """
        Initializer.

        :param ApproxConfig approx_config: how to show approximation information
        :param BaseConfig base_config: how to show base
        :param DigitsConfig digits_config: how to display digits
        :param StripConfig strip_config: how to strip zeros

        There are only two base prefixes acknowledged, 0 for octal and 0x for
        hexadecimal.
        """
        self.approx_config = approx_config
        self.base_config = base_config
        self.digits_config = digits_config
        self.strip_config = strip_config

    def __str__(self): # pragma: no cover
        values = {
           'approx_config' : self.approx_config,
           'base_config' : self.base_config,
           'digits_config' : self.digits_config,
           'strip_config' : self.strip_config
        }
        return "DisplayConfig(%s)" % (self._FMT_STR % values)
    __repr__ = __str__

class BasesConfig(object):
    """
    Configuration class for any bases things.
    """
    # pylint: disable=too-few-public-methods

    DISPLAY_CONFIG = DisplayConfig()

    @classmethod
    def set_display_config(cls, config): # pragma: no cover
        """
        Set configuration for superficial aspects of display.

        :param DisplayConfig config: a configuration object
        """
        cls.DISPLAY_CONFIG = DisplayConfig(
            approx_config=config.approx_config,
            base_config=config.base_config,
            digits_config=config.digits_config,
            strip_config=config.strip_config
        )
<|endoftext|>


<commit_before># Test consistency between galpy and NEMO
from __future__ import print_function, division
import os
import numpy
import subprocess
from galpy.orbit import Orbit
from galpy import potential
from galpy.util import bovy_conversion
def test_nemo_MN3ExponentialDiskPotential():
    mn= potential.MN3ExponentialDiskPotential(normalize=1.,hr=0.5,hz=0.1)
    tmax= 3.
    vo,ro= 215., 8.75
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mn,tmax,vo,ro)
    return None

def test_nemo_MiyamotoNagaiPotential():
    mp= potential.MiyamotoNagaiPotential(normalize=1.,a=0.5,b=0.1)
    tmax= 4.
    vo,ro= 220., 8.
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mp,tmax,vo,ro)
    return None

def test_nemo_NFWPotential():
    np= potential.NFWPotential(normalize=1.,a=3.)
    tmax= 3.
    vo,ro= 200., 7.
    o= Orbit([1.,0.5,1.3,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,np,tmax,vo,ro)
    return None

def test_nemo_PowerSphericalPotentialwCutoffPotential():
    pp= potential.PowerSphericalPotentialwCutoff(normalize=1.,alpha=1.,rc=0.4)
    tmax= 2.
    vo,ro= 180., 9.
    o= Orbit([1.,0.03,1.03,0.2,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,pp,tmax,vo,ro)
    return None

def test_nemo_LogarithmicHaloPotential():
    lp= potential.LogarithmicHaloPotential(normalize=1.)
    tmax= 2.
    vo,ro= 210., 8.5
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,lp,tmax,vo,ro,tol=0.03)
    return None

def test_nemo_MWPotential2014():
    mp= potential.MWPotential2014
    tmax= 4.
    vo,ro= 220., 8.
    o= Orbit([1.,0.1,1.1,0.2,0.1,1.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mp,tmax,vo,ro,isList=True)
    return None

def run_orbitIntegration_comparison(orb,pot,tmax,vo,ro,isList=False,
                                    tol=0.01):
    # Integrate in galpy
    ts= numpy.linspace(0.,tmax/bovy_conversion.time_in_Gyr(vo,ro),1001)
    orb.integrate(ts,pot)
    # Now setup a NEMO snapshot in the correct units ([x] = kpc, [v] = kpc/Gyr)
    numpy.savetxt('orb.dat',
                  numpy.array([[10.**-6.,orb.x(),orb.y(),orb.z(),
                                orb.vx(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro),
                                orb.vy(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro),
                                orb.vz(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro)]]))
    # Now convert to NEMO format
    try:
        convert_to_nemo('orb.dat','orb.nemo')
    finally:
        os.remove('orb.dat')
    # Integrate with gyrfalcON
    try:
        if isList:
            integrate_gyrfalcon('orb.nemo','orb_evol.nemo',tmax,
                                potential.nemo_accname(pot),
                                potential.nemo_accpars(pot,vo,ro))
        else:
            integrate_gyrfalcon('orb.nemo','orb_evol.nemo',tmax,
                                pot.nemo_accname(),pot.nemo_accpars(vo,ro))
    finally:
        os.remove('orb.nemo')
        os.remove('gyrfalcON.log')
    # Convert back to ascii
    try:
        convert_from_nemo('orb_evol.nemo','orb_evol.dat')
    finally:
        os.remove('orb_evol.nemo')
    # Read and compare
    try:
        nemodata= numpy.loadtxt('orb_evol.dat',comments='#')
        xdiff= numpy.fabs((nemodata[-1,1]-orb.x(ts[-1]))/nemodata[-1,1])
        ydiff= numpy.fabs((nemodata[-1,2]-orb.y(ts[-1]))/nemodata[-1,2])
        zdiff= numpy.fabs((nemodata[-1,3]-orb.z(ts[-1]))/nemodata[-1,3])
        vxdiff= numpy.fabs((nemodata[-1,4]-orb.vx(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,4])
        vydiff= numpy.fabs((nemodata[-1,5]-orb.vy(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,5])
        vzdiff= numpy.fabs((nemodata[-1,6]-orb.vz(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,6])
        assert xdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for x by %g' % xdiff
        assert ydiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for y by %g' % ydiff
        assert zdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for z by %g' % zdiff
        assert vxdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vx by %g' % vxdiff
        assert vydiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vy by %g' % vydiff
        assert vzdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vz by %g' % vzdiff
    finally:
        os.remove('orb_evol.dat')
    return None

def convert_to_nemo(infile,outfile):
    subprocess.check_call(['a2s','in=%s'% infile,'out=%s' % outfile,'N=1',
                           'read=mxv'])

def convert_from_nemo(infile,outfile):
    subprocess.check_call(['s2a','in=%s' % infile,'out=%s' % outfile])

def integrate_gyrfalcon(infile,outfile,tmax,nemo_accname,nemo_accpars):
    """Integrate a snapshot in infile until tmax in Gyr, save to outfile"""
    with open('gyrfalcON.log','w') as f:
        subprocess.check_call(['gyrfalcON',
                               'in=%s' % infile,
                               'out=%s' % outfile,
                               'tstop=%g' % tmax,
                               'eps=0.0015',
                               'step=0.01',
                               'kmax=10',
                               'Nlev=8',
                               'fac=0.01',
                               'accname=%s' % nemo_accname,
                               'accpars=%s' % nemo_accpars],
                              stdout=f)
    return None
<commit_msg>add NEMO test of Plummer<commit_after># Test consistency between galpy and NEMO
from __future__ import print_function, division
import os
import numpy
import subprocess
from galpy.orbit import Orbit
from galpy import potential
from galpy.util import bovy_conversion
def test_nemo_MN3ExponentialDiskPotential():
    mn= potential.MN3ExponentialDiskPotential(normalize=1.,hr=0.5,hz=0.1)
    tmax= 3.
    vo,ro= 215., 8.75
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mn,tmax,vo,ro)
    return None

def test_nemo_MiyamotoNagaiPotential():
    mp= potential.MiyamotoNagaiPotential(normalize=1.,a=0.5,b=0.1)
    tmax= 4.
    vo,ro= 220., 8.
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mp,tmax,vo,ro)
    return None

def test_nemo_NFWPotential():
    np= potential.NFWPotential(normalize=1.,a=3.)
    tmax= 3.
    vo,ro= 200., 7.
    o= Orbit([1.,0.5,1.3,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,np,tmax,vo,ro)
    return None

def test_nemo_PowerSphericalPotentialwCutoffPotential():
    pp= potential.PowerSphericalPotentialwCutoff(normalize=1.,alpha=1.,rc=0.4)
    tmax= 2.
    vo,ro= 180., 9.
    o= Orbit([1.,0.03,1.03,0.2,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,pp,tmax,vo,ro)
    return None

def test_nemo_LogarithmicHaloPotential():
    lp= potential.LogarithmicHaloPotential(normalize=1.)
    tmax= 2.
    vo,ro= 210., 8.5
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,lp,tmax,vo,ro,tol=0.03)
    return None

def test_nemo_PlummerPotential():
    pp= potential.PlummerPotential(normalize=1.,b=2.)
    tmax= 3.
    vo,ro= 213., 8.23
    o= Orbit([1.,0.1,1.1,0.3,0.1,0.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,pp,tmax,vo,ro,tol=0.03)
    return None

def test_nemo_MWPotential2014():
    mp= potential.MWPotential2014
    tmax= 4.
    vo,ro= 220., 8.
    o= Orbit([1.,0.1,1.1,0.2,0.1,1.4],ro=ro,vo=vo)
    run_orbitIntegration_comparison(o,mp,tmax,vo,ro,isList=True)
    return None

def run_orbitIntegration_comparison(orb,pot,tmax,vo,ro,isList=False,
                                    tol=0.01):
    # Integrate in galpy
    ts= numpy.linspace(0.,tmax/bovy_conversion.time_in_Gyr(vo,ro),1001)
    orb.integrate(ts,pot)
    # Now setup a NEMO snapshot in the correct units ([x] = kpc, [v] = kpc/Gyr)
    numpy.savetxt('orb.dat',
                  numpy.array([[10.**-6.,orb.x(),orb.y(),orb.z(),
                                orb.vx(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro),
                                orb.vy(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro),
                                orb.vz(use_physical=False)\
                                    *bovy_conversion.velocity_in_kpcGyr(vo,ro)]]))
    # Now convert to NEMO format
    try:
        convert_to_nemo('orb.dat','orb.nemo')
    finally:
        os.remove('orb.dat')
    # Integrate with gyrfalcON
    try:
        if isList:
            integrate_gyrfalcon('orb.nemo','orb_evol.nemo',tmax,
                                potential.nemo_accname(pot),
                                potential.nemo_accpars(pot,vo,ro))
        else:
            integrate_gyrfalcon('orb.nemo','orb_evol.nemo',tmax,
                                pot.nemo_accname(),pot.nemo_accpars(vo,ro))
    finally:
        os.remove('orb.nemo')
        os.remove('gyrfalcON.log')
    # Convert back to ascii
    try:
        convert_from_nemo('orb_evol.nemo','orb_evol.dat')
    finally:
        os.remove('orb_evol.nemo')
    # Read and compare
    try:
        nemodata= numpy.loadtxt('orb_evol.dat',comments='#')
        xdiff= numpy.fabs((nemodata[-1,1]-orb.x(ts[-1]))/nemodata[-1,1])
        ydiff= numpy.fabs((nemodata[-1,2]-orb.y(ts[-1]))/nemodata[-1,2])
        zdiff= numpy.fabs((nemodata[-1,3]-orb.z(ts[-1]))/nemodata[-1,3])
        vxdiff= numpy.fabs((nemodata[-1,4]-orb.vx(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,4])
        vydiff= numpy.fabs((nemodata[-1,5]-orb.vy(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,5])
        vzdiff= numpy.fabs((nemodata[-1,6]-orb.vz(ts[-1],use_physical=False)*bovy_conversion.velocity_in_kpcGyr(vo,ro))/nemodata[-1,6])
        assert xdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for x by %g' % xdiff
        assert ydiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for y by %g' % ydiff
        assert zdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for z by %g' % zdiff
        assert vxdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vx by %g' % vxdiff
        assert vydiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vy by %g' % vydiff
        assert vzdiff < tol, 'galpy and NEMO gyrfalcON orbit integration inconsistent for vz by %g' % vzdiff
    finally:
        os.remove('orb_evol.dat')
    return None

def convert_to_nemo(infile,outfile):
    subprocess.check_call(['a2s','in=%s'% infile,'out=%s' % outfile,'N=1',
                           'read=mxv'])

def convert_from_nemo(infile,outfile):
    subprocess.check_call(['s2a','in=%s' % infile,'out=%s' % outfile])

def integrate_gyrfalcon(infile,outfile,tmax,nemo_accname,nemo_accpars):
    """Integrate a snapshot in infile until tmax in Gyr, save to outfile"""
    with open('gyrfalcON.log','w') as f:
        subprocess.check_call(['gyrfalcON',
                               'in=%s' % infile,
                               'out=%s' % outfile,
                               'tstop=%g' % tmax,
                               'eps=0.0015',
                               'step=0.01',
                               'kmax=10',
                               'Nlev=8',
                               'fac=0.01',
                               'accname=%s' % nemo_accname,
                               'accpars=%s' % nemo_accpars],
                              stdout=f)
    return None
<|endoftext|>


<commit_before>"""
nose plugin for XTraceback

This module is placed outside of the xtraceback package because we don't want
nose to import the xtraceback package until it has started the coverage plugin.
"""

import sys

try:
    from nose.plugins import Plugin
except ImportError:
    Plugin = object

class NoseXTraceback(Plugin):
    """Extended traceback for error output."""

    name = "xtraceback"

    # must be before capture otherwise we don't see the real sys.stdout
    score = 600

    _options = (
        ("globals", "Include globals in tracebacks [%s]", "store_true"),
        ("globals_include", "Include only globals in this namespace [%s]", "store"),
        ("color", "Show color tracebacks - one of on,off,auto (default=auto) [%s]", "store"),
        )

    def options(self, parser, env):
        super(NoseXTraceback, self).options(parser, env)
        for name, help, action in self._options:
            env_opt = "NOSE_XTRACEBACK_%s" % name.upper()
            parser.add_option("--xtraceback-%s" % name.replace("_", "-"),
                              action=action,
                              dest="xtraceback_%s" % name,
                              default=env.get(env_opt),
                              help=help % env_opt)

    def configure(self, options, conf):
        super(NoseXTraceback, self).configure(options, conf)
        for name, help, dummy in self._options:
            name = "xtraceback_%s" % name
            setattr(self, name, getattr(options, name))
        if self.xtraceback_color is not None:
            if self.xtraceback_color == "on":
                self.xtraceback_color = True
            else:
                self.xtraceback_color = False

    def begin(self):
        # not importing in global scope because it messes with the coverage analysis
        import xtraceback
        options = dict(color=self.xtraceback_color,
                       stream=self.conf.stream,
                       show_globals=self.xtraceback_globals,
                       globals_module_include=self.xtraceback_globals_include)
        xtraceback.compat.defaults.update(**options)
        xtraceback.compat.install()

    def finalize(self, result):
        import xtraceback
        xtraceback.compat.uninstall()
<commit_msg>tidy imports<commit_after>"""
nose plugin for XTraceback

This module is placed outside of the xtraceback package because we don't want
nose to import the xtraceback package until it has started the coverage plugin.
"""

try:
    from nose.plugins import Plugin
except ImportError:
    Plugin = object

class NoseXTraceback(Plugin):
    """Extended traceback for error output."""

    name = "xtraceback"

    # must be before capture otherwise we don't see the real sys.stdout
    score = 600

    _options = (
        ("globals", "Include globals in tracebacks [%s]", "store_true"),
        ("globals_include", "Include only globals in this namespace [%s]", "store"),
        ("color", "Show color tracebacks - one of on,off,auto (default=auto) [%s]", "store"),
        )

    def options(self, parser, env):
        super(NoseXTraceback, self).options(parser, env)
        for name, help, action in self._options:
            env_opt = "NOSE_XTRACEBACK_%s" % name.upper()
            parser.add_option("--xtraceback-%s" % name.replace("_", "-"),
                              action=action,
                              dest="xtraceback_%s" % name,
                              default=env.get(env_opt),
                              help=help % env_opt)

    def configure(self, options, conf):
        super(NoseXTraceback, self).configure(options, conf)
        for name, help, dummy in self._options:
            name = "xtraceback_%s" % name
            setattr(self, name, getattr(options, name))
        if self.xtraceback_color is not None:
            if self.xtraceback_color == "on":
                self.xtraceback_color = True
            else:
                self.xtraceback_color = False

    def begin(self):
        # not importing in global scope because it messes with the coverage analysis
        import xtraceback
        options = dict(color=self.xtraceback_color,
                       stream=self.conf.stream,
                       show_globals=self.xtraceback_globals,
                       globals_module_include=self.xtraceback_globals_include)
        xtraceback.compat.defaults.update(**options)
        xtraceback.compat.install()

    def finalize(self, result):
        import xtraceback
        xtraceback.compat.uninstall()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import optparse
import subprocess
import signal
import traceback
import os
from os import path

from twisted.internet import reactor
from twisted.web      import proxy, server, resource

parser = optparse.OptionParser(r"""

Starts the app listening on localhost, for local development.

This script launches the Django and Tornado servers, then runs a reverse proxy
which serves to both of them.  After it's all up and running, browse to

    http://localhost:9991/

Note that, while runserver and runtornado have the usual auto-restarting
behavior, the reverse proxy itself does *not* automatically restart on changes
to this file.
""")

parser.add_option('--test',
    action='store_true', dest='test',
    help='Use the testing database and ports')

(options, args) = parser.parse_args()

base_port   = 9991
manage_args = ''
if options.test:
    base_port   = 9981
    manage_args = '--settings=humbug.test_settings'

proxy_port   = base_port
django_port  = base_port+1
tornado_port = base_port+2
proxy_host = 'localhost:%d' % (proxy_port,)

os.chdir(path.join(path.dirname(__file__), '..'))

# Set up a new process group, so that we can later kill run{server,tornado}
# and all of the processes they spawn.
os.setpgrp()

# Pass --nostatic because we configure static serving ourselves in
# humbug/urls.py.
for cmd in ['python manage.py runserver --nostatic %s localhost:%d'
                % (manage_args, django_port),
            'python manage.py runtornado %s localhost:%d'
                % (manage_args, tornado_port)]:
    subprocess.Popen(cmd, shell=True)

class Resource(resource.Resource):
    def getChild(self, name, request):
        request.requestHeaders.setRawHeaders('X-Forwarded-Host', [proxy_host])

        if request.uri in ['/json/get_updates', '/api/v1/get_messages']:
            return proxy.ReverseProxyResource('localhost', tornado_port, '/'+name)

        return proxy.ReverseProxyResource('localhost', django_port, '/'+name)

try:
    reactor.listenTCP(proxy_port, server.Site(Resource()), interface='127.0.0.1')
    reactor.run()
except:
    # Print the traceback before we get SIGTERM and die.
    traceback.print_exc()
    raise
finally:
    # Kill everything in our process group.
    os.killpg(0, signal.SIGTERM)
<commit_msg>run-dev: Clean repo first<commit_after>#!/usr/bin/env python
import optparse
import subprocess
import signal
import traceback
import os
from os import path

from twisted.internet import reactor
from twisted.web      import proxy, server, resource

parser = optparse.OptionParser(r"""

Starts the app listening on localhost, for local development.

This script launches the Django and Tornado servers, then runs a reverse proxy
which serves to both of them.  After it's all up and running, browse to

    http://localhost:9991/

Note that, while runserver and runtornado have the usual auto-restarting
behavior, the reverse proxy itself does *not* automatically restart on changes
to this file.
""")

parser.add_option('--test',
    action='store_true', dest='test',
    help='Use the testing database and ports')

(options, args) = parser.parse_args()

base_port   = 9991
manage_args = ''
if options.test:
    base_port   = 9981
    manage_args = '--settings=humbug.test_settings'

proxy_port   = base_port
django_port  = base_port+1
tornado_port = base_port+2
proxy_host = 'localhost:%d' % (proxy_port,)

os.chdir(path.join(path.dirname(__file__), '..'))

# Clean up stale .pyc files etc.
subprocess.check_call('./tools/clean-repo')

# Set up a new process group, so that we can later kill run{server,tornado}
# and all of the processes they spawn.
os.setpgrp()

# Pass --nostatic because we configure static serving ourselves in
# humbug/urls.py.
for cmd in ['python manage.py runserver --nostatic %s localhost:%d'
                % (manage_args, django_port),
            'python manage.py runtornado %s localhost:%d'
                % (manage_args, tornado_port)]:
    subprocess.Popen(cmd, shell=True)

class Resource(resource.Resource):
    def getChild(self, name, request):
        request.requestHeaders.setRawHeaders('X-Forwarded-Host', [proxy_host])

        if request.uri in ['/json/get_updates', '/api/v1/get_messages']:
            return proxy.ReverseProxyResource('localhost', tornado_port, '/'+name)

        return proxy.ReverseProxyResource('localhost', django_port, '/'+name)

try:
    reactor.listenTCP(proxy_port, server.Site(Resource()), interface='127.0.0.1')
    reactor.run()
except:
    # Print the traceback before we get SIGTERM and die.
    traceback.print_exc()
    raise
finally:
    # Kill everything in our process group.
    os.killpg(0, signal.SIGTERM)
<|endoftext|>


<commit_before>import re
import time
from threading import Thread

from cassandra import ConsistencyLevel
from ccmlib.node import NodetoolError, TimeoutError

from assertions import assert_almost_equal
from dtest import Tester
from tools import (debug, insert_c1c2, known_failure, no_vnodes, query_c1c2,
                   since)

class TestTopology(Tester):

    def do_not_join_ring_test(self):
        """
        @jira_ticket CASSANDRA-9034
        Check that AssertionError is not thrown on SizeEstimatesRecorder before node joins ring
        """
        cluster = self.cluster.populate(1)
        node1, = cluster.nodelist()

        node1.start(wait_for_binary_proto=True, join_ring=False,
                    jvm_args=["-Dcassandra.size_recorder_interval=1"])

        # initial delay is 30s
        time.sleep(40)

        node1.stop(gently=False)

    def simple_decommission_test(self):
        """
        @jira_ticket CASSANDRA-9912
        Check that AssertionError is not thrown on SizeEstimatesRecorder after node is decommissioned
        """
        cluster = self.cluster
        cluster.populate(3)
        cluster.start(wait_for_binary_proto=True, jvm_args=["-Dcassandra.size_recorder_interval=1"])
        [node1, node2, node3] = cluster.nodelist()

        # write some data
        node1.stress(['write', 'n=10K', '-rate', 'threads=8'])

        # Decommision node and wipe its data
        node2.decommission()
        node2.stop(wait_other_notice=True)

        time.sleep(10)

    @no_vnodes()
    def movement_test(self):
        cluster = self.cluster

        # Create an unbalanced ring
        cluster.populate(3, tokens=[0, 2**48, 2**62]).start()
        node1, node2, node3 = cluster.nodelist()

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE)

        cluster.flush()

        # Move nodes to balance the cluster
        def move_node(node, token, ip):
            mark = node.mark_log()
            node.move(token)  # can't assume 0 is balanced with m3p
            node.watch_log_for('{} state jump to NORMAL'.format(ip), from_mark=mark, timeout=180)
            time.sleep(3)

        balancing_tokens = cluster.balanced_tokens(3)

        move_node(node1, balancing_tokens[0], '127.0.0.1')
        move_node(node2, balancing_tokens[1], '127.0.0.2')
        move_node(node3, balancing_tokens[2], '127.0.0.3')

        time.sleep(1)

        cluster.cleanup()

        # Check we can get all the keys
        for n in xrange(0, 10000):
            query_c1c2(session, n, ConsistencyLevel.ONE)

        # Now the load should be basically even
        sizes = [node.data_size() for node in [node1, node2, node3]]

        assert_almost_equal(sizes[0], sizes[1])
        assert_almost_equal(sizes[0], sizes[2])
        assert_almost_equal(sizes[1], sizes[2])

    @no_vnodes()
    def decommission_test(self):
        cluster = self.cluster

        tokens = cluster.balanced_tokens(4)
        cluster.populate(4, tokens=tokens).start()
        node1, node2, node3, node4 = cluster.nodelist()

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 2)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=30000, consistency=ConsistencyLevel.QUORUM)

        cluster.flush()
        sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()]
        init_size = sizes[0]
        assert_almost_equal(*sizes)

        time.sleep(.5)
        node4.decommission()
        node4.stop()
        cluster.cleanup()
        time.sleep(.5)

        # Check we can get all the keys
        for n in xrange(0, 30000):
            query_c1c2(session, n, ConsistencyLevel.QUORUM)

        sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()]
        debug(sizes)
        three_node_sizes = sizes
        assert_almost_equal(sizes[0], sizes[1])
        assert_almost_equal((2.0 / 3.0) * sizes[0], sizes[2])
        assert_almost_equal(sizes[2], init_size)

    @no_vnodes()
    def move_single_node_test(self):
        """ Test moving a node in a single-node cluster (#4200) """
        cluster = self.cluster

        # Create an unbalanced ring
        cluster.populate(1, tokens=[0]).start()
        node1 = cluster.nodelist()[0]
        time.sleep(0.2)

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE)

        cluster.flush()

        node1.move(2**25)
        time.sleep(1)

        cluster.cleanup()

        # Check we can get all the keys
        for n in xrange(0, 10000):
            query_c1c2(session, n, ConsistencyLevel.ONE)

    @since('3.0')
    @no_vnodes()
    def decommissioned_node_cant_rejoin_test(self):
        '''
        @jira_ticket CASSANDRA-8801

        Test that a decommissioned node can't rejoin the cluster by:

        - creating a cluster,
        - decommissioning a node, and
        - asserting that the "decommissioned node won't rejoin" error is in the
        logs for that node and
        - asserting that the node is not running.
        '''
        rejoin_err = 'This node was decommissioned and will not rejoin the ring'
        try:
            self.ignore_log_patterns = list(self.ignore_log_patterns)
        except AttributeError:
            self.ignore_log_patterns = []
        self.ignore_log_patterns.append(rejoin_err)

        self.cluster.populate(3).start(wait_for_binary_proto=True)
        node1, node2, node3 = self.cluster.nodelist()

        debug('decommissioning...')
        node3.decommission()
        debug('stopping...')
        node3.stop()
        debug('attempting restart...')
        node3.start()
        try:
            # usually takes 3 seconds, so give it a generous 15
            node3.watch_log_for(rejoin_err, timeout=15)
        except TimeoutError:
            # TimeoutError is not very helpful to the reader of the test output;
            # let that pass and move on to string assertion below
            pass

        self.assertIn(rejoin_err,
                      '\n'.join(['\n'.join(err_list)
                                 for err_list in node3.grep_log_for_errors()]))

        # Give the node some time to shut down once it has detected
        # its invalid state
        start = time.time()
        while start + 5 > time.time() and node3.is_running():
            time.sleep(1)

        self.assertFalse(node3.is_running())

    @known_failure(failure_source='test',
                   jira_url='https://issues.apache.org/jira/browse/CASSANDRA-11377',
                   flaky=True,
                   notes="flaps when a node doesn't come up")
    @since('3.0')
    def crash_during_decommission_test(self):
        """
        If a node crashes whilst another node is being decommissioned,
        upon restarting the crashed node should not have invalid entries
        for the decommissioned node
        @jira_ticket CASSANDRA-10231
        """
        cluster = self.cluster
        cluster.populate(3).start(wait_other_notice=True)

        node1, node2 = cluster.nodelist()[0:2]

        t = DecommissionInParallel(node1)
        t.start()

        null_status_pattern = re.compile(".N(?:\s*)127\.0\.0\.1(?:.*)null(?:\s*)rack1")
        while t.is_alive():
            out = self.show_status(node2)
            if null_status_pattern.search(out):
                debug("Matched null status entry")
                break
            debug("Restarting node2")
            node2.stop(gently=False)
            node2.start(wait_for_binary_proto=True)

        debug("Waiting for decommission to complete")
        t.join()
        self.show_status(node2)

        debug("Sleeping for 30 seconds to allow gossip updates")
        time.sleep(30)
        out = self.show_status(node2)
        self.assertFalse(null_status_pattern.search(out))

    def show_status(self, node):
        out, err = node.nodetool('status')
        debug("Status as reported by node {}".format(node.address()))
        debug(out)
        return out

class DecommissionInParallel(Thread):

    def __init__(self, node):
        Thread.__init__(self)
        self.node = node

    def run(self):
        node = self.node
        mark = node.mark_log()
        try:
            out, err = node.nodetool("decommission")
            node.watch_log_for("DECOMMISSIONED", from_mark=mark)
            debug(out)
            debug(err)
        except NodetoolError as e:
            debug("Decommission failed with exception: " + str(e))
            pass
<commit_msg>remove unused variable<commit_after>import re
import time
from threading import Thread

from cassandra import ConsistencyLevel
from ccmlib.node import NodetoolError, TimeoutError

from assertions import assert_almost_equal
from dtest import Tester
from tools import (debug, insert_c1c2, known_failure, no_vnodes, query_c1c2,
                   since)

class TestTopology(Tester):

    def do_not_join_ring_test(self):
        """
        @jira_ticket CASSANDRA-9034
        Check that AssertionError is not thrown on SizeEstimatesRecorder before node joins ring
        """
        cluster = self.cluster.populate(1)
        node1, = cluster.nodelist()

        node1.start(wait_for_binary_proto=True, join_ring=False,
                    jvm_args=["-Dcassandra.size_recorder_interval=1"])

        # initial delay is 30s
        time.sleep(40)

        node1.stop(gently=False)

    def simple_decommission_test(self):
        """
        @jira_ticket CASSANDRA-9912
        Check that AssertionError is not thrown on SizeEstimatesRecorder after node is decommissioned
        """
        cluster = self.cluster
        cluster.populate(3)
        cluster.start(wait_for_binary_proto=True, jvm_args=["-Dcassandra.size_recorder_interval=1"])
        [node1, node2, node3] = cluster.nodelist()

        # write some data
        node1.stress(['write', 'n=10K', '-rate', 'threads=8'])

        # Decommision node and wipe its data
        node2.decommission()
        node2.stop(wait_other_notice=True)

        time.sleep(10)

    @no_vnodes()
    def movement_test(self):
        cluster = self.cluster

        # Create an unbalanced ring
        cluster.populate(3, tokens=[0, 2**48, 2**62]).start()
        node1, node2, node3 = cluster.nodelist()

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE)

        cluster.flush()

        # Move nodes to balance the cluster
        def move_node(node, token, ip):
            mark = node.mark_log()
            node.move(token)  # can't assume 0 is balanced with m3p
            node.watch_log_for('{} state jump to NORMAL'.format(ip), from_mark=mark, timeout=180)
            time.sleep(3)

        balancing_tokens = cluster.balanced_tokens(3)

        move_node(node1, balancing_tokens[0], '127.0.0.1')
        move_node(node2, balancing_tokens[1], '127.0.0.2')
        move_node(node3, balancing_tokens[2], '127.0.0.3')

        time.sleep(1)

        cluster.cleanup()

        # Check we can get all the keys
        for n in xrange(0, 10000):
            query_c1c2(session, n, ConsistencyLevel.ONE)

        # Now the load should be basically even
        sizes = [node.data_size() for node in [node1, node2, node3]]

        assert_almost_equal(sizes[0], sizes[1])
        assert_almost_equal(sizes[0], sizes[2])
        assert_almost_equal(sizes[1], sizes[2])

    @no_vnodes()
    def decommission_test(self):
        cluster = self.cluster

        tokens = cluster.balanced_tokens(4)
        cluster.populate(4, tokens=tokens).start()
        node1, node2, node3, node4 = cluster.nodelist()

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 2)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=30000, consistency=ConsistencyLevel.QUORUM)

        cluster.flush()
        sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()]
        init_size = sizes[0]
        assert_almost_equal(*sizes)

        time.sleep(.5)
        node4.decommission()
        node4.stop()
        cluster.cleanup()
        time.sleep(.5)

        # Check we can get all the keys
        for n in xrange(0, 30000):
            query_c1c2(session, n, ConsistencyLevel.QUORUM)

        sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()]
        debug(sizes)
        assert_almost_equal(sizes[0], sizes[1])
        assert_almost_equal((2.0 / 3.0) * sizes[0], sizes[2])
        assert_almost_equal(sizes[2], init_size)

    @no_vnodes()
    def move_single_node_test(self):
        """ Test moving a node in a single-node cluster (#4200) """
        cluster = self.cluster

        # Create an unbalanced ring
        cluster.populate(1, tokens=[0]).start()
        node1 = cluster.nodelist()[0]
        time.sleep(0.2)

        session = self.patient_cql_connection(node1)
        self.create_ks(session, 'ks', 1)
        self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

        insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE)

        cluster.flush()

        node1.move(2**25)
        time.sleep(1)

        cluster.cleanup()

        # Check we can get all the keys
        for n in xrange(0, 10000):
            query_c1c2(session, n, ConsistencyLevel.ONE)

    @since('3.0')
    @no_vnodes()
    def decommissioned_node_cant_rejoin_test(self):
        '''
        @jira_ticket CASSANDRA-8801

        Test that a decommissioned node can't rejoin the cluster by:

        - creating a cluster,
        - decommissioning a node, and
        - asserting that the "decommissioned node won't rejoin" error is in the
        logs for that node and
        - asserting that the node is not running.
        '''
        rejoin_err = 'This node was decommissioned and will not rejoin the ring'
        try:
            self.ignore_log_patterns = list(self.ignore_log_patterns)
        except AttributeError:
            self.ignore_log_patterns = []
        self.ignore_log_patterns.append(rejoin_err)

        self.cluster.populate(3).start(wait_for_binary_proto=True)
        node1, node2, node3 = self.cluster.nodelist()

        debug('decommissioning...')
        node3.decommission()
        debug('stopping...')
        node3.stop()
        debug('attempting restart...')
        node3.start()
        try:
            # usually takes 3 seconds, so give it a generous 15
            node3.watch_log_for(rejoin_err, timeout=15)
        except TimeoutError:
            # TimeoutError is not very helpful to the reader of the test output;
            # let that pass and move on to string assertion below
            pass

        self.assertIn(rejoin_err,
                      '\n'.join(['\n'.join(err_list)
                                 for err_list in node3.grep_log_for_errors()]))

        # Give the node some time to shut down once it has detected
        # its invalid state
        start = time.time()
        while start + 5 > time.time() and node3.is_running():
            time.sleep(1)

        self.assertFalse(node3.is_running())

    @known_failure(failure_source='test',
                   jira_url='https://issues.apache.org/jira/browse/CASSANDRA-11377',
                   flaky=True,
                   notes="flaps when a node doesn't come up")
    @since('3.0')
    def crash_during_decommission_test(self):
        """
        If a node crashes whilst another node is being decommissioned,
        upon restarting the crashed node should not have invalid entries
        for the decommissioned node
        @jira_ticket CASSANDRA-10231
        """
        cluster = self.cluster
        cluster.populate(3).start(wait_other_notice=True)

        node1, node2 = cluster.nodelist()[0:2]

        t = DecommissionInParallel(node1)
        t.start()

        null_status_pattern = re.compile(".N(?:\s*)127\.0\.0\.1(?:.*)null(?:\s*)rack1")
        while t.is_alive():
            out = self.show_status(node2)
            if null_status_pattern.search(out):
                debug("Matched null status entry")
                break
            debug("Restarting node2")
            node2.stop(gently=False)
            node2.start(wait_for_binary_proto=True)

        debug("Waiting for decommission to complete")
        t.join()
        self.show_status(node2)

        debug("Sleeping for 30 seconds to allow gossip updates")
        time.sleep(30)
        out = self.show_status(node2)
        self.assertFalse(null_status_pattern.search(out))

    def show_status(self, node):
        out, err = node.nodetool('status')
        debug("Status as reported by node {}".format(node.address()))
        debug(out)
        return out

class DecommissionInParallel(Thread):

    def __init__(self, node):
        Thread.__init__(self)
        self.node = node

    def run(self):
        node = self.node
        mark = node.mark_log()
        try:
            out, err = node.nodetool("decommission")
            node.watch_log_for("DECOMMISSIONED", from_mark=mark)
            debug(out)
            debug(err)
        except NodetoolError as e:
            debug("Decommission failed with exception: " + str(e))
            pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
INI file should be configured named and supplied as argument of fullpathtofile or just filename if in curent wd eg:

		$ python writeConfig.py

		$ python topomapp_main.py test.ini &> test.log
"""

import sys
import os
import subprocess
import logging
import os.path
from listpoints_make import getRasterDims as dims
import glob
import joblib

#====================================================================
#	Timer
#====================================================================
import time
start_time = time.time()

#====================================================================
#	Config setup
#====================================================================
#os.system("python writeConfig.py") # update config DONE IN run.sh file
from configobj import ConfigObj
config = ConfigObj(sys.argv[1])
wd = config["main"]["wd"]

#====================================================================
#	Creat wd dir if doesnt exist
#====================================================================
#directory = os.path.dirname(wd)
if not os.path.exists(wd):
	os.makedirs(wd)

#====================================================================
#	Logging
#====================================================================

logging.basicConfig(level=logging.DEBUG, filename=wd+"/logfile", filemode="a+",
                        format="%(asctime)-15s %(levelname)-8s %(message)s")

#====================================================================
#	Announce wd
#====================================================================
logging.info("----------------------- START RUN -----------------------")
logging.info("Simulation directory: " + wd  )
#====================================================================
#	Initialise run: this can be used to copy meteo and surfaces to a new sim directory.
# 	Main application is in ensemble runs
#====================================================================
if config["main"]["initSim"] == "TRUE":
	import TMinit
	TMinit.main(config, ensembRun=False)

#====================================================================
# Copy config to simulation directory
#====================================================================
configfilename = os.path.basename(sys.argv[1])

config.filename = wd +  "/" + configfilename
config.write()

#====================================================================
#	Setup domain
#====================================================================

# control statement to skip if "asp.tif" exist - indicator fileNOT ROBUST
fname = wd + "/predictors/asp.tif"
if os.path.isfile(fname) == False:

		# copy preexisting dem
	if config["main"]["demexists"] == "TRUE":

		cmd = "mkdir " + wd + "/predictors/"
		os.system(cmd)
		src = config["main"]["dempath"]
		dst = wd +"/predictors/dem.tif"
		cmd = "cp -r %s %s"%(src,dst)
		os.system(cmd)

	from domain_setup import getDEM2 as gdem
	logging.info("Retrive DEM")
	gdem.main(wd ,config["main"]["demDir"] ,config["era-interim"]["domain"], config["main"]["shp"])

	from domain_setup import makeKML as kml
	logging.info("Generate extent files")
	kml.main(wd, wd + "/predictors/ele.tif", "shape", wd + "/spatial/extent")
	kml.main(wd, wd + "/spatial/eraExtent.tif", "raster", wd + "/spatial/eraExtent")
	#kml.main(wd, wd + "/spatial/extentRequest.shp", "raster", wd + "/spatial/extentRequest")

	from domain_setup import computeTopo as topo
	logging.info("Compute topo predictors")
	topo.main(wd, config["toposcale"]["svfCompute"])

else:
	logging.info("Topo predictors computed")
#====================================================================
#	GET ERA
#====================================================================

fname1 = wd + "/eraDat/SURF.nc"
fname2 = wd + "/eraDat/PLEVEL.nc"
if os.path.isfile(fname2) == False and os.path.isfile(fname2) == False: #NOT ROBUST

	# set ccords to those of downloaded dem extent
	#if config["main"]["runtype"] == "points":
	from getERA import getExtent as ext
	latN = ext.main(wd + "/predictors/ele.tif" , "latN")
	latS = ext.main(wd + "/predictors/ele.tif" , "latS")
	lonW = ext.main(wd + "/predictors/ele.tif" , "lonW")
	lonE = ext.main(wd + "/predictors/ele.tif" , "lonE")

	config["main"]["latn"]  = latN
	config["main"]["lats"]  = latS
	config["main"]["lonw"]  = lonW
	config["main"]["lone"]  = lonE

	eraDir = wd + "/eraDat"
	if not os.path.exists(eraDir):
		os.mkdir(eraDir)

	from getERA import eraRetrievePLEVEL_pl as plevel
	logging.info( "Retrieving ECWMF pressure-level data")
	#plevel.retrieve_interim( config["main"]["startDate"], config["main"]["endDate"], latN, latS, lonE, lonW, config["era-interim"]["grid"],eraDir, config["era-interim"]["dataset"] )
	plevel.retrieve_interim( config, eraDir  , latN, latS, lonE, lonW)

	from getERA import eraRetrieveSURFACE_pl as surf
	logging.info( "Retrieving ECWMF surface data")
	#surf.retrieve_interim(config["main"]["startDate"], config["main"]["endDate"], latN, latS, lonE, lonW, config["era-interim"]["grid"],eraDir, config["era-interim"]["dataset"] )
	surf.retrieve_interim( config, eraDir  , latN, latS, lonE, lonW)

	# Merge NC timeseries (requires linux package cdo)
	import subprocess
	#os.chdir(eraDir)
	cmd     = "cdo -b F64 -f nc2 mergetime " + wd + "/eraDat/interim_daily_PLEVEL* " +  wd + "/eraDat/PLEVEL.nc"

	if os.path.exists(wd + "eraDat/PLEVEL.nc"):
	    os.remove(wd + "eraDat/PLEVEL.nc")
	    logging.info( "removed original PLEVEL.nc")

	logging.info("Running:" + str(cmd))
	subprocess.check_output(cmd, shell = "TRUE")

	cmd     = "cdo -b F64 -f nc2 mergetime " + wd +  "/eraDat/interim_daily_SURF* " + wd +"/eraDat/SURF.nc"

	if os.path.exists(wd + "eraDat/SURF.nc"):
	    os.remove(wd + "eraDat/SURF.nc")
	    logging.info( "removed original SURF.nc" )

	logging.info(" Running:" + str(cmd))
	subprocess.check_output(cmd, shell = "TRUE")

else:
	logging.info( "ERA data retrieved" )

	#os.chdir(config["main"]["srcdir"])

#====================================================================
#	Preprocess ERA
#====================================================================
fname1 = wd + "/eraDat/all/tSurf" # check for last file gerenrated by era_prep script

if os.path.isfile(fname2) == False:

	logging.info("Preprocessing ERA data")

	if config["era-interim"]["dataset"] == "interim":
		from getERA import era_prep as prep
		prep.main(wd, config["main"]["startDate"], config["main"]["endDate"])

	if config["era-interim"]["dataset"] == "era5":
		from getERA import era_prep2 as prep
		prep.main(wd, config["main"]["startDate"], config["main"]["endDate"])
else:
	logging.info( "ERA data processed")
#====================================================================
#	Prepare simulation directories - grid
#====================================================================

# check if sim directories already exist
grid_dirs = glob.glob(wd +"/grid*")
if len(grid_dirs) < 1:

	if config["main"]["initSim"] != "TRUE":
		logging.info( "initialising simulation directories")
		from getERA import prepSims as sim
		sim.main(wd)

	# define ncells here based on occurances of grid* directoriers created by prepSims or copied if initSim == True
	grid_dirs = glob.glob(wd +"/grid*")
	ncells = len(grid_dirs)

else:
	logging.info( "Simulation directories initialised")
#====================================================================
#	Create MODIS dir for NDVI at wd level
#====================================================================

# make output directory at wd level so grids can share hdf scenes if overlap - save download time
# in contrast sca is saved to gridpath and hdf not retained due to volume of files
ndvi_wd=wd + "/MODIS/NDVI"
if not os.path.exists(ndvi_wd):
	os.makedirs(ndvi_wd)

#====================================================================
#	dEFINE WHICH GRIDS CONTAIN POINTS: POINTS ONLY is this still required?
#====================================================================

# # re define ncells here based on occurances of grid* directoriers after removals
# grid_dirs = glob.glob(wd +"/grid*")
# ncells = len(grid_dirs)
# logging.info( " This simulation now contains ", ncells, " grids" )
# logging.info( " grids to be computed " + str(grid_dirs) )

# + LAST SECTION OF MAKELISPOINTS2.R
# from listpoints_make import findGridsWithPoints
# grid_dirs = findGridsWithPoints.main(wd, wd +"/spatial/eraExtent.tif" , config["main"]["shp"])

# logging.info( grid_dirs)
# ncells = len(grid_dirs)
# logging.info( " This simulation now contains ", str(ncells), " grids" )
# logging.info( " grids to be computed " + str(grid_dirs) )
#====================================================================
#	Start main Ngrid loop
#====================================================================

# start main grid loop here - make parallel here
# need config to prioritise grids here
logging.info("Grids to loop over = ")
logging.info( [os.path.basename(os.path.normpath(x)) for x in grid_dirs] )

for Ngrid in grid_dirs:
	gridpath = Ngrid

	# skip to next iteratyion if results exist
	fname1 = gridpath + "/groundResults"
	fname2 = gridpath + "/surfaceResults"
	if os.path.isfile(fname2) == True and os.path.isfile(fname2) == True:
		logging.info( "Results already exist for " + Ngrid +" skipping to next grid" )
		continue

	logging.info( "----- NOW COMPUTING GRID: " + os.path.basename(os.path.normpath(Ngrid)) + "-----")
#====================================================================
#	Does grid contain points?
#====================================================================
	if config['main']['runtype'] == points:

		# read in points
		import fiona
		point = fiona.open(config['main']['shp'])

		# get grid extent
		from getERA import getExtent as ext
		latN = ext.main(gridpath + "/predictors/ele.tif" , "latN")
		latS = ext.main(gridpath + "/predictors/ele.tif" , "latS")
		lonW = ext.main(gridpath + "/predictors/ele.tif" , "lonW")
		lonE = ext.main(gridpath + "/predictors/ele.tif" , "lonE")

		from shapely.geometry import Point
		from shapely.geometry.polygon import Polygon

		polygon = Polygon([(lonW,latS), (lonW, latN), (lonE, latN), (lonE, latS)])
		logging.info(polygon.contains(point))

#====================================================================
#	Compute svf here
#====================================================================
	if config['toposcale']['svfCompute'] == 'TRUE':

		logging.info( "Calculating SVF layer: " + os.path.basename(os.path.normpath(Ngrid)) )

		from domain_setup import computeSVF
		computeSVF.main(gridpath, angles=str(6), dist=str(500))
#====================================================================
#	Download MODIS NDVI here
#====================================================================
	# only run if surface tif doesnt exist
	fname = gridpath + "/predictors/surface.tif"
	if os.path.isfile(fname) == False:

		logging.info( "Preparing land surface layer from MODIS: " + os.path.basename(os.path.normpath(Ngrid)) )

		# compute from dem of small grid
		from getERA import getExtent as ext
		latN = ext.main(gridpath + "/predictors/ele.tif" , "latN")
		latS = ext.main(gridpath + "/predictors/ele.tif" , "latS")
		lonW = ext.main(gridpath + "/predictors/ele.tif" , "lonW")
		lonE = ext.main(gridpath + "/predictors/ele.tif" , "lonE")

		#need to run loop of five requests at set dates (can be fixed for now)
		mydates=["2000-08-12","2004-08-12","2008-08-12","2012-08-12"]#,"2016-08-12"]
		for date in mydates:
			# call bash script that does grep type stuff to update values in options file
			cmd = ["./DA/updateOptions.sh" , lonW , latS , lonE , latN , date , date, config["modis"]["options_file_NDVI"], ndvi_wd,config['modis']['tileX_start'] , config['modis']['tileX_end'] , config['modis']['tileY_start'] , config['modis']['tileY_end']]
			subprocess.check_output(cmd)

			# run MODIStsp tool
			from DA import getMODIS as gmod
			gmod.main("FALSE" , config["modis"]["options_file_NDVI"]) #  able to run non-interactively now

		from domain_setup import makeSurface as surf
		surf.main(gridpath, ndvi_wd )

#====================================================================
#	Run bbox script
#====================================================================
	if config["main"]["runtype"] == "bbox":
		import TMgrid
		TMgrid.main(wd, Ngrid, config)

#====================================================================
#	Run points script
#====================================================================

# make into proper modules with arguments etc
# import config and Ngrid
	if config["main"]["runtype"] == "points":
		import TMpoints
		TMpoints.main(wd, Ngrid, config)

#====================================================================
#	Aggregate results and clean up
#====================================================================
	from topoResults import resultsCube
	resultsCube.main(Ngrid)

#====================================================================
#	Run ensemble
#====================================================================
	# if config["ensemble"]["run"] == "TRUE":
	# 	import ensembleRun
	# 	ensembleRun.main(config)

#====================================================================
#	Run DA
#====================================================================
	#if config["main"]["mode"] == "da":
		#import TMensemble.py
		#import TMda.py

#====================================================================
#	Polat plot
#====================================================================
	# src = "./rsrc/polarPlot.R"
	# dst = gridpath
	# cmd = "Rscript %s %s"%(src,dst)
	# os.system(cmd)

logging.info(" %f minutes for run" % round((time.time()/60 - start_time/60),2) )

<commit_msg>skip grid no points routine<commit_after>#!/usr/bin/env python

"""
INI file should be configured named and supplied as argument of fullpathtofile or just filename if in curent wd eg:

		$ python writeConfig.py

		$ python topomapp_main.py test.ini &> test.log
"""

import sys
import os
import subprocess
import logging
import os.path
from listpoints_make import getRasterDims as dims
import glob
import joblib

#====================================================================
#	Timer
#====================================================================
import time
start_time = time.time()

#====================================================================
#	Config setup
#====================================================================
#os.system("python writeConfig.py") # update config DONE IN run.sh file
from configobj import ConfigObj
config = ConfigObj(sys.argv[1])
wd = config["main"]["wd"]

#====================================================================
#	Creat wd dir if doesnt exist
#====================================================================
#directory = os.path.dirname(wd)
if not os.path.exists(wd):
	os.makedirs(wd)

#====================================================================
#	Logging
#====================================================================

logging.basicConfig(level=logging.DEBUG, filename=wd+"/logfile", filemode="a+",
                        format="%(asctime)-15s %(levelname)-8s %(message)s")

#====================================================================
#	Announce wd
#====================================================================
logging.info("----------------------- START RUN -----------------------")
logging.info("Simulation directory: " + wd  )
#====================================================================
#	Initialise run: this can be used to copy meteo and surfaces to a new sim directory.
# 	Main application is in ensemble runs
#====================================================================
if config["main"]["initSim"] == "TRUE":
	import TMinit
	TMinit.main(config, ensembRun=False)

#====================================================================
# Copy config to simulation directory
#====================================================================
configfilename = os.path.basename(sys.argv[1])

config.filename = wd +  "/" + configfilename
config.write()

#====================================================================
#	Setup domain
#====================================================================

# control statement to skip if "asp.tif" exist - indicator fileNOT ROBUST
fname = wd + "/predictors/asp.tif"
if os.path.isfile(fname) == False:

		# copy preexisting dem
	if config["main"]["demexists"] == "TRUE":

		cmd = "mkdir " + wd + "/predictors/"
		os.system(cmd)
		src = config["main"]["dempath"]
		dst = wd +"/predictors/dem.tif"
		cmd = "cp -r %s %s"%(src,dst)
		os.system(cmd)

	from domain_setup import getDEM2 as gdem
	logging.info("Retrive DEM")
	gdem.main(wd ,config["main"]["demDir"] ,config["era-interim"]["domain"], config["main"]["shp"])

	from domain_setup import makeKML as kml
	logging.info("Generate extent files")
	kml.main(wd, wd + "/predictors/ele.tif", "shape", wd + "/spatial/extent")
	kml.main(wd, wd + "/spatial/eraExtent.tif", "raster", wd + "/spatial/eraExtent")
	#kml.main(wd, wd + "/spatial/extentRequest.shp", "raster", wd + "/spatial/extentRequest")

	from domain_setup import computeTopo as topo
	logging.info("Compute topo predictors")
	topo.main(wd, config["toposcale"]["svfCompute"])

else:
	logging.info("Topo predictors computed")
#====================================================================
#	GET ERA
#====================================================================

fname1 = wd + "/eraDat/SURF.nc"
fname2 = wd + "/eraDat/PLEVEL.nc"
if os.path.isfile(fname2) == False and os.path.isfile(fname2) == False: #NOT ROBUST

	# set ccords to those of downloaded dem extent
	#if config["main"]["runtype"] == "points":
	from getERA import getExtent as ext
	latN = ext.main(wd + "/predictors/ele.tif" , "latN")
	latS = ext.main(wd + "/predictors/ele.tif" , "latS")
	lonW = ext.main(wd + "/predictors/ele.tif" , "lonW")
	lonE = ext.main(wd + "/predictors/ele.tif" , "lonE")

	config["main"]["latn"]  = latN
	config["main"]["lats"]  = latS
	config["main"]["lonw"]  = lonW
	config["main"]["lone"]  = lonE

	eraDir = wd + "/eraDat"
	if not os.path.exists(eraDir):
		os.mkdir(eraDir)

	from getERA import eraRetrievePLEVEL_pl as plevel
	logging.info( "Retrieving ECWMF pressure-level data")
	#plevel.retrieve_interim( config["main"]["startDate"], config["main"]["endDate"], latN, latS, lonE, lonW, config["era-interim"]["grid"],eraDir, config["era-interim"]["dataset"] )
	plevel.retrieve_interim( config, eraDir  , latN, latS, lonE, lonW)

	from getERA import eraRetrieveSURFACE_pl as surf
	logging.info( "Retrieving ECWMF surface data")
	#surf.retrieve_interim(config["main"]["startDate"], config["main"]["endDate"], latN, latS, lonE, lonW, config["era-interim"]["grid"],eraDir, config["era-interim"]["dataset"] )
	surf.retrieve_interim( config, eraDir  , latN, latS, lonE, lonW)

	# Merge NC timeseries (requires linux package cdo)
	import subprocess
	#os.chdir(eraDir)
	cmd     = "cdo -b F64 -f nc2 mergetime " + wd + "/eraDat/interim_daily_PLEVEL* " +  wd + "/eraDat/PLEVEL.nc"

	if os.path.exists(wd + "eraDat/PLEVEL.nc"):
	    os.remove(wd + "eraDat/PLEVEL.nc")
	    logging.info( "removed original PLEVEL.nc")

	logging.info("Running:" + str(cmd))
	subprocess.check_output(cmd, shell = "TRUE")

	cmd     = "cdo -b F64 -f nc2 mergetime " + wd +  "/eraDat/interim_daily_SURF* " + wd +"/eraDat/SURF.nc"

	if os.path.exists(wd + "eraDat/SURF.nc"):
	    os.remove(wd + "eraDat/SURF.nc")
	    logging.info( "removed original SURF.nc" )

	logging.info(" Running:" + str(cmd))
	subprocess.check_output(cmd, shell = "TRUE")

else:
	logging.info( "ERA data retrieved" )

	#os.chdir(config["main"]["srcdir"])

#====================================================================
#	Preprocess ERA
#====================================================================
fname1 = wd + "/eraDat/all/tSurf" # check for last file gerenrated by era_prep script

if os.path.isfile(fname2) == False:

	logging.info("Preprocessing ERA data")

	if config["era-interim"]["dataset"] == "interim":
		from getERA import era_prep as prep
		prep.main(wd, config["main"]["startDate"], config["main"]["endDate"])

	if config["era-interim"]["dataset"] == "era5":
		from getERA import era_prep2 as prep
		prep.main(wd, config["main"]["startDate"], config["main"]["endDate"])
else:
	logging.info( "ERA data processed")
#====================================================================
#	Prepare simulation directories - grid
#====================================================================

# check if sim directories already exist
grid_dirs = glob.glob(wd +"/grid*")
if len(grid_dirs) < 1:

	if config["main"]["initSim"] != "TRUE":
		logging.info( "initialising simulation directories")
		from getERA import prepSims as sim
		sim.main(wd)

	# define ncells here based on occurances of grid* directoriers created by prepSims or copied if initSim == True
	grid_dirs = glob.glob(wd +"/grid*")
	ncells = len(grid_dirs)

else:
	logging.info( "Simulation directories initialised")
#====================================================================
#	Create MODIS dir for NDVI at wd level
#====================================================================

# make output directory at wd level so grids can share hdf scenes if overlap - save download time
# in contrast sca is saved to gridpath and hdf not retained due to volume of files
ndvi_wd=wd + "/MODIS/NDVI"
if not os.path.exists(ndvi_wd):
	os.makedirs(ndvi_wd)

#====================================================================
#	dEFINE WHICH GRIDS CONTAIN POINTS: POINTS ONLY is this still required?
#====================================================================

# # re define ncells here based on occurances of grid* directoriers after removals
# grid_dirs = glob.glob(wd +"/grid*")
# ncells = len(grid_dirs)
# logging.info( " This simulation now contains ", ncells, " grids" )
# logging.info( " grids to be computed " + str(grid_dirs) )

# + LAST SECTION OF MAKELISPOINTS2.R
# from listpoints_make import findGridsWithPoints
# grid_dirs = findGridsWithPoints.main(wd, wd +"/spatial/eraExtent.tif" , config["main"]["shp"])

# logging.info( grid_dirs)
# ncells = len(grid_dirs)
# logging.info( " This simulation now contains ", str(ncells), " grids" )
# logging.info( " grids to be computed " + str(grid_dirs) )
#====================================================================
#	Start main Ngrid loop
#====================================================================

# start main grid loop here - make parallel here
# need config to prioritise grids here
logging.info("Grids to loop over = ")
logging.info( [os.path.basename(os.path.normpath(x)) for x in grid_dirs] )

for Ngrid in grid_dirs:
	gridpath = Ngrid

	# skip to next iteratyion if results exist
	fname1 = gridpath + "/groundResults"
	fname2 = gridpath + "/surfaceResults"
	if os.path.isfile(fname2) == True and os.path.isfile(fname2) == True:
		logging.info( "Results already exist for " + Ngrid +" skipping to next grid" )
		continue

	logging.info( "----- NOW COMPUTING GRID: " + os.path.basename(os.path.normpath(Ngrid)) + "-----")
#====================================================================
#	Does grid contain points?
#====================================================================
	if config['main']['runtype'] == "points":

		# read in points
		import fiona
		point = fiona.open(config['main']['shp'])

		# get grid extent
		from getERA import getExtent as ext
		latN = ext.main(gridpath + "/predictors/ele.tif" , "latN")
		latS = ext.main(gridpath + "/predictors/ele.tif" , "latS")
		lonW = ext.main(gridpath + "/predictors/ele.tif" , "lonW")
		lonE = ext.main(gridpath + "/predictors/ele.tif" , "lonE")

		from shapely.geometry import Point
		from shapely.geometry.polygon import Polygon

		polygon = Polygon([(lonW,latS), (lonW, latN), (lonE, latN), (lonE, latS)])
		logging.info(polygon.contains(point))

#====================================================================
#	Compute svf here
#====================================================================
	if config['toposcale']['svfCompute'] == 'TRUE':

		logging.info( "Calculating SVF layer: " + os.path.basename(os.path.normpath(Ngrid)) )

		from domain_setup import computeSVF
		computeSVF.main(gridpath, angles=str(6), dist=str(500))
#====================================================================
#	Download MODIS NDVI here
#====================================================================
	# only run if surface tif doesnt exist
	fname = gridpath + "/predictors/surface.tif"
	if os.path.isfile(fname) == False:

		logging.info( "Preparing land surface layer from MODIS: " + os.path.basename(os.path.normpath(Ngrid)) )

		# compute from dem of small grid
		from getERA import getExtent as ext
		latN = ext.main(gridpath + "/predictors/ele.tif" , "latN")
		latS = ext.main(gridpath + "/predictors/ele.tif" , "latS")
		lonW = ext.main(gridpath + "/predictors/ele.tif" , "lonW")
		lonE = ext.main(gridpath + "/predictors/ele.tif" , "lonE")

		#need to run loop of five requests at set dates (can be fixed for now)
		mydates=["2000-08-12","2004-08-12","2008-08-12","2012-08-12"]#,"2016-08-12"]
		for date in mydates:
			# call bash script that does grep type stuff to update values in options file
			cmd = ["./DA/updateOptions.sh" , lonW , latS , lonE , latN , date , date, config["modis"]["options_file_NDVI"], ndvi_wd,config['modis']['tileX_start'] , config['modis']['tileX_end'] , config['modis']['tileY_start'] , config['modis']['tileY_end']]
			subprocess.check_output(cmd)

			# run MODIStsp tool
			from DA import getMODIS as gmod
			gmod.main("FALSE" , config["modis"]["options_file_NDVI"]) #  able to run non-interactively now

		from domain_setup import makeSurface as surf
		surf.main(gridpath, ndvi_wd )

#====================================================================
#	Run bbox script
#====================================================================
	if config["main"]["runtype"] == "bbox":
		import TMgrid
		TMgrid.main(wd, Ngrid, config)

#====================================================================
#	Run points script
#====================================================================

# make into proper modules with arguments etc
# import config and Ngrid
	if config["main"]["runtype"] == "points":
		import TMpoints
		TMpoints.main(wd, Ngrid, config)

#====================================================================
#	Aggregate results and clean up
#====================================================================
	from topoResults import resultsCube
	resultsCube.main(Ngrid)

#====================================================================
#	Run ensemble
#====================================================================
	# if config["ensemble"]["run"] == "TRUE":
	# 	import ensembleRun
	# 	ensembleRun.main(config)

#====================================================================
#	Run DA
#====================================================================
	#if config["main"]["mode"] == "da":
		#import TMensemble.py
		#import TMda.py

#====================================================================
#	Polat plot
#====================================================================
	# src = "./rsrc/polarPlot.R"
	# dst = gridpath
	# cmd = "Rscript %s %s"%(src,dst)
	# os.system(cmd)

logging.info(" %f minutes for run" % round((time.time()/60 - start_time/60),2) )

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
============================
RDoCdb Data Structure Client
============================

"""
__author__ = 'Nolan Nichols <https://orcid.org/0000-0003-1099-3328>'

import os
import sys

import requests

class DataDictionary(object):
    def __init__(self, url):
        self.url = url
        self.request = requests.get(self.url)
        self.json = self.request.json()
        self.name = self.json.get('name')
        self.version = self.json.get('version')
        self.description = self.json.get('description')
        self.operations = self.json.get('operations')

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.name)

    def get_data_sructures(self):
        ds_url = '{0}/datastructure'.format(self.url)
        request = requests.get(ds_url)
        data = request.json()
        result = dict()
        for ds in data:
            result.update({ds.get('shortName'): DataStructure(ds)})
        return result

class DataStructure(object):
    def __init__(self, data):
        self.category = data.get('category')
        self.data_type = data.get('dataType')
        self.ndar_url = data.get('ndarURL')
        self.publish_date = data.get('publishDate')
        self.short_name = data.get('shortName')
        self.source = data.get('source')
        self.status = data.get('status')
        self.title = data.get('title')

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.short_name)

def main(args=None):
    pass

if __name__ == "__main__":
    import argparse

    formatter = argparse.RawDescriptionHelpFormatter
    default = 'default: %(default)s'
    parser = argparse.ArgumentParser(prog="datastructure.py",
                                     description=__doc__,
                                     formatter_class=formatter)
    args = parser.parse_args()
    sys.exit(main(args=args))<commit_msg>add: data element object<commit_after>#!/usr/bin/env python
"""
============================
RDoCdb Data Structure Client
============================

"""
__author__ = 'Nolan Nichols <https://orcid.org/0000-0003-1099-3328>'

import os
import sys

import requests

class Connection(object):
    def __init__(self, url):
        self.url = url
        self.request = requests.get(self.url)

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.name)

class DataDictionary(Connection):
    def __init__(self, url):
        super(DataDictionary, self).__init__(url)
        self.json = self.request.json()
        self.name = self.json.get('name')
        self.version = self.json.get('version')
        self.description = self.json.get('description')
        self.operations = self.json.get('operations')

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.name)

    def get_data_sructures(self):
        ds_url = '{0}/datastructure'.format(self.url)
        request = requests.get(ds_url)
        data = request.json()
        result = dict()
        for ds in data:
            result.update({ds.get('shortName'): DataStructure(ds, self.url)})
        return result

class DataStructure(object):
    def __init__(self, data, url):
        self.category = data.get('category')
        self.data_type = data.get('dataType')
        self.ndar_url = data.get('ndarURL')
        self.publish_date = data.get('publishDate')
        self.short_name = data.get('shortName')
        self.source = data.get('source')
        self.status = data.get('status')
        self.title = data.get('title')
        self.url = url

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.short_name)

    def get_data_elements(self):
        ds_url = '{0}/datastructure/{1}'.format(self.url, self.short_name)
        request = requests.get(ds_url)
        data = request.json()
        result = dict()
        for de in data.get('dataElements'):
            result.update({de.get('name'): DataElement(de)})
        return result

class DataElement(object):
    def __init__(self, data):
        self.required = data.get('required')
        self.aliases = data.get('aliases')
        self.position = data.get('position')
        self.name = data.get('name')
        self.type = data.get('type')
        self.description = data.get('description')
        self.value_range = data.get('valueRange')
        self.title = data.get('title')

    def __repr__(self):
        return "{0}:{1}".format(self.__class__, self.name)

def main(args=None):
    pass

if __name__ == "__main__":
    import argparse

    formatter = argparse.RawDescriptionHelpFormatter
    default = 'default: %(default)s'
    parser = argparse.ArgumentParser(prog="datastructure.py",
                                     description=__doc__,
                                     formatter_class=formatter)
    args = parser.parse_args()
    sys.exit(main(args=args))<|endoftext|>