<commit_before>""" Isentropic properties. """

from __future__ import division, absolute_import

import numpy as np
import scipy as sp

from skaero.util.decorators import implicit

def mach_angle(M):
    r"""Returns Mach angle given supersonic Mach number.

    .. math::

        \mu = \arcsin{\left ( \frac{1}{M} \right )}

    Parameters
    ----------
    M : float
        Mach number.

    Returns
    -------
    mu : float
        Mach angle.

    Raises
    ------
    ValueError
        If given Mach number is subsonic.

    """
    try:
        with np.errstate(invalid="raise"):
            mu = np.arcsin(1 / M)
    except FloatingPointError:
        raise ValueError("Mach number must be supersonic")
    return mu

def mach_from_area_ratio(A_Astar, fl=None):
    """Computes the Mach number given an area ratio asuming isentropic flow.

    Uses the relation between Mach number and area ratio for isentropic flow,
    and returns both the subsonic and the supersonic solution.

    Parameters
    ----------
    A_Astar : float
        Cross sectional area.
    fl : IsentropicFlow, optional
        Isentropic flow object, default flow with gamma = 7 / 5.

    Returns
    -------
    out : tuple of floats
        Subsonic and supersonic Mach number solution of the equation.

    Raises
    ------
    ValueError
        If the area ratio is less than 1.0 (the critical area is always the
        minimum).

    """
    if not fl:
        fl = IsentropicFlow(gamma=1.4)
    eq = implicit(fl.A_Astar)
    if A_Astar < 1.0:
        raise ValueError("Area ratio must be greater than 1")
    elif A_Astar == 1.0:
        M_sub = M_sup = 1.0
    else:
        M_sub = sp.optimize.bisect(eq, 0.0, 1.0, args=(A_Astar,))
        M_sup = sp.optimize.newton(eq, 2.0, args=(A_Astar,))

    return M_sub, M_sup

def mach_from_nu(nu, in_radians=True, gamma=1.4):
    r"""Computes the Mach number given a Prandtl-Meyer angle, :math:`\nu`.

    Uses the relation between Mach number and Prandtl-Meyer angle for
    isentropic flow, to iteratively compute and return the Mach number.

    Parameters
    ----------
    nu : float
        Prandtl-Meyer angle, by default in radians.
    in_radians : bool, optional
        When set as False, converts nu from degrees to radians.
    gamma : float, optional
        Specific heat ratio.

    Returns
    -------
    M : float
        Mach number corresponding to :math:`\nu`.

    Raises
    ------
    ValueError
        If :math:`\nu` is 0 or negative or above the theoretical maxima based on
        :math:`\gamma`.

    """
    if not in_radians:
        nu = np.radians(nu)

    nu_max = np.pi / 2.0 * (np.sqrt((gamma + 1.0) / (gamma - 1.0)) - 1)
    if nu <= 0.0 or nu >= nu_max:
        raise ValueError(
            "Prandtl-Meyer angle must be between (0, %f) radians." % nu_max
        )

    eq = implicit(PrandtlMeyerExpansion.nu)
    M = sp.optimize.newton(eq, 2.0, args=(nu,))

    return M

class IsentropicFlow(object):
    """Class representing an isentropic gas flow.

    Isentropic flow is characterized by:

    * Viscous and heat conductivity effects are negligible.
    * No chemical or radioactive heat production.

    """

    def __init__(self, gamma=1.4):
        """Constructor of IsentropicFlow.

        Parameters
        ----------
        gamma : float, optional
            Specific heat ratio, default 7 / 5.

        """
        self.gamma = gamma

    def p_p0(self, M):
        r"""Pressure ratio from Mach number.

        .. math::
            \left ( \frac{P}{P_{0}} \right ) = \left ( \frac{T}{T_{0}} \right )^{\frac{\gamma}{(\gamma - 1)}}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        p_p0 : array_like
            Pressure ratio.

        """

        M = np.asanyarray(M)
        p_p0 = self.T_T0(M) ** (self.gamma / (self.gamma - 1))

        return p_p0

    def rho_rho0(self, M):
        r"""Density ratio from Mach number.

        .. math::
            \left ( \frac{\rho}{\rho_{0}} \right ) = \left ( \frac{T}{T_{0}} \right )^{\frac{1}{(\gamma - 1)}}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        rho_rho0 : array_like
            Density ratio.

        """

        M = np.asanyarray(M)
        rho_rho0 = self.T_T0(M) ** (1 / (self.gamma - 1))
        return rho_rho0

    def T_T0(self, M):
        r"""Temperature ratio from Mach number.

        .. math::
            \left ( \frac{T}{T_{0}} \right ) = \left (1 + \frac{\gamma - 1}{2}M^{2} \right )^{-1}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        T_T0 : array_like
            Temperature ratio.

        """
        M = np.asanyarray(M)
        T_T0 = 1 / (1 + (self.gamma - 1) * M * M / 2)
        return T_T0

    def A_Astar(self, M):
        """Area ratio from Mach number.

        Duct area divided by critial area given Mach number.

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        A_Astar : array_like
            Area ratio.

        """
        M = np.asanyarray(M)
        # If there is any zero entry, NumPy array division gives infinity,
        # which is correct.
        with np.errstate(divide="ignore"):
            A_Astar = (2 / self.T_T0(M) / (self.gamma + 1)) ** (
                (self.gamma + 1) / (2 * (self.gamma - 1))
            ) / M
        return A_Astar

    def a_a0(self, M):
        """ Speed of sound ratio from Mach number.

        Parameters
        ----------
        M: array_like
            Mach number.

        Returns
        -------
        a_a0: array_like
            Speed of sound ratio.
        """

        M = np.asarray(M)
        a_a0 = self.T_T0(M) ** 0.5

        return a_a0

class PrandtlMeyerExpansion(object):
    """Class representing a Prandtl-Meyer expansion fan.

    """

    @staticmethod
    def nu(M, gamma=1.4):
        r"""Prandtl-Meyer angle for a given Mach number.

        The result is given by evaluating the Prandtl-Meyer function.

        .. math::
            \nu = \sqrt{\frac{\gamma + 1}{\gamma - 1}} \tan^{-1}\left [ \sqrt{\frac{\gamma - 1}{\gamma + 1}(M^{2} - 1)} \right ] - \tan^{-1}(\sqrt{M^{2} - 1})

        Parameters
        ----------
        M : float
            Mach number.
        gamma : float, optional
            Specific heat ratio, default 7 / 5.

        Returns
        -------
        nu : float
            Prandtl-Meyer angle, in radians.

        Raises
        ------
        ValueError
            If Mach number is subsonic.

        """
        try:
            with np.errstate(invalid="raise"):
                sgpgm = np.sqrt((gamma + 1) / (gamma - 1))
                nu = sgpgm * np.arctan(np.sqrt(M * M - 1) / sgpgm) - np.arctan(
                    np.sqrt(M * M - 1)
                )
        except FloatingPointError:
            raise ValueError("Mach number must be supersonic")
        return nu

    def __init__(self, M_1, theta, fl=None, gamma=1.4):
        """Constructor of PrandtlMeyerExpansion.

        Parameters
        ----------
        M_1 : float
            Upstream Mach number.
        theta : float
            Deflection angle, in radians.
        fl : IsentropicFlow, optional.
            Flow to be expanded
        gamma : float, optional
            Specific heat ratio, default value = 7 / 5.

        Raises
        ------
        ValueError
            If given Mach number is subsonic.

        """
        if not fl:
            fl = IsentropicFlow(gamma=gamma)
        nu_max = PrandtlMeyerExpansion.nu(np.inf, fl.gamma) - PrandtlMeyerExpansion.nu(
            M_1, fl.gamma
        )
        if theta > nu_max:
            raise ValueError(
                "Deflection angle must be lower than maximum {:.2f}°".format(
                    np.degrees(nu_max)
                )
            )
        self.M_1 = M_1
        self.theta = theta
        self.fl = fl

    @property
    def nu_1(self):
        """Upstream Prandtl-Meyer angle."""
        return PrandtlMeyerExpansion.nu(self.M_1, self.fl.gamma)

    @property
    def nu_2(self):
        """Downstream Prandtl-Meyer angle."""
        return self.nu_1 + self.theta

    @property
    def M_2(self):
        """Downstream Mach number.

        """
        return mach_from_nu(nu=self.nu_2, gamma=self.fl.gamma)

    @property
    def mu_1(self):
        """Angle of forward Mach line.

        """
        return mach_angle(self.M_1)

    @property
    def mu_2(self):
        """Angle of rearward Mach line.

        """
        return mach_angle(self.M_2)

    @property
    def p2_p1(self):
        """Pressure ratio across the expansion fan.

        """
        p2_p1 = self.fl.p_p0(self.M_2) / self.fl.p_p0(self.M_1)
        return p2_p1

    @property
    def T2_T1(self):
        """Temperature ratio across the expansion fan.

        """
        T2_T1 = self.fl.T_T0(self.M_2) / self.fl.T_T0(self.M_1)
        return T2_T1

    @property
    def rho2_rho1(self):
        """Density ratio across the expansion fan.

        """
        return self.p2_p1 / self.T2_T1
<commit_msg>Change namespasce for scipy<commit_after>""" Isentropic properties. """

from __future__ import division, absolute_import

import numpy as np
import scipy as sp
from scipy.optimize import bisect, newton

from skaero.util.decorators import implicit

def mach_angle(M):
    r"""Returns Mach angle given supersonic Mach number.

    .. math::

        \mu = \arcsin{\left ( \frac{1}{M} \right )}

    Parameters
    ----------
    M : float
        Mach number.

    Returns
    -------
    mu : float
        Mach angle.

    Raises
    ------
    ValueError
        If given Mach number is subsonic.

    """
    try:
        with np.errstate(invalid="raise"):
            mu = np.arcsin(1 / M)
    except FloatingPointError:
        raise ValueError("Mach number must be supersonic")
    return mu

def mach_from_area_ratio(A_Astar, fl=None):
    """Computes the Mach number given an area ratio asuming isentropic flow.

    Uses the relation between Mach number and area ratio for isentropic flow,
    and returns both the subsonic and the supersonic solution.

    Parameters
    ----------
    A_Astar : float
        Cross sectional area.
    fl : IsentropicFlow, optional
        Isentropic flow object, default flow with gamma = 7 / 5.

    Returns
    -------
    out : tuple of floats
        Subsonic and supersonic Mach number solution of the equation.

    Raises
    ------
    ValueError
        If the area ratio is less than 1.0 (the critical area is always the
        minimum).

    """
    if not fl:
        fl = IsentropicFlow(gamma=1.4)
    eq = implicit(fl.A_Astar)
    if A_Astar < 1.0:
        raise ValueError("Area ratio must be greater than 1")
    elif A_Astar == 1.0:
        M_sub = M_sup = 1.0
    else:
        M_sub = bisect(eq, 0.0, 1.0, args=(A_Astar,))
        M_sup = newton(eq, 2.0, args=(A_Astar,))

    return M_sub, M_sup

def mach_from_nu(nu, in_radians=True, gamma=1.4):
    r"""Computes the Mach number given a Prandtl-Meyer angle, :math:`\nu`.

    Uses the relation between Mach number and Prandtl-Meyer angle for
    isentropic flow, to iteratively compute and return the Mach number.

    Parameters
    ----------
    nu : float
        Prandtl-Meyer angle, by default in radians.
    in_radians : bool, optional
        When set as False, converts nu from degrees to radians.
    gamma : float, optional
        Specific heat ratio.

    Returns
    -------
    M : float
        Mach number corresponding to :math:`\nu`.

    Raises
    ------
    ValueError
        If :math:`\nu` is 0 or negative or above the theoretical maxima based on
        :math:`\gamma`.

    """
    if not in_radians:
        nu = np.radians(nu)

    nu_max = np.pi / 2.0 * (np.sqrt((gamma + 1.0) / (gamma - 1.0)) - 1)
    if nu <= 0.0 or nu >= nu_max:
        raise ValueError(
            "Prandtl-Meyer angle must be between (0, %f) radians." % nu_max
        )

    eq = implicit(PrandtlMeyerExpansion.nu)
    M = newton(eq, 2.0, args=(nu,))

    return M

class IsentropicFlow(object):
    """Class representing an isentropic gas flow.

    Isentropic flow is characterized by:

    * Viscous and heat conductivity effects are negligible.
    * No chemical or radioactive heat production.

    """

    def __init__(self, gamma=1.4):
        """Constructor of IsentropicFlow.

        Parameters
        ----------
        gamma : float, optional
            Specific heat ratio, default 7 / 5.

        """
        self.gamma = gamma

    def p_p0(self, M):
        r"""Pressure ratio from Mach number.

        .. math::
            \left ( \frac{P}{P_{0}} \right ) = \left ( \frac{T}{T_{0}} \right )^{\frac{\gamma}{(\gamma - 1)}}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        p_p0 : array_like
            Pressure ratio.

        """

        M = np.asanyarray(M)
        p_p0 = self.T_T0(M) ** (self.gamma / (self.gamma - 1))

        return p_p0

    def rho_rho0(self, M):
        r"""Density ratio from Mach number.

        .. math::
            \left ( \frac{\rho}{\rho_{0}} \right ) = \left ( \frac{T}{T_{0}} \right )^{\frac{1}{(\gamma - 1)}}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        rho_rho0 : array_like
            Density ratio.

        """

        M = np.asanyarray(M)
        rho_rho0 = self.T_T0(M) ** (1 / (self.gamma - 1))
        return rho_rho0

    def T_T0(self, M):
        r"""Temperature ratio from Mach number.

        .. math::
            \left ( \frac{T}{T_{0}} \right ) = \left (1 + \frac{\gamma - 1}{2}M^{2} \right )^{-1}

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        T_T0 : array_like
            Temperature ratio.

        """
        M = np.asanyarray(M)
        T_T0 = 1 / (1 + (self.gamma - 1) * M * M / 2)
        return T_T0

    def A_Astar(self, M):
        """Area ratio from Mach number.

        Duct area divided by critial area given Mach number.

        Parameters
        ----------
        M : array_like
            Mach number.

        Returns
        -------
        A_Astar : array_like
            Area ratio.

        """
        M = np.asanyarray(M)
        # If there is any zero entry, NumPy array division gives infinity,
        # which is correct.
        with np.errstate(divide="ignore"):
            A_Astar = (2 / self.T_T0(M) / (self.gamma + 1)) ** (
                (self.gamma + 1) / (2 * (self.gamma - 1))
            ) / M
        return A_Astar

    def a_a0(self, M):
        """ Speed of sound ratio from Mach number.

        Parameters
        ----------
        M: array_like
            Mach number.

        Returns
        -------
        a_a0: array_like
            Speed of sound ratio.
        """

        M = np.asarray(M)
        a_a0 = self.T_T0(M) ** 0.5

        return a_a0

class PrandtlMeyerExpansion(object):
    """Class representing a Prandtl-Meyer expansion fan.

    """

    @staticmethod
    def nu(M, gamma=1.4):
        r"""Prandtl-Meyer angle for a given Mach number.

        The result is given by evaluating the Prandtl-Meyer function.

        .. math::
            \nu = \sqrt{\frac{\gamma + 1}{\gamma - 1}} \tan^{-1}\left [ \sqrt{\frac{\gamma - 1}{\gamma + 1}(M^{2} - 1)} \right ] - \tan^{-1}(\sqrt{M^{2} - 1})

        Parameters
        ----------
        M : float
            Mach number.
        gamma : float, optional
            Specific heat ratio, default 7 / 5.

        Returns
        -------
        nu : float
            Prandtl-Meyer angle, in radians.

        Raises
        ------
        ValueError
            If Mach number is subsonic.

        """
        try:
            with np.errstate(invalid="raise"):
                sgpgm = np.sqrt((gamma + 1) / (gamma - 1))
                nu = sgpgm * np.arctan(np.sqrt(M * M - 1) / sgpgm) - np.arctan(
                    np.sqrt(M * M - 1)
                )
        except FloatingPointError:
            raise ValueError("Mach number must be supersonic")
        return nu

    def __init__(self, M_1, theta, fl=None, gamma=1.4):
        """Constructor of PrandtlMeyerExpansion.

        Parameters
        ----------
        M_1 : float
            Upstream Mach number.
        theta : float
            Deflection angle, in radians.
        fl : IsentropicFlow, optional.
            Flow to be expanded
        gamma : float, optional
            Specific heat ratio, default value = 7 / 5.

        Raises
        ------
        ValueError
            If given Mach number is subsonic.

        """
        if not fl:
            fl = IsentropicFlow(gamma=gamma)
        nu_max = PrandtlMeyerExpansion.nu(np.inf, fl.gamma) - PrandtlMeyerExpansion.nu(
            M_1, fl.gamma
        )
        if theta > nu_max:
            raise ValueError(
                "Deflection angle must be lower than maximum {:.2f}°".format(
                    np.degrees(nu_max)
                )
            )
        self.M_1 = M_1
        self.theta = theta
        self.fl = fl

    @property
    def nu_1(self):
        """Upstream Prandtl-Meyer angle."""
        return PrandtlMeyerExpansion.nu(self.M_1, self.fl.gamma)

    @property
    def nu_2(self):
        """Downstream Prandtl-Meyer angle."""
        return self.nu_1 + self.theta

    @property
    def M_2(self):
        """Downstream Mach number.

        """
        return mach_from_nu(nu=self.nu_2, gamma=self.fl.gamma)

    @property
    def mu_1(self):
        """Angle of forward Mach line.

        """
        return mach_angle(self.M_1)

    @property
    def mu_2(self):
        """Angle of rearward Mach line.

        """
        return mach_angle(self.M_2)

    @property
    def p2_p1(self):
        """Pressure ratio across the expansion fan.

        """
        p2_p1 = self.fl.p_p0(self.M_2) / self.fl.p_p0(self.M_1)
        return p2_p1

    @property
    def T2_T1(self):
        """Temperature ratio across the expansion fan.

        """
        T2_T1 = self.fl.T_T0(self.M_2) / self.fl.T_T0(self.M_1)
        return T2_T1

    @property
    def rho2_rho1(self):
        """Density ratio across the expansion fan.

        """
        return self.p2_p1 / self.T2_T1
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Calls C-Reduce to create a minimal reproducer for clang crashes.

Requires C-Reduce and not (part of LLVM utils) to be installed.
"""

from argparse import ArgumentParser
import os
import re
import stat
import sys
import subprocess
import pipes
from distutils.spawn import find_executable

def create_test(build_script, llvm_not):
  """
  Create an interestingness test from the crash output.
  Return as a string.
  """
  # Get clang call from build script
  # Assumes the call is the last line of the script
  with open(build_script) as f:
    cmd = f.readlines()[-1].rstrip('\n\r')

  # Get crash output
  p = subprocess.Popen(build_script,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT)
  crash_output, _ = p.communicate()

  output = ['#!/bin/bash']
  output.append('%s --crash %s >& t.log || exit 1' % (pipes.quote(llvm_not),
                                                      cmd))

  # Add messages from crash output to the test
  # If there is an Assertion failure, use that; otherwise use the
  # last five stack trace functions
  assertion_re = r'Assertion `([^\']+)\' failed'
  assertion_match = re.search(assertion_re, crash_output)
  if assertion_match:
    msg = assertion_match.group(1)
    output.append('grep %s t.log || exit 1' % pipes.quote(msg))
  else:
    stacktrace_re = r'#[0-9]+\s+0[xX][0-9a-fA-F]+\s*([^(]+)\('
    matches = re.findall(stacktrace_re, crash_output)
    del matches[:-5]
    output += ['grep %s t.log || exit 1' % pipes.quote(msg) for msg in matches]

  return output

def main():
  parser = ArgumentParser(description=__doc__)
  parser.add_argument('build_script', type=str, nargs=1,
                      help='Name of the script that generates the crash.')
  parser.add_argument('file_to_reduce', type=str, nargs=1,
                      help='Name of the file to be reduced.')
  parser.add_argument('-o', '--output', dest='output', type=str,
                      help='Name of the output file for the reduction. Optional.')
  parser.add_argument('--llvm-not', dest='llvm_not', type=str,
                      help="The path to the llvm-not executable. "
                      "Required if 'not' is not in PATH environment.");
  parser.add_argument('--creduce', dest='creduce', type=str,
                      help="The path to the C-Reduce executable. "
                      "Required if 'creduce' is not in PATH environment.");
  args = parser.parse_args()

  build_script = os.path.abspath(args.build_script[0])
  file_to_reduce = os.path.abspath(args.file_to_reduce[0])
  llvm_not = (find_executable(args.llvm_not) if args.llvm_not else
              find_executable('not'))
  creduce = (find_executable(args.creduce) if args.creduce else
             find_executable('creduce'))

  if not os.path.isfile(build_script):
    print(("ERROR: input file '%s' does not exist") % build_script)
    return 1

  if not os.path.isfile(file_to_reduce):
    print(("ERROR: input file '%s' does not exist") % file_to_reduce)
    return 1

  if not llvm_not:
    parser.print_help()
    return 1

  if not creduce:
    parser.print_help()
    return 1

  # Write interestingness test to file
  test_contents = create_test(build_script, llvm_not)
  testname, _ = os.path.splitext(file_to_reduce)
  testfile = testname + '.test.sh'
  with open(testfile, 'w') as f:
    f.write('\n'.join(test_contents))
  os.chmod(testfile, os.stat(testfile).st_mode | stat.S_IEXEC)

  # Confirm that the interestingness test passes
  try:
    with open(os.devnull, 'w') as devnull:
      subprocess.check_call(testfile, stdout=devnull)
  except subprocess.CalledProcessError:
    print("For some reason the interestingness test does not return zero")
    return 1

  # FIXME: try running clang preprocessor first

  try:
    p = subprocess.Popen([creduce, testfile, file_to_reduce])
    p.communicate()
  except KeyboardInterrupt:
    # Hack to kill C-Reduce because it jumps into its own pgid
    print('\n\nctrl-c detected, killed creduce')
    p.kill()

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>creduce-clang-crash.py: preprocess file + reduce commandline<commit_after>#!/usr/bin/env python
"""Calls C-Reduce to create a minimal reproducer for clang crashes.
"""

from argparse import ArgumentParser
import os
import re
import stat
import sys
import subprocess
import pipes
import shlex
import tempfile
import shutil
from distutils.spawn import find_executable

verbose = False
llvm_bin = None
creduce_cmd = None
not_cmd = None

def check_file(fname):
  if not os.path.isfile(fname):
    sys.exit("ERROR: %s does not exist" % (fname))
  return fname

def check_cmd(cmd_name, cmd_dir, cmd_path=None):
  """
  Returns absolute path to cmd_path if it is given,
  or absolute path to cmd_dir/cmd_name.
  """
  if cmd_path:
    cmd = find_executable(cmd_path)
    if cmd:
      return cmd
    sys.exit("ERROR: executable %s not found" % (cmd_path))

  cmd = find_executable(cmd_name, path=cmd_dir)
  if cmd:
    return cmd
  sys.exit("ERROR: %s not found in %s" % (cmd_name, cmd_dir))

def quote_cmd(cmd):
  return ' '.join(arg if arg.startswith('$') else pipes.quote(arg)
                  for arg in cmd)

def get_crash_cmd(crash_script):
  with open(crash_script) as f:
    # Assume clang call is on the last line of the script
    line = f.readlines()[-1]
    cmd = shlex.split(line)

    # Overwrite the script's clang with the user's clang path
    new_clang = check_cmd('clang', llvm_bin)
    cmd[0] = pipes.quote(new_clang)
    return cmd

def has_expected_output(crash_cmd, expected_output):
  p = subprocess.Popen(crash_cmd,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT)
  crash_output, _ = p.communicate()
  return all(msg in crash_output for msg in expected_output)

def get_expected_output(crash_cmd):
  p = subprocess.Popen(crash_cmd,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT)
  crash_output, _ = p.communicate()

  # If there is an assertion failure, use that;
  # otherwise use the last five stack trace functions
  assertion_re = r'Assertion `([^\']+)\' failed'
  assertion_match = re.search(assertion_re, crash_output)
  if assertion_match:
    return [assertion_match.group(1)]
  else:
    stacktrace_re = r'#[0-9]+\s+0[xX][0-9a-fA-F]+\s*([^(]+)\('
    matches = re.findall(stacktrace_re, crash_output)
    return matches[-5:]

def write_interestingness_test(testfile, crash_cmd, expected_output,
                               file_to_reduce):
  filename = os.path.basename(file_to_reduce)
  if filename not in crash_cmd:
    sys.exit("ERROR: expected %s to be in the crash command" % filename)

  # Replace all instances of file_to_reduce with a command line variable
  output = ['#!/bin/bash',
            'if [ -z "$1" ] ; then',
            '  f=%s' % (pipes.quote(filename)),
            'else',
            '  f="$1"',
            'fi']
  cmd = ['$f' if s == filename else s for s in crash_cmd]

  output.append('%s --crash %s >& t.log || exit 1' % (pipes.quote(not_cmd),
                                                      quote_cmd(cmd)))

  for msg in expected_output:
    output.append('grep %s t.log || exit 1' % pipes.quote(msg))

  with open(testfile, 'w') as f:
    f.write('\n'.join(output))
  os.chmod(testfile, os.stat(testfile).st_mode | stat.S_IEXEC)

def check_interestingness(testfile, file_to_reduce):
  testfile = os.path.abspath(testfile)

  # Check that the test considers the original file interesting
  with open(os.devnull, 'w') as devnull:
    returncode = subprocess.call(testfile, stdout=devnull)
  if returncode:
    sys.exit("The interestingness test does not pass for the original file.")

  # Check that an empty file is not interesting
  _, empty_file = tempfile.mkstemp()
  with open(os.devnull, 'w') as devnull:
    returncode = subprocess.call([testfile, empty_file], stdout=devnull)
  os.remove(empty_file)
  if not returncode:
    sys.exit("The interestingness test passes for an empty file.")

def clang_preprocess(file_to_reduce, crash_cmd, expected_output):
  _, tmpfile = tempfile.mkstemp()
  shutil.copy(file_to_reduce, tmpfile)

  cmd = crash_cmd + ['-E', '-P']
  p = subprocess.Popen(cmd,
                       stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT)
  preprocessed, _ = p.communicate()

  with open(file_to_reduce, 'w') as f:
    f.write(preprocessed)

  if has_expected_output(crash_cmd, expected_output):
    if verbose:
      print("Successfuly preprocessed with %s" % (quote_cmd(cmd)))
    os.remove(tmpfile)
  else:
    if verbose:
      print("Failed to preprocess with %s" % (quote_cmd(cmd)))
    shutil.move(tmpfile, file_to_reduce)

def filter_args(args, opts_startswith=[]):
  result = [arg for arg in args if all(not arg.startswith(a) for a in
                                       opts_startswith)]
  return result

def try_remove_args(cmd, expected_output, msg=None, extra_arg=None, **kwargs):
  new_cmd = filter_args(cmd, **kwargs)
  if extra_arg and extra_arg not in new_cmd:
    new_cmd.append(extra_arg)
  if new_cmd != cmd and has_expected_output(new_cmd, expected_output):
    if msg and verbose:
      print(msg)
    return new_cmd
  return cmd

def simplify_crash_cmd(crash_cmd, expected_output):
  new_cmd = try_remove_args(crash_cmd, expected_output,
                            msg="Removed debug info options",
                            opts_startswith=["-gcodeview",
                                             "-dwarf-column-info",
                                             "-debug-info-kind=",
                                             "-debugger-tuning=",
                                             "-gdwarf"])
  new_cmd = try_remove_args(new_cmd, expected_output,
                            msg="Replaced -W options with -w",
                            extra_arg='-w',
                            opts_startswith=["-W"])
  new_cmd = try_remove_args(new_cmd, expected_output,
                            msg="Replaced optimization level with -O0",
                            extra_arg="-O0",
                            opts_startswith=["-O"])
  return new_cmd

def main():
  global verbose
  global llvm_bin
  global creduce_cmd
  global not_cmd

  parser = ArgumentParser(description=__doc__)
  parser.add_argument('crash_script', type=str, nargs=1,
                      help="Name of the script that generates the crash.")
  parser.add_argument('file_to_reduce', type=str, nargs=1,
                      help="Name of the file to be reduced.")
  parser.add_argument('--llvm-bin', dest='llvm_bin', type=str,
                      required=True, help="Path to the LLVM bin directory.")
  parser.add_argument('--llvm-not', dest='llvm_not', type=str,
                      help="The path to the `not` executable. "
                      "By default uses the llvm-bin directory.")
  parser.add_argument('--creduce', dest='creduce', type=str,
                      help="The path to the `creduce` executable. "
                      "Required if `creduce` is not in PATH environment.")
  parser.add_argument('-v', '--verbose', action='store_true')
  args = parser.parse_args()

  verbose = args.verbose
  llvm_bin = os.path.abspath(args.llvm_bin)
  creduce_cmd = check_cmd('creduce', None, args.creduce)
  not_cmd = check_cmd('not', llvm_bin, args.llvm_not)
  crash_script = check_file(args.crash_script[0])
  file_to_reduce = check_file(args.file_to_reduce[0])

  print("\nParsing the crash script and getting expected output...")
  crash_cmd = get_crash_cmd(crash_script)

  expected_output = get_expected_output(crash_cmd)
  if len(expected_output) < 1:
    sys.exit("ERROR: no crash was found")

  print("\nSimplifying the crash command...")
  crash_cmd = simplify_crash_cmd(crash_cmd, expected_output)

  print("\nWriting interestingness test to file...")
  testfile = os.path.splitext(file_to_reduce)[0] + '.test.sh'
  write_interestingness_test(testfile, crash_cmd, expected_output,
                             file_to_reduce)
  check_interestingness(testfile, file_to_reduce)

  print("\nPreprocessing the file to reduce...")
  clang_preprocess(file_to_reduce, crash_cmd, expected_output)

  print("\nRunning C-Reduce...")
  try:
    p = subprocess.Popen([creduce_cmd, testfile, file_to_reduce])
    p.communicate()
  except KeyboardInterrupt:
    # Hack to kill C-Reduce because it jumps into its own pgid
    print('\n\nctrl-c detected, killed creduce')
    p.kill()

  # FIXME: reduce the clang crash command

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>from django.test import TestCase
from django.core.urlresolvers import reverse
from django.test.client import Client

from wagtail.wagtailsearch.backends import get_search_backend

from molo.core.models import ArticlePage

class TestSearch(TestCase):

    def test_search(self):
        self.backend = get_search_backend('default')
        self.backend.reset_index()

        for a in range(0, 20):
            ArticlePage.objects.create(
                title='article %s' % (a,), depth=a,
                subtitle='article %s subtitle' % (a,),
                slug='article-%s' % (a,), path=[a])

        self.backend.refresh_index()

        client = Client()
        response = client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'Page 1 of 2')
        self.assertContains(response, '→')
        self.assertNotContains(response, '←')

        response = client.get(reverse('search'), {
            'q': 'article',
            'p': '2',
        })
        self.assertContains(response, 'Page 2 of 2')
        self.assertNotContains(response, '→')
        self.assertContains(response, '←')

        response = client.get(reverse('search'), {
            'q': 'article',
            'p': 'foo',
        })
        self.assertContains(response, 'Page 1 of 2')

        response = client.get(reverse('search'), {
            'q': 'article',
            'p': '4',
        })
        self.assertContains(response, 'Page 2 of 2')

        response = client.get(reverse('search'), {
            'q': 'magic'
        })
        self.assertContains(response, 'No search results for magic')

        response = client.get(reverse('search'))
        self.assertContains(response, 'No search results for None')
<commit_msg>added tests<commit_after>from django.test import TestCase
from django.core.urlresolvers import reverse
from django.test.client import Client

from wagtail.wagtailsearch.backends import get_search_backend

from molo.core.models import ArticlePage, SiteLanguage
from molo.core.tests.base import MoloTestCaseMixin

class TestSearch(TestCase, MoloTestCaseMixin):

    def setUp(self):
        self.client = Client()
        # Creates Main language
        self.english = SiteLanguage.objects.create(
            locale='en',
        )
        # Creates translation Language
        self.french = SiteLanguage.objects.create(
            locale='fr',
        )
        self.mk_main()

        # Creates a section under the index page
        self.english_section = self.mk_section(
            self.section_index, title='English section')

    def test_search(self):
        self.backend = get_search_backend('default')
        self.backend.reset_index()

        for a in range(0, 20):
            ArticlePage.objects.create(
                title='article %s' % (a,), depth=a,
                subtitle='article %s subtitle' % (a,),
                slug='article-%s' % (a,), path=[a])

        self.backend.refresh_index()

        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'Page 1 of 2')
        self.assertContains(response, '→')
        self.assertNotContains(response, '←')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': '2',
        })
        self.assertContains(response, 'Page 2 of 2')
        self.assertNotContains(response, '→')
        self.assertContains(response, '←')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': 'foo',
        })
        self.assertContains(response, 'Page 1 of 2')

        response = self.client.get(reverse('search'), {
            'q': 'article',
            'p': '4',
        })
        self.assertContains(response, 'Page 2 of 2')

        response = self.client.get(reverse('search'), {
            'q': 'magic'
        })
        self.assertContains(response, 'No search results for magic')

        response = self.client.get(reverse('search'))
        self.assertContains(response, 'No search results for None')

    def test_search_works_with_multilanguages(self):
        self.backend = get_search_backend('default')
        self.backend.reset_index()
        eng_article = self.mk_article(
            self.english_section, title="English article")

        self.mk_article_translation(
            eng_article, self.french, title='French article')

        self.backend.refresh_index()

        self.client.get('/locale/en/')
        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        print response
        self.assertContains(response, 'English article')
        self.assertNotContains(response, 'French article')

        self.client.get('/locale/fr/')
        response = self.client.get(reverse('search'), {
            'q': 'article'
        })
        self.assertContains(response, 'French article')
        self.assertNotContains(response, 'English article')
<|endoftext|>


<commit_before>#!/usr/bin/python

# This tool helps assess the impact of automatically applying
# heuristics that omit 'needless' words from APIs imported from Clang
# into Swift.

import getopt
import sys
import subprocess

# Print help
def help():
    print('omit-needless-words.py [options] -m <modulename>')
    print('')
    print('Summary:')
    print("\tDetermines the effects of omitting 'needless' words from imported APIs")
    print('')
    print('Options:')
    print('\t-s <sdkname>\t\t\tThe SDK to use (e.g., macosx)')
    print("\t--sdk=<sdkname>'\t\tDefaults to 'macosx'")
    print('')
    print('\t-t <triple>\t\t\tThe target triple to use (e.g., x86_64-apple-macosx10.10)')
    print("\t--target=<triple>'\t\tDefaults to 'x86_64-apple-macosx10.10'")
    print('')
    print('\t-i <executable>\t\t\tThe swift-ide-test executable')
    print("\t--swift-ide-test=<executable>\tDefaults to 'swift-ide-test'")
    print('')
    print('\t-d <executable>\t\t\tThe tool to use to diff the results')
    print("\t--diff_tool=<executable>\tDefaults to 'opendiff'")
    print('')
    print('Examples:')
    print('\tpython omit-needless-words.py -m AppKit')

# Configuration information
sdk = 'macosx'
target = ''
module = ''
source_filename = 'omit-needless-words.swift'
swift_ide_test = 'swift-ide-test'
diff_tool = 'opendiff'

# Parse command-line arguments.
try:
    opts, args = getopt.getopt(sys.argv[1:], 'hs:t:m:i:d:',
                               ['help', 'sdk=', 'target=', 'module=',
                                'swift-ide-test=','diff_tool='])
except getopt.GetoptError:
    help()
    sys.exit(2)

for opt, arg in opts:
    if opt in ('-h', '--help'):
        help()
        sys.exit()

    if opt in ('-s', '--sdk'):
        sdk = arg
        continue

    if opt in ('-t', '--target'):
        target = arg
        continue

    if opt in ('-m', '--module'):
        module = arg
        continue

    if opt in ('-i', '--swift-ide-test'):
        swift_ide_test = arg
        continue

    if opt in ('-d', '--diff_tool'):
        diff_tool = arg
        continue

    help()
    sys.exit(2)

if module == '':
    help()
    sys.exit(2)

if target == '':
    if sdk in ('macosx', 'macosx.internal'):
        target = 'x86_64-apple-macosx10.10'
    if sdk in ('iphoneos', 'iphoneos.internal'):
        target = 'arm64-apple-ios8.0'
    if sdk == 'iphonesimulator':
        target = 'x86_64-apple-ios8.0'
    if sdk in ('watchos', 'watchos.internal'):
        target = 'armv7k-apple-watchos2.0'
    if sdk == 'watchos.simulator':
        target = 'i386-apple-watchos2.0'

# Figure out the SDK root for the requested SDK
sdkroot = subprocess.check_output(['xcrun', '--show-sdk-path', '--sdk', sdk]).rstrip()
print('SDK Root = %s' % (sdkroot))

swift_ide_test_cmd = [swift_ide_test, '-print-module', '-source-filename', source_filename, '-sdk', sdkroot, '-target', target, '-module-print-skip-overlay', '-skip-unavailable', '-module-print-submodules', '-skip-parameter-names', '-enable-infer-default-arguments', '-module-to-print=%s' % (module)]
omit_needless_words_args = ['-enable-omit-needless-words']

# Determine the output files.
before_filename = '%s.before.txt' % (module)
after_filename = '%s.after.txt' % (module)

# Create a .swift file we can feed into swift-ide-test
subprocess.call(['touch', source_filename])

# Print the interface without omitting needless words
print('Writing %s...' % before_filename)
before_file = open(before_filename, 'w')
subprocess.call(swift_ide_test_cmd, stdout=before_file)
before_file.close()

# Print the interface omitting needless words
print('Writing %s...' % after_filename)
after_file = open(after_filename, 'w')
subprocess.call(swift_ide_test_cmd + omit_needless_words_args, stdout=after_file)
after_file.close()

# Remove the .swift file we fed into swift-ide-test
subprocess.call(['rm', '-f', source_filename])

# Diff them.
if diff_tool != '':
    subprocess.call([diff_tool, before_filename, after_filename])
<commit_msg>Teach the omit-needless-words script about the tvOS SDK.<commit_after>#!/usr/bin/python

# This tool helps assess the impact of automatically applying
# heuristics that omit 'needless' words from APIs imported from Clang
# into Swift.

import getopt
import sys
import subprocess

# Print help
def help():
    print('omit-needless-words.py [options] -m <modulename>')
    print('')
    print('Summary:')
    print("\tDetermines the effects of omitting 'needless' words from imported APIs")
    print('')
    print('Options:')
    print('\t-s <sdkname>\t\t\tThe SDK to use (e.g., macosx)')
    print("\t--sdk=<sdkname>'\t\tDefaults to 'macosx'")
    print('')
    print('\t-t <triple>\t\t\tThe target triple to use (e.g., x86_64-apple-macosx10.10)')
    print("\t--target=<triple>'\t\tDefaults to 'x86_64-apple-macosx10.10'")
    print('')
    print('\t-i <executable>\t\t\tThe swift-ide-test executable')
    print("\t--swift-ide-test=<executable>\tDefaults to 'swift-ide-test'")
    print('')
    print('\t-d <executable>\t\t\tThe tool to use to diff the results')
    print("\t--diff_tool=<executable>\tDefaults to 'opendiff'")
    print('')
    print('Examples:')
    print('\tpython omit-needless-words.py -m AppKit')

# Configuration information
sdk = 'macosx'
target = ''
module = ''
source_filename = 'omit-needless-words.swift'
swift_ide_test = 'swift-ide-test'
diff_tool = 'opendiff'

# Parse command-line arguments.
try:
    opts, args = getopt.getopt(sys.argv[1:], 'hs:t:m:i:d:',
                               ['help', 'sdk=', 'target=', 'module=',
                                'swift-ide-test=','diff_tool='])
except getopt.GetoptError:
    help()
    sys.exit(2)

for opt, arg in opts:
    if opt in ('-h', '--help'):
        help()
        sys.exit()

    if opt in ('-s', '--sdk'):
        sdk = arg
        continue

    if opt in ('-t', '--target'):
        target = arg
        continue

    if opt in ('-m', '--module'):
        module = arg
        continue

    if opt in ('-i', '--swift-ide-test'):
        swift_ide_test = arg
        continue

    if opt in ('-d', '--diff_tool'):
        diff_tool = arg
        continue

    help()
    sys.exit(2)

if module == '':
    help()
    sys.exit(2)

if target == '':
    if sdk in ('macosx', 'macosx.internal'):
        target = 'x86_64-apple-macosx10.10'
    if sdk in ('iphoneos', 'iphoneos.internal'):
        target = 'arm64-apple-ios8.0'
    if sdk == 'iphonesimulator':
        target = 'x86_64-apple-ios8.0'
    if sdk in ('watchos', 'watchos.internal'):
        target = 'armv7k-apple-watchos2.0'
    if sdk == 'watchos.simulator':
        target = 'i386-apple-watchos2.0'
    if sdk in ('appletvos', 'appletvos.internal'):
        target = 'arm64-apple-tvos9'
    if sdk == 'appletvos.simulator':
        target = 'x86_64-apple-tvos9'

# Figure out the SDK root for the requested SDK
sdkroot = subprocess.check_output(['xcrun', '--show-sdk-path', '--sdk', sdk]).rstrip()
print('SDK Root = %s' % (sdkroot))

swift_ide_test_cmd = [swift_ide_test, '-print-module', '-source-filename', source_filename, '-sdk', sdkroot, '-target', target, '-module-print-skip-overlay', '-skip-unavailable', '-module-print-submodules', '-skip-parameter-names', '-enable-infer-default-arguments', '-module-to-print=%s' % (module)]
omit_needless_words_args = ['-enable-omit-needless-words']

# Determine the output files.
before_filename = '%s.before.txt' % (module)
after_filename = '%s.after.txt' % (module)

# Create a .swift file we can feed into swift-ide-test
subprocess.call(['touch', source_filename])

# Print the interface without omitting needless words
print('Writing %s...' % before_filename)
before_file = open(before_filename, 'w')
subprocess.call(swift_ide_test_cmd, stdout=before_file)
before_file.close()

# Print the interface omitting needless words
print('Writing %s...' % after_filename)
after_file = open(after_filename, 'w')
subprocess.call(swift_ide_test_cmd + omit_needless_words_args, stdout=after_file)
after_file.close()

# Remove the .swift file we fed into swift-ide-test
subprocess.call(['rm', '-f', source_filename])

# Diff them.
if diff_tool != '':
    subprocess.call([diff_tool, before_filename, after_filename])
<|endoftext|>


<commit_before>from will.plugin import WillPlugin
from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template

class MarketingPlugin(WillPlugin):

    @route("/marketing-deployed/<success_code>")
    def deploy_output(self, success_code):
        if success_code == 0:
            self.say("Marketing site deployed successfully: http://gk-marketing.s3-website-us-east-1.amazonaws.com/")
        else:
            self.say("Error deploying marketing site.")
        return "Success"

<commit_msg>Success code is a string.<commit_after>from will.plugin import WillPlugin
from will.decorators import respond_to, periodic, hear, randomly, route, rendered_template

class MarketingPlugin(WillPlugin):

    @route("/marketing-deployed/<success_code>")
    def deploy_output(self, success_code):
        if success_code == "0":
            self.say("Marketing site deployed successfully: http://gk-marketing.s3-website-us-east-1.amazonaws.com/")
        else:
            self.say("Error deploying marketing site.")
        return "Success"

<|endoftext|>


<commit_before>"""
This code includes various non-linear activation layer classes.
"""

import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as MRG
from collections import OrderedDict
from lemontree.layers.layer import BaseLayer

class ReLU(BaseLayer):
    """
    This class implements ReLU activation function.
    """
    def __init__(self, alpha=0, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        alpha: float, default: 0
            a positive float value which indicates the tangent of x < 0 range.
            if alpha is not 0, this function become a leaky ReLU.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(ReLU, self).__init__(name)
        # check asserts
        assert alpha >= 0, '"alpha" should be a non-negative float value.'

        # set members
        self.alpha = alpha

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Creates symbolic function to compute output from an input.
        ReLU is element-wise operation.

        Math Expression
        -------------------
        y = maximum(x, 0)
        y = ifelse(x > 0, x, \alpha * x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.relu(input_, self.alpha)

class Linear(BaseLayer):
    """
    This class implements Linear activation function.
    Not non-linear, just return itself.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Linear, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Return input itself.

        Math Expression
        -------------------
        y = maximum(x, 0)
        y = ifelse(x > 0, x, \alpha * x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return input_

class Tanh(BaseLayer):
    """
    This class implements tanh activation function.
    """
    def __init__(self, name = None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Tanh, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Tanh is element-wise operation.

        Math Expression
        -------------------
        y = tanh(x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.tanh(input_)

class ELU(BaseLayer):
    """
    This class implements ELU activation function.
    """
    def __init__(self, alpha=1, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        alpha: float, default: 1
            a positive float value which indicates the tangent of x < 0 range.
            if alpha is not 0, this function become a leaky ReLU.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        # check asserts
        super(ELU, self).__init__(name)
        assert alpha >= 0, '"alpha" should be a non-negative float value.'
        self.alpha = alpha

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        ELU is element-wise operation.
        If alpha = 0, same as ReLU.

        Math Expression
        -------------------
        y = ifelse(x > 0, x, \alpha * (exp(x) - 1))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.switch(T.gt(input_, 0), input_, self.alpha * (T.exp(input_) - 1))

class Sigmoid(BaseLayer):
    """
    This class implements Sigmoid activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Sigmoid, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Sigmoid is element-wise operation.

        Math Expression
        -------------------
        y = 1 / (1 + exp(-x)) = exp(x) / (1 + exp(x))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.sigmoid(input_)

class HardSigmoid(BaseLayer):
    """
    This class implements Hard sigmoid activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(HardSigmoid, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Sigmoid is element-wise operation.

        Math Expression
        -------------------
        y = 1 / (1 + exp(-x)) = exp(x) / (1 + exp(x))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.hard_sigmoid(input_)

class Softmax(BaseLayer):
    """
    This class implements softmax activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Softmax, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        y_k = exp(x_k) / \sum(exp(x_i))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.softmax(input_)

class DistilationSoftmax(BaseLayer):
    """
    This class implements distillation softmax activation function.
    Softmax distillation is widely used for sharpen / soften distribution.
    Also used for knowledge transfer.
    """
    def __init__(self, temperature_init=1.0, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        temperature_init: float, default: 1
            a positive float value.
            if T > 1, become more soften, and T < 1, become more sharpen.
            if temperature is 1, same as normal softmax.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(DistilationSoftmax, self).__init__(name)
        # check asserts
        assert temperature_init > 0, '"temperature_init" should be a positive float.'

        # set members
        temperature = np.array(temperature_init).astype('float32')
        self.temperature = theano.shared(temperature, 'temperature')
        self.temperature.tags = ['temperature']

    def change_temperature(self, new_temperature):
        """
        This function changes the temperature for softmax.

        Parameters
        ----------
        new_temperature: float
            a positive float value which will be a new temperature.

        Returns
        -------
        None.
        """
        # check asserts
        assert new_temperature > 0, '"new_temperature" should be a positive float.'

        self.temperature.set_value(float(new_temperature))

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        y_k = exp(x_k / T) / \sum(exp(x_i / T))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.softmax(input_ / self.temperature)  # divide by temperature

class GumbelSoftmax(BaseLayer):
    """
    This class implements gumbel softmax activation.
    See "Categorical Reparameterization with Gumbel-softmax".
    (Eric Jang, Shixiang Gu, Ben Poole, 2016.)
    """
    def __init__(self, temperature_init=0.1, seed=9683, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        temperature_init: float, default: 1
            a positive float value.
            if T > 1, become more soften, and T < 1, become more sharpen.
            if temperature is 1, same as normal softmax.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(GumbelSoftmax, self).__init__(name)
        # check asserts
        assert temperature_init > 0, '"temperature_init" should be a positive float.'

        # set members
        temperature = np.array(temperature_init).astype('float32')
        self.temperature = theano.shared(temperature, 'temperature')
        self.temperature.tags = ['temperature']
        self.rng = MRG(seed)

    def change_temperature(self, new_temperature):
        """
        This function changes the temperature for softmax.

        Parameters
        ----------
        new_temperature: float
            a positive float value which will be a new temperature.

        Returns
        -------
        None.
        """
        # check asserts
        assert new_temperature > 0, '"new_temperature" should be a positive float.'

        self.temperature.set_value(float(new_temperature))

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        g_k ~ Gumbel(0, 1)
        y_k = exp((x_k + g_k) / T) / \sum(exp((x_i + g_k) / T))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        # generate random gumbel distribution
        uniform_random = self.rng.uniform(input_.shape, 0, 1)
        gumbel_random = -T.log(-T.log(uniform_random + 1e-7) + 1e-7)
        return T.nnet.softmax((input_ + gumbel_random) / self.temperature)  # divide by temperature<commit_msg>gumbel<commit_after>"""
This code includes various non-linear activation layer classes.
"""

import numpy as np
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as MRG
from collections import OrderedDict
from lemontree.layers.layer import BaseLayer

class ReLU(BaseLayer):
    """
    This class implements ReLU activation function.
    """
    def __init__(self, alpha=0, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        alpha: float, default: 0
            a positive float value which indicates the tangent of x < 0 range.
            if alpha is not 0, this function become a leaky ReLU.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(ReLU, self).__init__(name)
        # check asserts
        assert alpha >= 0, '"alpha" should be a non-negative float value.'

        # set members
        self.alpha = alpha

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Creates symbolic function to compute output from an input.
        ReLU is element-wise operation.

        Math Expression
        -------------------
        y = maximum(x, 0)
        y = ifelse(x > 0, x, \alpha * x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.relu(input_, self.alpha)

class Linear(BaseLayer):
    """
    This class implements Linear activation function.
    Not non-linear, just return itself.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Linear, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Return input itself.

        Math Expression
        -------------------
        y = maximum(x, 0)
        y = ifelse(x > 0, x, \alpha * x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return input_

class Tanh(BaseLayer):
    """
    This class implements tanh activation function.
    """
    def __init__(self, name = None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Tanh, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Tanh is element-wise operation.

        Math Expression
        -------------------
        y = tanh(x)

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.tanh(input_)

class ELU(BaseLayer):
    """
    This class implements ELU activation function.
    """
    def __init__(self, alpha=1, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        alpha: float, default: 1
            a positive float value which indicates the tangent of x < 0 range.
            if alpha is not 0, this function become a leaky ReLU.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        # check asserts
        super(ELU, self).__init__(name)
        assert alpha >= 0, '"alpha" should be a non-negative float value.'
        self.alpha = alpha

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        ELU is element-wise operation.
        If alpha = 0, same as ReLU.

        Math Expression
        -------------------
        y = ifelse(x > 0, x, \alpha * (exp(x) - 1))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.switch(T.gt(input_, 0), input_, self.alpha * (T.exp(input_) - 1))

class Sigmoid(BaseLayer):
    """
    This class implements Sigmoid activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Sigmoid, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Sigmoid is element-wise operation.

        Math Expression
        -------------------
        y = 1 / (1 + exp(-x)) = exp(x) / (1 + exp(x))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.sigmoid(input_)

class HardSigmoid(BaseLayer):
    """
    This class implements Hard sigmoid activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(HardSigmoid, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Sigmoid is element-wise operation.

        Math Expression
        -------------------
        y = 1 / (1 + exp(-x)) = exp(x) / (1 + exp(x))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.hard_sigmoid(input_)

class Softmax(BaseLayer):
    """
    This class implements softmax activation function.
    """
    def __init__(self, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(Softmax, self).__init__(name)

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        y_k = exp(x_k) / \sum(exp(x_i))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.softmax(input_)

class DistilationSoftmax(BaseLayer):
    """
    This class implements distillation softmax activation function.
    Softmax distillation is widely used for sharpen / soften distribution.
    Also used for knowledge transfer.
    """
    def __init__(self, temperature_init=1.0, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        temperature_init: float, default: 1
            a positive float value.
            if T > 1, become more soften, and T < 1, become more sharpen.
            if temperature is 1, same as normal softmax.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(DistilationSoftmax, self).__init__(name)
        # check asserts
        assert temperature_init > 0, '"temperature_init" should be a positive float.'

        # set members
        temperature = np.array(temperature_init).astype('float32')
        self.temperature = theano.shared(temperature, 'temperature')
        self.temperature.tags = ['temperature']

    def change_temperature(self, new_temperature):
        """
        This function changes the temperature for softmax.

        Parameters
        ----------
        new_temperature: float
            a positive float value which will be a new temperature.

        Returns
        -------
        None.
        """
        # check asserts
        assert new_temperature > 0, '"new_temperature" should be a positive float.'

        self.temperature.set_value(float(new_temperature))

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        y_k = exp(x_k / T) / \sum(exp(x_i / T))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        return T.nnet.softmax(input_ / self.temperature)  # divide by temperature

class GumbelSoftmax(BaseLayer):
    """
    This class implements gumbel softmax activation.
    See "Categorical Reparameterization with Gumbel-softmax".
    (Eric Jang, Shixiang Gu, Ben Poole, 2016.)
    """
    def __init__(self, shape, temperature_init=0.1, seed=9683, name=None):
        """
        This function initializes the class.

        Parameters
        ----------
        shape: tuple
            a tuple of shape of gumbel random distribution.
            this is required for scan to not affected.
        temperature_init: float, default: 1
            a positive float value.
            if T > 1, become more soften, and T < 1, become more sharpen.
            if temperature is 1, same as normal softmax.
        name: string
            a string name of this class.

        Returns
        -------
        None.
        """
        super(GumbelSoftmax, self).__init__(name)
        # check asserts
        assert isinstance(shape, tuple), '"shape" should be a tuple of shape.'
        assert temperature_init > 0, '"temperature_init" should be a positive float.'

        # set members
        self.shape = shape
        temperature = np.array(temperature_init).astype('float32')
        self.temperature = theano.shared(temperature, 'temperature')
        self.temperature.tags = ['temperature']
        self.rng = MRG(seed)

    def change_temperature(self, new_temperature):
        """
        This function changes the temperature for softmax.

        Parameters
        ----------
        new_temperature: float
            a positive float value which will be a new temperature.

        Returns
        -------
        None.
        """
        # check asserts
        assert new_temperature > 0, '"new_temperature" should be a positive float.'

        self.temperature.set_value(float(new_temperature))

    def get_output(self, input_):
        """
        This function overrides the parents' one.
        Softmax converts output energy to probability distributuion.

        Math Expression
        -------------------
        g_k ~ Gumbel(0, 1)
        y_k = exp((x_k + g_k) / T) / \sum(exp((x_i + g_k) / T))

        Parameters
        ----------
        input_: TensorVariable

        Returns
        -------
        TensorVariable
        """
        # generate random gumbel distribution
        uniform_random = self.rng.uniform(self.shape, 0, 1)
        gumbel_random = -T.log(-T.log(uniform_random + 1e-7) + 1e-7)
        return T.nnet.softmax((input_ + gumbel_random) / self.temperature)  # divide by temperature<|endoftext|>


<commit_before>import os
import uuid
import json
from hashlib import sha256
from itertools import chain
from collections import OrderedDict, defaultdict
import threading
from tempfile import NamedTemporaryFile
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

import rasterio
import pycurl
import numpy as np

import gbdxtools as gbdx
from gbdxtools.ipe.util import IPE_TO_DTYPE
from gbdxtools.ipe.graph import VIRTUAL_IPE_URL, register_ipe_graph, get_ipe_metadata
from gbdxtools.images.meta import DaskMeta
from gbdxtools.auth import Auth

import warnings
warnings.filterwarnings('ignore')

_curl_pool = defaultdict(pycurl.Curl)
MAX_RETRIES = 5
try:
    basestring
except NameError:
    basestring = str

try:
    xrange
except NameError:
    xrange = range

NAMESPACE_UUID = uuid.NAMESPACE_DNS

def load_url(url, token, shape=(8, 256, 256)):
    """ Loads a geotiff url inside a thread and returns as an ndarray """
    thread_id = threading.current_thread().ident
    _curl = _curl_pool[thread_id]
    _curl.setopt(_curl.URL, url)
    _curl.setopt(pycurl.NOSIGNAL, 1)
    _curl.setopt(pycurl.HTTPHEADER, ['Authorization: Bearer {}'.format(token)])
    _, ext = os.path.splitext(urlparse(url).path)
    success = False
    for i in xrange(MAX_RETRIES):
        with NamedTemporaryFile(suffix="."+ext) as temp: # TODO: apply correct file extension
            _curl.setopt(_curl.WRITEDATA, temp.file)
            _curl.perform()
            code = _curl.getinfo(pycurl.HTTP_CODE)
            try:
                if(code != 200):
                    raise TypeError("Request for {} returned unexpected error code: {}".format(url, code))
                temp.file.flush()
                with rasterio.open(temp.name) as dataset:
                    arr = dataset.read()
                success = True
                return arr
            except (TypeError, rasterio.RasterioIOError) as e:
                print(e)
                temp.seek(0)
                print(temp.read())
                _curl.close()
                del _curl_pool[thread_id]
    if success is False:
        arr = np.zeros(shape, dtype=np.float32)
    return arr

class ContentHashedDict(dict):
    @property
    def _id(self):
        _id = str(uuid.uuid5(NAMESPACE_UUID, self.__hash__()))
        return _id

    def __hash__(self):
        dup = OrderedDict({k:v for k,v in self.items() if k is not "id"})
        return sha256(str(dup).encode('utf-8')).hexdigest()

    def populate_id(self):
        self.update({"id": self._id})

class Op(DaskMeta):
    def __init__(self, name, interface=None):
        self._operator = name
        self._edges = []
        self._nodes = []

        self._ipe_id = None
        self._ipe_graph = None
        self._ipe_meta = None

        self._interface = interface

    @property
    def _id(self):
        return self._nodes[0]._id

    def __call__(self, *args, **kwargs):
        if len(args) > 0 and all([isinstance(arg, gbdx.images.ipe_image.IpeImage) for arg in args]):
            return self._ipe_image_call(*args, **kwargs)
        self._nodes = [ContentHashedDict({
            "operator": self._operator,
            "_ancestors": [arg._id for arg in args],
            "parameters": OrderedDict({
                k:json.dumps(v, sort_keys=True) if not isinstance(v, basestring) else v
                for k,v in sorted(kwargs.items(), key=lambda x: x[0])})
        })]
        for arg in args:
            self._nodes.extend(arg._nodes)

        self._edges = [ContentHashedDict({"index": idx + 1, "source": arg._nodes[0]._id,
                                          "destination": self._nodes[0]._id})
                       for idx, arg in enumerate(args)]
        for arg in args:
            self._edges.extend(arg._edges)

        for e in chain(self._nodes, self._edges):
            e.populate_id()
        return self

    def _ipe_image_call(self, *args, **kwargs):
        out = self(*[arg.ipe for arg in args], **kwargs)
        ipe_img = gbdx.images.ipe_image.IpeImage(out)
        return ipe_img

    def graph(self, conn=None):
        if(self._ipe_id is not None and
           self._ipe_graph is not None):
            return self._ipe_graph

        _nodes = [{k:v for k,v in node.items() if not k.startswith('_')} for node in self._nodes]
        graph = {
            "edges": self._edges,
            "nodes": _nodes
        }

        if self._interface is not None and conn is None:
            conn = self._interface.gbdx_futures_session

        if conn is not None:
            self._ipe_id = register_ipe_graph(conn, graph)
            self._ipe_graph = graph
            self._ipe_meta = get_ipe_metadata(conn, self._ipe_id, self._id)
            return self._ipe_graph

        return graph

    @property
    def metadata(self):
        assert self.graph() is not None
        if self._ipe_meta is not None:
            return self._ipe_meta
        if self._interface is not None:
            self._ipe_meta = get_ipe_metadata(self._interface.gbdx_futures_session, self._ipe_id, self._id)
        return self._ipe_meta

    @property
    def dask(self):
        token = self._interface.gbdx_connection.access_token
        _chunks = self.chunks
        _name = self.name
        img_md = self.metadata["image"]
        return {(_name, 0, y - img_md['minTileY'], x - img_md['minTileX']): (load_url, url, token, _chunks)
                for (y, x), url in self._collect_urls().items()}

    @property
    def name(self):
        return "image-{}".format(self._id)

    @property
    def chunks(self):
        img_md = self.metadata["image"]
        return (img_md["numBands"], img_md["tileYSize"], img_md["tileXSize"])

    @property
    def dtype(self):
        try:
            data_type = self.metadata["image"]["dataType"]
            return IPE_TO_DTYPE[data_type]
        except KeyError:
            raise TypeError("Metadata indicates an unrecognized data type: {}".format(data_type))

    @property
    def shape(self):
        img_md = self.metadata["image"]
        return (img_md["numBands"],
                (img_md["maxTileY"] - img_md["minTileY"] + 1)*img_md["tileYSize"],
                (img_md["maxTileX"] - img_md["minTileX"] + 1)*img_md["tileXSize"])

    def _ipe_tile(self, x, y, ipe_id, _id):
        return "{}/tile/{}/{}/{}/{}/{}.tif".format(VIRTUAL_IPE_URL, "idaho-virtual", ipe_id, _id, x, y)

    def _collect_urls(self):
        img_md = self.metadata["image"]
        ipe_id = self._ipe_id
        _id = self._id
        return {(y, x): self._ipe_tile(x, y, ipe_id, _id)
                for y in xrange(img_md['minTileY'], img_md["maxTileY"]+1)
                for x in xrange(img_md['minTileX'], img_md["maxTileX"]+1)}

class Ipe(object):
    def __getattr__(self, name):
        return Op(name=name, interface=Auth())
<commit_msg>fixed curl pool issue<commit_after>import os
import uuid
import json
from hashlib import sha256
from itertools import chain
from collections import OrderedDict, defaultdict
import threading
from tempfile import NamedTemporaryFile
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

import rasterio
import pycurl
import numpy as np

import gbdxtools as gbdx
from gbdxtools.ipe.util import IPE_TO_DTYPE
from gbdxtools.ipe.graph import VIRTUAL_IPE_URL, register_ipe_graph, get_ipe_metadata
from gbdxtools.images.meta import DaskMeta
from gbdxtools.auth import Auth

import warnings
warnings.filterwarnings('ignore')

_curl_pool = defaultdict(pycurl.Curl)
MAX_RETRIES = 5
try:
    basestring
except NameError:
    basestring = str

try:
    xrange
except NameError:
    xrange = range

NAMESPACE_UUID = uuid.NAMESPACE_DNS

def load_url(url, token, shape=(8, 256, 256)):
    """ Loads a geotiff url inside a thread and returns as an ndarray """
    _, ext = os.path.splitext(urlparse(url).path)
    success = False
    for i in xrange(MAX_RETRIES):
        thread_id = threading.current_thread().ident
        _curl = _curl_pool[thread_id]
        _curl.setopt(_curl.URL, url)
        _curl.setopt(pycurl.NOSIGNAL, 1)
        _curl.setopt(pycurl.HTTPHEADER, ['Authorization: Bearer {}'.format(token)])
        with NamedTemporaryFile(suffix="."+ext) as temp: # TODO: apply correct file extension
            _curl.setopt(_curl.WRITEDATA, temp.file)
            _curl.perform()
            code = _curl.getinfo(pycurl.HTTP_CODE)
            try:
                if(code != 200):
                    raise TypeError("Request for {} returned unexpected error code: {}".format(url, code))
                temp.file.flush()
                with rasterio.open(temp.name) as dataset:
                    arr = dataset.read()
                success = True
                return arr
            except (TypeError, rasterio.RasterioIOError) as e:
                print(e)
                temp.seek(0)
                print(temp.read())
                _curl.close()
                del _curl_pool[thread_id]
    if success is False:
        arr = np.zeros(shape, dtype=np.float32)
    return arr

class ContentHashedDict(dict):
    @property
    def _id(self):
        _id = str(uuid.uuid5(NAMESPACE_UUID, self.__hash__()))
        return _id

    def __hash__(self):
        dup = OrderedDict({k:v for k,v in self.items() if k is not "id"})
        return sha256(str(dup).encode('utf-8')).hexdigest()

    def populate_id(self):
        self.update({"id": self._id})

class Op(DaskMeta):
    def __init__(self, name, interface=None):
        self._operator = name
        self._edges = []
        self._nodes = []

        self._ipe_id = None
        self._ipe_graph = None
        self._ipe_meta = None

        self._interface = interface

    @property
    def _id(self):
        return self._nodes[0]._id

    def __call__(self, *args, **kwargs):
        if len(args) > 0 and all([isinstance(arg, gbdx.images.ipe_image.IpeImage) for arg in args]):
            return self._ipe_image_call(*args, **kwargs)
        self._nodes = [ContentHashedDict({
            "operator": self._operator,
            "_ancestors": [arg._id for arg in args],
            "parameters": OrderedDict({
                k:json.dumps(v, sort_keys=True) if not isinstance(v, basestring) else v
                for k,v in sorted(kwargs.items(), key=lambda x: x[0])})
        })]
        for arg in args:
            self._nodes.extend(arg._nodes)

        self._edges = [ContentHashedDict({"index": idx + 1, "source": arg._nodes[0]._id,
                                          "destination": self._nodes[0]._id})
                       for idx, arg in enumerate(args)]
        for arg in args:
            self._edges.extend(arg._edges)

        for e in chain(self._nodes, self._edges):
            e.populate_id()
        return self

    def _ipe_image_call(self, *args, **kwargs):
        out = self(*[arg.ipe for arg in args], **kwargs)
        ipe_img = gbdx.images.ipe_image.IpeImage(out)
        return ipe_img

    def graph(self, conn=None):
        if(self._ipe_id is not None and
           self._ipe_graph is not None):
            return self._ipe_graph

        _nodes = [{k:v for k,v in node.items() if not k.startswith('_')} for node in self._nodes]
        graph = {
            "edges": self._edges,
            "nodes": _nodes
        }

        if self._interface is not None and conn is None:
            conn = self._interface.gbdx_futures_session

        if conn is not None:
            self._ipe_id = register_ipe_graph(conn, graph)
            self._ipe_graph = graph
            self._ipe_meta = get_ipe_metadata(conn, self._ipe_id, self._id)
            return self._ipe_graph

        return graph

    @property
    def metadata(self):
        assert self.graph() is not None
        if self._ipe_meta is not None:
            return self._ipe_meta
        if self._interface is not None:
            self._ipe_meta = get_ipe_metadata(self._interface.gbdx_futures_session, self._ipe_id, self._id)
        return self._ipe_meta

    @property
    def dask(self):
        token = self._interface.gbdx_connection.access_token
        _chunks = self.chunks
        _name = self.name
        img_md = self.metadata["image"]
        return {(_name, 0, y - img_md['minTileY'], x - img_md['minTileX']): (load_url, url, token, _chunks)
                for (y, x), url in self._collect_urls().items()}

    @property
    def name(self):
        return "image-{}".format(self._id)

    @property
    def chunks(self):
        img_md = self.metadata["image"]
        return (img_md["numBands"], img_md["tileYSize"], img_md["tileXSize"])

    @property
    def dtype(self):
        try:
            data_type = self.metadata["image"]["dataType"]
            return IPE_TO_DTYPE[data_type]
        except KeyError:
            raise TypeError("Metadata indicates an unrecognized data type: {}".format(data_type))

    @property
    def shape(self):
        img_md = self.metadata["image"]
        return (img_md["numBands"],
                (img_md["maxTileY"] - img_md["minTileY"] + 1)*img_md["tileYSize"],
                (img_md["maxTileX"] - img_md["minTileX"] + 1)*img_md["tileXSize"])

    def _ipe_tile(self, x, y, ipe_id, _id):
        return "{}/tile/{}/{}/{}/{}/{}.tif".format(VIRTUAL_IPE_URL, "idaho-virtual", ipe_id, _id, x, y)

    def _collect_urls(self):
        img_md = self.metadata["image"]
        ipe_id = self._ipe_id
        _id = self._id
        return {(y, x): self._ipe_tile(x, y, ipe_id, _id)
                for y in xrange(img_md['minTileY'], img_md["maxTileY"]+1)
                for x in xrange(img_md['minTileX'], img_md["maxTileX"]+1)}

class Ipe(object):
    def __getattr__(self, name):
        return Op(name=name, interface=Auth())
<|endoftext|>


<commit_before>import configparser
import os

import sys

from agents.base_agent import BaseAgent, BOT_CONFIG_LOADOUT_HEADER, BOT_CONFIG_LOADOUT_ORANGE_HEADER, \
    BOT_CONFIG_MODULE_HEADER
from utils.agent_creator import import_agent
from utils.custom_config import ConfigObject

PARTICPANT_CONFIGURATION_HEADER = 'Participant Configuration'
PARTICPANT_BOT_KEY = 'participant_is_bot'
PARTICPANT_RLBOT_KEY = 'participant_is_rlbot_controlled'
PARTICPANT_CONFIG_KEY = 'participant_config'
PARTICPANT_BOT_SKILL_KEY = 'participant_bot_skill'
PARTICPANT_TEAM = 'participant_team'
RLBOT_CONFIG_FILE = 'rlbot.cfg'
RLBOT_CONFIGURATION_HEADER = 'RLBot Configuration'

# Cut off at 31 characters and handle duplicates
def get_sanitized_bot_name(dict, name):
    if name not in dict:
        new_name = name[:31]  # Make sure name does not exceed 31 characters
        dict[name] = 1
    else:
        count = dict[name]
        new_name = name[:27] + "(" + str(count + 1) + ")"  # Truncate at 27 because we can have up to '(10)' appended
        dict[name] = count + 1

    return new_name

def get_bot_config_file_list(botCount, config, bot_configs):
    """
    Adds all the config files or config objects.
    :param botCount:
    :param config:
    :param bot_configs: These are configs that have been loaded from the gui, they get assigned a bot index.
    :return:
    """
    config_file_list = []
    for i in range(botCount):
        if i in bot_configs:
            config_file_list.append(bot_configs[i])
        else:
            bot_config = config.get(PARTICPANT_CONFIGURATION_HEADER, PARTICPANT_CONFIG_KEY, i)
            bot_config_path = bot_config
            sys.path.append(os.path.dirname(bot_config_path))
            raw_bot_config = configparser.RawConfigParser()
            raw_bot_config.read(bot_config_path)
            config_file_list.append(raw_bot_config)

    return config_file_list

def create_bot_config_layout():
    config_object = ConfigObject()
    rlbot_header = config_object.add_header_name(RLBOT_CONFIGURATION_HEADER)
    rlbot_header.add_value('num_participants', int, default=2, description='The number of players on the field')

    participant_header = config_object.add_header_name(PARTICPANT_CONFIGURATION_HEADER, is_indexed=True)
    participant_header.add_value(PARTICPANT_TEAM, int, default=0,
                                 description="Which team the player should be on:" +
                                             "\nteam 0 (blue) shoots on positive goal, " +
                                             "team 1 (orange) shoots on negative goal")

    participant_header.add_value(PARTICPANT_CONFIG_KEY, str, default='./agents/atba/atba.cfg',
                                 description="The location of the configuration file for a specific agent")

    participant_header.add_value(PARTICPANT_BOT_KEY, bool, default='yes',
                                 description='Accepted values are "1", "yes", "true", and "on", for True,' +
                                             ' and "0", "no", "false", and "off", for False\n' +
                                             'You can have up to 4 local players and they must ' +
                                             'be activated in game or it will crash.\n' +
                                             'If no player is specified you will be spawned in as spectator!')

    participant_header.add_value(PARTICPANT_RLBOT_KEY, bool, default='yes',
                                 description='Accepted values are "1", "yes", "true", and "on", for True,' +
                                             ' and "0", "no", "false", and "off", for False\n' +
                                             'By specifying \'no\' here you can use default bots ' +
                                             'like the rookie, all-star, etc.')

    participant_header.add_value(PARTICPANT_BOT_SKILL_KEY, float, default=1.0,
                                 description='If participant is a bot and not RLBot controlled,' +
                                             ' this value will be used to set bot skill.\n' +
                                             '0.0 is Rookie, 0.5 is pro, 1.0 is all-star. ' +
                                             ' You can set values in-between as well.')
    return config_object

def parse_configurations(gameInputPacket, config_parser, bot_configs):
    bot_names = []
    bot_teams = []
    bot_modules = []

    # Determine number of participants
    num_participants = config_parser.getint(RLBOT_CONFIGURATION_HEADER, 'num_participants')

    # Retrieve bot config files
    participant_configs = get_bot_config_file_list(num_participants, config_parser, bot_configs)

    # Create empty lists

    bot_parameter_list = []
    name_dict = dict()

    gameInputPacket.iNumPlayers = num_participants

    bot_config_object = BaseAgent.create_agent_configurations()

    # Set configuration values for bots and store name and team
    for i in range(num_participants):
        raw_bot_config = participant_configs[i]
        bot_config_object.reset()
        bot_config_object.parse_file(participant_configs[i])

        team_num = config_parser.getint(PARTICPANT_CONFIGURATION_HEADER,
                                        PARTICPANT_TEAM, i)

        loadout_header = BOT_CONFIG_LOADOUT_HEADER
        if (team_num == 1 and bot_config_object.has_section(BOT_CONFIG_LOADOUT_ORANGE_HEADER)):
            loadout_header = BOT_CONFIG_LOADOUT_ORANGE_HEADER

        gameInputPacket.sPlayerConfiguration[i].bBot = config_parser.getboolean(PARTICPANT_CONFIGURATION_HEADER,
                                                                                PARTICPANT_BOT_KEY, i)
        gameInputPacket.sPlayerConfiguration[i].bRLBotControlled = config_parser.getboolean(
            PARTICPANT_CONFIGURATION_HEADER,
            PARTICPANT_RLBOT_KEY, i)
        gameInputPacket.sPlayerConfiguration[i].fBotSkill = config_parser.getfloat(PARTICPANT_CONFIGURATION_HEADER,
                                                                                PARTICPANT_BOT_SKILL_KEY, i)
        gameInputPacket.sPlayerConfiguration[i].iPlayerIndex = i

        gameInputPacket.sPlayerConfiguration[i].wName = get_sanitized_bot_name(name_dict,
                                                                               bot_config_object.get(loadout_header, 'name'))
        gameInputPacket.sPlayerConfiguration[i].ucTeam = team_num

        BaseAgent.parse_bot_loadout(gameInputPacket.sPlayerConfiguration[i], bot_config_object, loadout_header)

        bot_names.append(bot_config_object.get(loadout_header, 'name'))
        bot_teams.append(config_parser.getint(PARTICPANT_CONFIGURATION_HEADER, PARTICPANT_TEAM, i))

        if gameInputPacket.sPlayerConfiguration[i].bRLBotControlled:
            agent_module = bot_config_object.get(BOT_CONFIG_MODULE_HEADER, 'agent_module')
            bot_modules.append(agent_module)
            agent = import_agent(agent_module)
            agent_configuration = agent.create_agent_configurations()
            agent_configuration.parse_file(raw_bot_config)
            bot_parameter_list.append(agent_configuration)
        else:
            bot_modules.append('NO_MODULE_FOR_PARTICIPANT')
            bot_parameter_list.append(None)

    return num_participants, bot_names, bot_teams, bot_modules, bot_parameter_list
<commit_msg>moved parsing a bot to its own file<commit_after>import configparser
import os

import sys

from agents.base_agent import BaseAgent, BOT_CONFIG_LOADOUT_HEADER, BOT_CONFIG_LOADOUT_ORANGE_HEADER, \
    BOT_CONFIG_MODULE_HEADER
from utils.agent_creator import import_agent
from utils.custom_config import ConfigObject

PARTICPANT_CONFIGURATION_HEADER = 'Participant Configuration'
PARTICPANT_BOT_KEY = 'participant_is_bot'
RLBOT_KEY = 'participant_is_rlbot_controlled'
PARTICPANT_CONFIG_KEY = 'participant_config'
BOT_SKILL_KEY = 'participant_bot_skill'
PARTICPANT_TEAM = 'participant_team'
RLBOT_CONFIG_FILE = 'rlbot.cfg'
RLBOT_CONFIGURATION_HEADER = 'RLBot Configuration'

# Cut off at 31 characters and handle duplicates
def get_sanitized_bot_name(dict, name):
    if name not in dict:
        new_name = name[:31]  # Make sure name does not exceed 31 characters
        dict[name] = 1
    else:
        count = dict[name]
        new_name = name[:27] + "(" + str(count + 1) + ")"  # Truncate at 27 because we can have up to '(10)' appended
        dict[name] = count + 1

    return new_name

def get_bot_config_file_list(botCount, config, bot_configs):
    """
    Adds all the config files or config objects.
    :param botCount:
    :param config:
    :param bot_configs: These are configs that have been loaded from the gui, they get assigned a bot index.
    :return:
    """
    config_file_list = []
    for i in range(botCount):
        if i in bot_configs:
            config_file_list.append(bot_configs[i])
        else:
            bot_config = config.get(PARTICPANT_CONFIGURATION_HEADER, PARTICPANT_CONFIG_KEY, i)
            bot_config_path = bot_config
            sys.path.append(os.path.dirname(bot_config_path))
            raw_bot_config = configparser.RawConfigParser()
            raw_bot_config.read(bot_config_path)
            config_file_list.append(raw_bot_config)

    return config_file_list

def create_bot_config_layout():
    config_object = ConfigObject()
    rlbot_header = config_object.add_header_name(RLBOT_CONFIGURATION_HEADER)
    rlbot_header.add_value('num_participants', int, default=2, description='The number of players on the field')

    participant_header = config_object.add_header_name(PARTICPANT_CONFIGURATION_HEADER, is_indexed=True)
    participant_header.add_value(PARTICPANT_TEAM, int, default=0,
                                 description="Which team the player should be on:" +
                                             "\nteam 0 (blue) shoots on positive goal, " +
                                             "team 1 (orange) shoots on negative goal")

    participant_header.add_value(PARTICPANT_CONFIG_KEY, str, default='./agents/atba/atba.cfg',
                                 description="The location of the configuration file for a specific agent")

    participant_header.add_value(PARTICPANT_BOT_KEY, bool, default='yes',
                                 description='Accepted values are "1", "yes", "true", and "on", for True,' +
                                             ' and "0", "no", "false", and "off", for False\n' +
                                             'You can have up to 4 local players and they must ' +
                                             'be activated in game or it will crash.\n' +
                                             'If no player is specified you will be spawned in as spectator!')

    participant_header.add_value(RLBOT_KEY, bool, default='yes',
                                 description='Accepted values are "1", "yes", "true", and "on", for True,' +
                                             ' and "0", "no", "false", and "off", for False\n' +
                                             'By specifying \'no\' here you can use default bots ' +
                                             'like the rookie, all-star, etc.')

    participant_header.add_value(BOT_SKILL_KEY, float, default=1.0,
                                 description='If participant is a bot and not RLBot controlled,' +
                                             ' this value will be used to set bot skill.\n' +
                                             '0.0 is Rookie, 0.5 is pro, 1.0 is all-star. ' +
                                             ' You can set values in-between as well.')
    return config_object

def parse_configurations(gameInputPacket, config_parser, bot_configs):
    bot_names = []
    bot_teams = []
    bot_modules = []

    # Determine number of participants
    num_participants = config_parser.getint(RLBOT_CONFIGURATION_HEADER, 'num_participants')

    # Retrieve bot config files
    participant_configs = get_bot_config_file_list(num_participants, config_parser, bot_configs)

    # Create empty lists

    bot_parameter_list = []
    name_dict = dict()

    gameInputPacket.iNumPlayers = num_participants

    bot_config_object = BaseAgent.create_agent_configurations()

    # Set configuration values for bots and store name and team
    for i in range(num_participants):
        bot_config_object.reset()
        bot_config_object.parse_file(participant_configs[i])

        bot_name, team_number, bot_module, bot_parameters = load_bot_config(i, gameInputPacket.sPlayerConfiguration[i],
                                                                            bot_config_object, config_parser, name_dict)

        bot_names.append(bot_name)
        bot_teams.append(team_number)
        bot_modules.append(bot_module)
        bot_parameter_list.append(bot_parameters)

    return num_participants, bot_names, bot_teams, bot_modules, bot_parameter_list

def load_bot_config(index, bot_configuration, bot_config_object, overall_config, name_dict):
    """
    Loads the config data of a single bot
    :param index: This is the bot index (where it appears in game_cars)
    :param bot_configuration: This is the game_tick_packet configuration that is sent back to the game
    :param bot_config_object: A config object for a single bot
    :param overall_config: This is the config for the entire session not one particular bot
    :param name_dict: A mapping of used names so we can make sure to not reuse bot names.
    :return:
    """
    team_num = overall_config.getint(PARTICPANT_CONFIGURATION_HEADER,
                                    PARTICPANT_TEAM, index)

    bot_configuration.ucTeam = team_num

    # Setting up data about what type of bot it is
    bot_configuration.bBot = overall_config.getboolean(PARTICPANT_CONFIGURATION_HEADER, PARTICPANT_BOT_KEY, index)
    bot_configuration.bRLBotControlled = overall_config.getboolean(PARTICPANT_CONFIGURATION_HEADER, RLBOT_KEY, index)
    bot_configuration.fBotSkill = overall_config.getfloat(PARTICPANT_CONFIGURATION_HEADER, BOT_SKILL_KEY, index)
    bot_configuration.iPlayerIndex = index

    loadout_header = BOT_CONFIG_LOADOUT_HEADER
    if team_num == 1 and bot_config_object.has_section(BOT_CONFIG_LOADOUT_ORANGE_HEADER):
        loadout_header = BOT_CONFIG_LOADOUT_ORANGE_HEADER

    # Setting up the bots name
    bot_name = bot_config_object.get(loadout_header, 'name')
    bot_configuration.wName = get_sanitized_bot_name(name_dict, bot_name)

    BaseAgent.parse_bot_loadout(bot_configuration, bot_config_object, loadout_header)

    bot_module = 'NO_MODULE_FOR_PARTICIPANT'
    bot_parameters = None

    if bot_configuration.bRLBotControlled:
        bot_module = bot_config_object.get(BOT_CONFIG_MODULE_HEADER, 'agent_module')
        agent = import_agent(bot_module)
        bot_parameters = agent.create_agent_configurations()
        bot_parameters.parse_file(bot_config_object.get_raw_file())

    return bot_name, team_num, bot_module, bot_parameters
<|endoftext|>