<commit_before># Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""A subclass of the ipaddress library that includes comments for ipaddress."""

import collections
import ipaddress
import itertools
from typing import Union

import capirca.utils.iputils as iputils

def IP(ip, comment='', token='', strict=True):
  """Take an ip string and return an object of the correct type.

  Args:
    ip: the ip address.
    comment: option comment field
    token: option token name where this address was extracted from
    strict: If strict should be used in ipaddress object.

  Returns:
    ipaddress.IPv4 or ipaddress.IPv6 object or raises ValueError.

  Raises:
    ValueError: if the string passed isn't either a v4 or a v6 address.
  """
  if isinstance(ip, ipaddress._BaseNetwork):  # pylint disable=protected-access
    imprecise_ip = ip
  else:
    imprecise_ip = ipaddress.ip_network(ip, strict=strict)
  if imprecise_ip.version == 4:
    return IPv4(ip, comment, token, strict=strict)
  elif imprecise_ip.version == 6:
    return IPv6(ip, comment, token, strict=strict)
  raise ValueError('Provided IP string "%s" is not a valid v4 or v6 address'
                   % ip)

# TODO(robankeny) remove once at 3.7
@staticmethod
def _is_subnet_of(a, b):  # pylint: disable=invalid-name
  try:
    # Always false if one is v4 and the other is v6.
    if a.version != b.version:
      raise TypeError('%s and %s are not of the same version' % (a, b))
    return (b.network_address <= a.network_address and
            b.broadcast_address >= a.broadcast_address)
  except AttributeError:
    raise TypeError(
        'Unable to test subnet containment between %s and %s' % (a, b))

class IPv4(ipaddress.IPv4Network):
  """This subclass allows us to keep text comments related to each object."""

  def __init__(self, ip_string, comment='', token='', strict=True):
    self.text = comment
    self.token = token
    self.parent_token = token

    # Using a tuple of IP integer/prefixlength is significantly faster than
    # using the BaseNetwork object for recreating the IP network
    if isinstance(ip_string, ipaddress._BaseNetwork):  # pylint disable=protected-access
      ip = (ip_string.network_address._ip, ip_string.prefixlen)  # pylint disable=protected-access # pytype: disable=attribute-error
    else:
      ip = ip_string
    super().__init__(ip, strict)

  def subnet_of(self, other):
    """Return True if this network is a subnet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(self, other)

  def supernet_of(self, other):
    """Return True if this network is a supernet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(other, self)

  def __deepcopy__(self, memo):
    result = self.__class__(self)
    result.text = self.text
    result.token = self.token
    result.parent_token = self.parent_token
    return result

  def AddComment(self, comment=''):
    """Append comment to self.text, comma separated.

    Don't add the comment if it's the same as self.text.

    Args:
      comment: comment to be added.
    """
    if self.text:
      if comment and comment not in self.text:
        self.text += ', ' + comment
    else:
      self.text = comment

  def supernet(self, prefixlen_diff=1):
    """Override ipaddress.IPv4 supernet so we can maintain comments.

    See ipaddress.IPv4.Supernet for complete documentation.

    Args:
      prefixlen_diff: Prefix length difference.

    Returns:
      An IPv4 object

    Raises:
      PrefixlenDiffInvalidError: Raised when prefixlen - prefixlen_diff results
        in a negative number.
    """
    if self.prefixlen == 0:
      return self
    if self.prefixlen - prefixlen_diff < 0:
      raise PrefixlenDiffInvalidError(
          'current prefixlen is %d, cannot have a prefixlen_diff of %d' % (
              self.prefixlen, prefixlen_diff))
    ret_addr = IPv4(ipaddress.IPv4Network.supernet(self, prefixlen_diff),
                    comment=self.text, token=self.token)
    return ret_addr

  # Backwards compatibility name from v1.
  Supernet = supernet
  _is_subnet_of = _is_subnet_of

class IPv6(ipaddress.IPv6Network):
  """This subclass allows us to keep text comments related to each object."""

  def __init__(self, ip_string, comment='', token='', strict=True):
    self.text = comment
    self.token = token
    self.parent_token = token

    # Using a tuple of IP integer/prefixlength is significantly faster than
    # using the BaseNetwork object for recreating the IP network
    if isinstance(ip_string, ipaddress._BaseNetwork):  # pylint disable=protected-access
      ip = (ip_string.network_address._ip, ip_string.prefixlen)  # pylint disable=protected-access # pytype: disable=attribute-error
    else:
      ip = ip_string
    super().__init__(ip, strict)

  def subnet_of(self, other):
    """Return True if this network is a subnet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(self, other)

  def supernet_of(self, other):
    """Return True if this network is a supernet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(other, self)

  def __deepcopy__(self, memo):
    result = self.__class__(self)
    result.text = self.text
    result.token = self.token
    result.parent_token = self.parent_token
    return result

  def supernet(self, prefixlen_diff=1):
    """Override ipaddress.IPv6Network supernet so we can maintain comments.

    See ipaddress.IPv6Network.Supernet for complete documentation.
    Args:
      prefixlen_diff: Prefix length difference.

    Returns:
      An IPv4 object

    Raises:
      PrefixlenDiffInvalidError: Raised when prefixlen - prefixlen_diff results
        in a negative number.
    """
    if self.prefixlen == 0:
      return self
    if self.prefixlen - prefixlen_diff < 0:
      raise PrefixlenDiffInvalidError(
          'current prefixlen is %d, cannot have a prefixlen_diff of %d' % (
              self.prefixlen, prefixlen_diff))
    ret_addr = IPv6(ipaddress.IPv6Network.supernet(self, prefixlen_diff),
                    comment=self.text, token=self.token)
    return ret_addr

  # Backwards compatibility name from v1.
  Supernet = supernet
  _is_subnet_of = _is_subnet_of

  def AddComment(self, comment=''):
    """Append comment to self.text, comma separated.

    Don't add the comment if it's the same as self.text.

    Args:
      comment: comment to be added.
    """
    if self.text:
      if comment and comment not in self.text:
        self.text += ', ' + comment
    else:
      self.text = comment

IPType = Union[IPv4, IPv6]

def _InNetList(adders, ip):
  """Returns True if ip is contained in adders."""
  for addr in adders:
    if ip.subnet_of(addr):
      return True
  return False

def IsSuperNet(supernets, subnets):
  """Returns True if subnets are fully consumed by supernets."""
  for net in subnets:
    if not _InNetList(supernets, net):
      return False
  return True

def CollapseAddrListPreserveTokens(addresses):
  """Collapse an array of IPs only when their tokens are the same.

  Args:
     addresses: list of ipaddress.IPNetwork objects.

  Returns:
    list of ipaddress.IPNetwork objects.
  """
  ret_array = []
  for grp in itertools.groupby(sorted(addresses, key=lambda x: x.parent_token),
                               lambda x: x.parent_token):
    ret_array.append(CollapseAddrList(list(grp[1])))
  dedup_array = []
  i = 0
  while len(ret_array) > i:
    ip = ret_array.pop(0)
    k = 0
    to_add = True
    while k < len(dedup_array):
      if IsSuperNet(dedup_array[k], ip):
        to_add = False
        break
      elif IsSuperNet(ip, dedup_array[k]):
        del dedup_array[k]
      k += 1
    if to_add:
      dedup_array.append(ip)
  return [i for sublist in dedup_array for i in sublist]

def _SafeToMerge(address, merge_target, check_addresses):
  """Determine if it's safe to merge address into merge target.

  Checks given address against merge target and a list of check_addresses
  if it's OK to roll address into merge target such that it not less specific
  than any of the check_addresses. See description of why ir is important
  within public function CollapseAddrList.

  Args:
    address: Address that is being merged.
    merge_target: Merge candidate address.
    check_addresses: A dict networks_address->addrs to compare specificity with.

  Returns:
    True if safe to merge, False otherwise.
  """
  for check_address in check_addresses.get(address.network_address, []):
    if merge_target.netmask <= check_address.netmask < address.netmask:
      return False
  return True

def _CollapseAddrListInternal(addresses, complements_by_network):
  """Collapses consecutive netblocks until reaching a fixed point.

   Example:

   ip1 = ipaddress.IPv4Network('1.1.0.0/24')
   ip2 = ipaddress.IPv4Network('1.1.1.0/24')
   ip3 = ipaddress.IPv4Network('1.1.2.0/24')
   ip4 = ipaddress.IPv4Network('1.1.3.0/24')
   ip5 = ipaddress.IPv4Network('1.1.4.0/24')
   ip6 = ipaddress.IPv4Network('1.1.0.1/22')

   _CollapseAddrListInternal([ip1, ip2, ip3, ip4, ip5, ip6]) ->
   [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')]

   Note, this shouldn't be called directly, but is called via
   CollapseAddrList([])

  Args:
    addresses: List of IPv4 or IPv6 objects
    complements_by_network: Dict of IPv4 or IPv6 objects indexed by
      network_address, that if present will be considered to avoid harmful
      optimizations.

  Returns:
    List of IPv4 or IPv6 objects (depending on what we were passed)
  """
  ret_array = []
  for addr in addresses:
    addr_is_fresh = True
    while addr_is_fresh:
      addr_is_fresh = False
      if not ret_array:
        ret_array.append(addr)
        continue

      prev_addr = ret_array[-1]
      if not _SafeToMerge(addr, prev_addr, complements_by_network):
        ret_array.append(addr)
      elif prev_addr.supernet_of(addr):
        # Preserve addr's comment, then subsume it.
        prev_addr.AddComment(addr.text)
      elif (prev_addr.version == addr.version and
            prev_addr.prefixlen == addr.prefixlen and
            # It's faster to compare integers than IP objects
            prev_addr.broadcast_address._ip + 1 == addr.network_address._ip and  # pylint disable=protected-access
            # Generating Supernet is relatively intensive compared to doing bit
            # operations
            (prev_addr.netmask._ip << 1) & prev_addr.network_address._ip ==      # pylint disable=protected-access
            prev_addr.network_address._ip):                                      # pylint disable=protected-access
        # Preserve addr's comment, then merge with it.
        prev_addr.AddComment(addr.text)
        addr = ret_array.pop().Supernet()
        addr_is_fresh = True
      else:
        ret_array.append(addr)

  return ret_array

def CollapseAddrList(addresses, complement_addresses=None):
  """Collapse an array of IP objects.

  Example:  CollapseAddrList(
    [IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) -> [IPv4('1.1.0.0/23')]
    Note: this works just as well with IPv6 addresses too.

  On platforms that support exclude semantics with most specific match,
  this method should _always_ be called with complement addresses supplied.
  Not doing so can lead to *reversal* of intent. Consider this case:

    destination-address:: 10.0.0.0/8, 10.0.0.0/10
    destination-exclude:: 10.0.0.0/9

  Without optimization, 10.0.0.1 will _match_. With optimization, most specific
  prefix will _not_ match, reversing the intent. Supplying complement_addresses
  allows this method to consider those implications.

  Args:
     addresses: list of ipaddress.IPNetwork objects
     complement_addresses: list of ipaddress.IPNetwork objects that, if present,
      will be considered to avoid harmful optimizations.

  Returns:
    list of ipaddress.IPNetwork objects
  """
  complements_dict = collections.defaultdict(list)
  address_set = set([a.network_address for a in addresses])
  for ca in complement_addresses or []:
    if ca.network_address in address_set:
      complements_dict[ca.network_address].append(ca)
  return _CollapseAddrListInternal(
      sorted(addresses, key=ipaddress.get_mixed_type_key), complements_dict)

def SortAddrList(addresses):
  """Return a sorted list of nacaddr objects."""
  return sorted(addresses, key=ipaddress.get_mixed_type_key)

def RemoveAddressFromList(superset, exclude):
  """Remove a single address from a list of addresses.

  Args:
    superset: a List of nacaddr IPv4 or IPv6 addresses
    exclude: a single nacaddr IPv4 or IPv6 address

  Returns:
    a List of nacaddr IPv4 or IPv6 addresses
  """
  ret_array = []
  for addr in superset:
    if exclude == addr or addr.subnet_of(exclude):
      pass
    elif exclude.version == addr.version and exclude.subnet_of(addr):
      # this could be optimized except that one group uses this
      # code with ipaddrs (instead of nacaddrs).
      ret_array.extend(IP(x) for x in iputils.exclude_address(addr, exclude))
    else:
      ret_array.append(addr)
  return sorted(ret_array)

def AddressListExclude(superset, excludes, collapse_addrs=True):
  """Remove a list of addresses from another list of addresses.

  Args:
    superset: a List of nacaddr IPv4 or IPv6 addresses
    excludes: a List nacaddr IPv4 or IPv6 addresses
    collapse_addrs: whether or not to collapse contiguous CIDRs togethe

  Returns:
    a List of nacaddr IPv4 or IPv6 addresses
  """
  if collapse_addrs:
    superset = CollapseAddrList(superset)[::-1]
    excludes = CollapseAddrList(excludes)[::-1]
  else:
    superset = sorted(superset, reverse=True)
    excludes = sorted(excludes, reverse=True)

  ret_array = []
  while superset and excludes:
    if superset[-1].overlaps(excludes[-1]):
      ip = superset.pop()
      superset.extend(
          reversed(RemoveAddressFromList([ip], excludes[-1])))
    elif superset[-1]._get_networks_key() < excludes[-1]._get_networks_key():  # pylint: disable=protected-access
      ret_array.append(superset.pop())
    else:
      excludes.pop()
  if collapse_addrs:
    return CollapseAddrList(ret_array + superset)
  else:
    return sorted(set(ret_array + superset))

ExcludeAddrs = AddressListExclude

class PrefixlenDiffInvalidError(ipaddress.NetmaskValueError):
  """Holdover from ipaddr v1."""

if __name__ == '__main__':
  pass
<commit_msg>Fix to use a more appropriate sort.<commit_after># Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""A subclass of the ipaddress library that includes comments for ipaddress."""

import collections
import ipaddress
import itertools
from typing import Union

import capirca.utils.iputils as iputils

def IP(ip, comment='', token='', strict=True):
  """Take an ip string and return an object of the correct type.

  Args:
    ip: the ip address.
    comment: option comment field
    token: option token name where this address was extracted from
    strict: If strict should be used in ipaddress object.

  Returns:
    ipaddress.IPv4 or ipaddress.IPv6 object or raises ValueError.

  Raises:
    ValueError: if the string passed isn't either a v4 or a v6 address.
  """
  if isinstance(ip, ipaddress._BaseNetwork):  # pylint disable=protected-access
    imprecise_ip = ip
  else:
    imprecise_ip = ipaddress.ip_network(ip, strict=strict)
  if imprecise_ip.version == 4:
    return IPv4(ip, comment, token, strict=strict)
  elif imprecise_ip.version == 6:
    return IPv6(ip, comment, token, strict=strict)
  raise ValueError('Provided IP string "%s" is not a valid v4 or v6 address'
                   % ip)

# TODO(robankeny) remove once at 3.7
@staticmethod
def _is_subnet_of(a, b):  # pylint: disable=invalid-name
  try:
    # Always false if one is v4 and the other is v6.
    if a.version != b.version:
      raise TypeError('%s and %s are not of the same version' % (a, b))
    return (b.network_address <= a.network_address and
            b.broadcast_address >= a.broadcast_address)
  except AttributeError:
    raise TypeError(
        'Unable to test subnet containment between %s and %s' % (a, b))

class IPv4(ipaddress.IPv4Network):
  """This subclass allows us to keep text comments related to each object."""

  def __init__(self, ip_string, comment='', token='', strict=True):
    self.text = comment
    self.token = token
    self.parent_token = token

    # Using a tuple of IP integer/prefixlength is significantly faster than
    # using the BaseNetwork object for recreating the IP network
    if isinstance(ip_string, ipaddress._BaseNetwork):  # pylint disable=protected-access
      ip = (ip_string.network_address._ip, ip_string.prefixlen)  # pylint disable=protected-access # pytype: disable=attribute-error
    else:
      ip = ip_string
    super().__init__(ip, strict)

  def subnet_of(self, other):
    """Return True if this network is a subnet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(self, other)

  def supernet_of(self, other):
    """Return True if this network is a supernet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(other, self)

  def __deepcopy__(self, memo):
    result = self.__class__(self)
    result.text = self.text
    result.token = self.token
    result.parent_token = self.parent_token
    return result

  def AddComment(self, comment=''):
    """Append comment to self.text, comma separated.

    Don't add the comment if it's the same as self.text.

    Args:
      comment: comment to be added.
    """
    if self.text:
      if comment and comment not in self.text:
        self.text += ', ' + comment
    else:
      self.text = comment

  def supernet(self, prefixlen_diff=1):
    """Override ipaddress.IPv4 supernet so we can maintain comments.

    See ipaddress.IPv4.Supernet for complete documentation.

    Args:
      prefixlen_diff: Prefix length difference.

    Returns:
      An IPv4 object

    Raises:
      PrefixlenDiffInvalidError: Raised when prefixlen - prefixlen_diff results
        in a negative number.
    """
    if self.prefixlen == 0:
      return self
    if self.prefixlen - prefixlen_diff < 0:
      raise PrefixlenDiffInvalidError(
          'current prefixlen is %d, cannot have a prefixlen_diff of %d' % (
              self.prefixlen, prefixlen_diff))
    ret_addr = IPv4(ipaddress.IPv4Network.supernet(self, prefixlen_diff),
                    comment=self.text, token=self.token)
    return ret_addr

  # Backwards compatibility name from v1.
  Supernet = supernet
  _is_subnet_of = _is_subnet_of

class IPv6(ipaddress.IPv6Network):
  """This subclass allows us to keep text comments related to each object."""

  def __init__(self, ip_string, comment='', token='', strict=True):
    self.text = comment
    self.token = token
    self.parent_token = token

    # Using a tuple of IP integer/prefixlength is significantly faster than
    # using the BaseNetwork object for recreating the IP network
    if isinstance(ip_string, ipaddress._BaseNetwork):  # pylint disable=protected-access
      ip = (ip_string.network_address._ip, ip_string.prefixlen)  # pylint disable=protected-access # pytype: disable=attribute-error
    else:
      ip = ip_string
    super().__init__(ip, strict)

  def subnet_of(self, other):
    """Return True if this network is a subnet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(self, other)

  def supernet_of(self, other):
    """Return True if this network is a supernet of other."""
    if self.version != other.version:
      return False
    return self._is_subnet_of(other, self)

  def __deepcopy__(self, memo):
    result = self.__class__(self)
    result.text = self.text
    result.token = self.token
    result.parent_token = self.parent_token
    return result

  def supernet(self, prefixlen_diff=1):
    """Override ipaddress.IPv6Network supernet so we can maintain comments.

    See ipaddress.IPv6Network.Supernet for complete documentation.
    Args:
      prefixlen_diff: Prefix length difference.

    Returns:
      An IPv4 object

    Raises:
      PrefixlenDiffInvalidError: Raised when prefixlen - prefixlen_diff results
        in a negative number.
    """
    if self.prefixlen == 0:
      return self
    if self.prefixlen - prefixlen_diff < 0:
      raise PrefixlenDiffInvalidError(
          'current prefixlen is %d, cannot have a prefixlen_diff of %d' % (
              self.prefixlen, prefixlen_diff))
    ret_addr = IPv6(ipaddress.IPv6Network.supernet(self, prefixlen_diff),
                    comment=self.text, token=self.token)
    return ret_addr

  # Backwards compatibility name from v1.
  Supernet = supernet
  _is_subnet_of = _is_subnet_of

  def AddComment(self, comment=''):
    """Append comment to self.text, comma separated.

    Don't add the comment if it's the same as self.text.

    Args:
      comment: comment to be added.
    """
    if self.text:
      if comment and comment not in self.text:
        self.text += ', ' + comment
    else:
      self.text = comment

IPType = Union[IPv4, IPv6]

def _InNetList(adders, ip):
  """Returns True if ip is contained in adders."""
  for addr in adders:
    if ip.subnet_of(addr):
      return True
  return False

def IsSuperNet(supernets, subnets):
  """Returns True if subnets are fully consumed by supernets."""
  for net in subnets:
    if not _InNetList(supernets, net):
      return False
  return True

def CollapseAddrListPreserveTokens(addresses):
  """Collapse an array of IPs only when their tokens are the same.

  Args:
     addresses: list of ipaddress.IPNetwork objects.

  Returns:
    list of ipaddress.IPNetwork objects.
  """
  ret_array = []
  for grp in itertools.groupby(sorted(addresses, key=lambda x: x.parent_token),
                               lambda x: x.parent_token):
    ret_array.append(CollapseAddrList(list(grp[1])))
  dedup_array = []
  i = 0
  while len(ret_array) > i:
    ip = ret_array.pop(0)
    k = 0
    to_add = True
    while k < len(dedup_array):
      if IsSuperNet(dedup_array[k], ip):
        to_add = False
        break
      elif IsSuperNet(ip, dedup_array[k]):
        del dedup_array[k]
      k += 1
    if to_add:
      dedup_array.append(ip)
  return [i for sublist in dedup_array for i in sublist]

def _SafeToMerge(address, merge_target, check_addresses):
  """Determine if it's safe to merge address into merge target.

  Checks given address against merge target and a list of check_addresses
  if it's OK to roll address into merge target such that it not less specific
  than any of the check_addresses. See description of why ir is important
  within public function CollapseAddrList.

  Args:
    address: Address that is being merged.
    merge_target: Merge candidate address.
    check_addresses: A dict networks_address->addrs to compare specificity with.

  Returns:
    True if safe to merge, False otherwise.
  """
  for check_address in check_addresses.get(address.network_address, []):
    if merge_target.netmask <= check_address.netmask < address.netmask:
      return False
  return True

def _CollapseAddrListInternal(addresses, complements_by_network):
  """Collapses consecutive netblocks until reaching a fixed point.

   Example:

   ip1 = ipaddress.IPv4Network('1.1.0.0/24')
   ip2 = ipaddress.IPv4Network('1.1.1.0/24')
   ip3 = ipaddress.IPv4Network('1.1.2.0/24')
   ip4 = ipaddress.IPv4Network('1.1.3.0/24')
   ip5 = ipaddress.IPv4Network('1.1.4.0/24')
   ip6 = ipaddress.IPv4Network('1.1.0.1/22')

   _CollapseAddrListInternal([ip1, ip2, ip3, ip4, ip5, ip6]) ->
   [IPv4Network('1.1.0.0/22'), IPv4Network('1.1.4.0/24')]

   Note, this shouldn't be called directly, but is called via
   CollapseAddrList([])

  Args:
    addresses: List of IPv4 or IPv6 objects
    complements_by_network: Dict of IPv4 or IPv6 objects indexed by
      network_address, that if present will be considered to avoid harmful
      optimizations.

  Returns:
    List of IPv4 or IPv6 objects (depending on what we were passed)
  """
  ret_array = []
  for addr in addresses:
    addr_is_fresh = True
    while addr_is_fresh:
      addr_is_fresh = False
      if not ret_array:
        ret_array.append(addr)
        continue

      prev_addr = ret_array[-1]
      if not _SafeToMerge(addr, prev_addr, complements_by_network):
        ret_array.append(addr)
      elif prev_addr.supernet_of(addr):
        # Preserve addr's comment, then subsume it.
        prev_addr.AddComment(addr.text)
      elif (prev_addr.version == addr.version and
            prev_addr.prefixlen == addr.prefixlen and
            # It's faster to compare integers than IP objects
            prev_addr.broadcast_address._ip + 1 == addr.network_address._ip and  # pylint disable=protected-access
            # Generating Supernet is relatively intensive compared to doing bit
            # operations
            (prev_addr.netmask._ip << 1) & prev_addr.network_address._ip ==      # pylint disable=protected-access
            prev_addr.network_address._ip):                                      # pylint disable=protected-access
        # Preserve addr's comment, then merge with it.
        prev_addr.AddComment(addr.text)
        addr = ret_array.pop().Supernet()
        addr_is_fresh = True
      else:
        ret_array.append(addr)

  return ret_array

def CollapseAddrList(addresses, complement_addresses=None):
  """Collapse an array of IP objects.

  Example:  CollapseAddrList(
    [IPv4('1.1.0.0/24'), IPv4('1.1.1.0/24')]) -> [IPv4('1.1.0.0/23')]
    Note: this works just as well with IPv6 addresses too.

  On platforms that support exclude semantics with most specific match,
  this method should _always_ be called with complement addresses supplied.
  Not doing so can lead to *reversal* of intent. Consider this case:

    destination-address:: 10.0.0.0/8, 10.0.0.0/10
    destination-exclude:: 10.0.0.0/9

  Without optimization, 10.0.0.1 will _match_. With optimization, most specific
  prefix will _not_ match, reversing the intent. Supplying complement_addresses
  allows this method to consider those implications.

  Args:
     addresses: list of ipaddress.IPNetwork objects
     complement_addresses: list of ipaddress.IPNetwork objects that, if present,
      will be considered to avoid harmful optimizations.

  Returns:
    list of ipaddress.IPNetwork objects
  """
  complements_dict = collections.defaultdict(list)
  address_set = set([a.network_address for a in addresses])
  for ca in complement_addresses or []:
    if ca.network_address in address_set:
      complements_dict[ca.network_address].append(ca)
  return _CollapseAddrListInternal(
      sorted(addresses, key=ipaddress.get_mixed_type_key), complements_dict)

def SortAddrList(addresses):
  """Return a sorted list of nacaddr objects."""
  return sorted(addresses, key=ipaddress.get_mixed_type_key)

def RemoveAddressFromList(superset, exclude):
  """Remove a single address from a list of addresses.

  Args:
    superset: a List of nacaddr IPv4 or IPv6 addresses
    exclude: a single nacaddr IPv4 or IPv6 address

  Returns:
    a List of nacaddr IPv4 or IPv6 addresses
  """
  ret_array = []
  for addr in superset:
    if exclude == addr or addr.subnet_of(exclude):
      pass
    elif exclude.version == addr.version and exclude.subnet_of(addr):
      # this could be optimized except that one group uses this
      # code with ipaddrs (instead of nacaddrs).
      ret_array.extend(IP(x) for x in iputils.exclude_address(addr, exclude))
    else:
      ret_array.append(addr)
  return SortAddrList(ret_array)

def AddressListExclude(superset, excludes, collapse_addrs=True):
  """Remove a list of addresses from another list of addresses.

  Args:
    superset: a List of nacaddr IPv4 or IPv6 addresses
    excludes: a List nacaddr IPv4 or IPv6 addresses
    collapse_addrs: whether or not to collapse contiguous CIDRs togethe

  Returns:
    a List of nacaddr IPv4 or IPv6 addresses
  """
  if collapse_addrs:
    superset = CollapseAddrList(superset)[::-1]
    excludes = CollapseAddrList(excludes)[::-1]
  else:
    superset = sorted(superset, reverse=True)
    excludes = sorted(excludes, reverse=True)

  ret_array = []
  while superset and excludes:
    if superset[-1].overlaps(excludes[-1]):
      ip = superset.pop()
      superset.extend(
          reversed(RemoveAddressFromList([ip], excludes[-1])))
    elif superset[-1]._get_networks_key() < excludes[-1]._get_networks_key():  # pylint: disable=protected-access
      ret_array.append(superset.pop())
    else:
      excludes.pop()
  if collapse_addrs:
    return CollapseAddrList(ret_array + superset)
  else:
    return sorted(set(ret_array + superset))

ExcludeAddrs = AddressListExclude

class PrefixlenDiffInvalidError(ipaddress.NetmaskValueError):
  """Holdover from ipaddr v1."""

if __name__ == '__main__':
  pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
from os import path as op
import re
from time import sleep
import logging
from pprint import pformat as pf
from pkg_resources import resource_filename as pkgrf
from builtins import object

from cappat import AGAVE_JOB_LOGS, AGAVE_JOB_OUTPUT
from ..tpl import Template
from ..utils import check_folder

from .tools import (
    time_fraction as _tf,
    format_modules as _format_modules,
    run_cmd as _run_cmd)

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    _settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = None
    GROUP_TEMPLATE = op.abspath(pkgrf('cappat', 'tpl/group-wrapper.jnj2'))

    def __init__(self, task_list, settings=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list
        self._jobs = {}

        if settings is not None:
            self._settings.update(settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self._settings['child_runtime'] = _tf(self._settings['max_runtime'])

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self._settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )

        self._group_cmd = [self._settings['executable'], self._settings['bids_dir'],
                           AGAVE_JOB_OUTPUT, 'group']

        if self._settings.get('participant_label', None):
            part_labels = self._settings['participant_label']
            if not isinstance(part_labels, list):
                part_labels = [part_labels]
            self._group_cmd += ['%s' % part for part in part_labels]

        if self._settings.get('group_args'):
            self._group_cmd += [self._settings.get('group_args')]

        JOB_LOG.info('Automatically inferred group level command: "%s"',
                     ' '.join(self.group_cmd))

        self._settings['modules'] = _format_modules(self._settings.get('modules', []))
        JOB_LOG.info('Created TaskManager type "%s" with default settings: \n\t%s',
                     self.__class__.__name__, pf(self._settings))

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    @group_cmd.setter
    def group_cmd(self, value):
        if not isinstance(value, list):
            value = [value]

        self._group_cmd = value

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        regexp = re.compile('(?P<jobid>\\d*) +(?P<status>\\w*)\\+? +'
                            '(?P<exit_code>\\d+):\\d+')
        exit_codes = []
        for line in results.split('\n'):
            m = regexp.search(line)
            if m is not None and all(m.groups()):
                self._jobs[m.group('jobid')] = m.group('status')
                if not m.group('status').startswith('CANCEL'):
                    exit_codes.append(int(m.group('exit_code')))
                else:
                    exit_codes.append(128)

        return exit_codes

    def _get_jobs_status(self):
        squeue = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if squeue is None:
            JOB_LOG.warn('Command "squeue" was empty: jobs are completed.')
            return True

        if 'Invalid job id specified' in squeue:
            JOB_LOG.warn('Jobs completed - squeue: %s', squeue)
            return True

        pending = []
        sqexp = re.compile('(?P<jobid>\\d*),(?P<jobstatus>[' +
                           '|'.join(SLURM_WAIT_STATUS + SLURM_FAIL_STATUS) + ']*)')
        statuses = squeue.split('\n')
        for line in statuses:
            m = sqexp.search(line)
            if m is not None and all(m.groups()):
                status = m.groups()
                self._jobs[status[0]] = status[1]

                if status[1] in SLURM_FAIL_STATUS:
                    JOB_LOG.warn('Job id %s failed (%s).', *status)
                else:
                    pending.append(status[0])

        if pending:
            JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
            return False

        JOB_LOG.info('Jobs %s not present in squeue list, finishing polling.',
                     ', '.join(self.job_ids))
        return True

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        sbatch_files = self._generate_sbatch()
        for i, task in enumerate(sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s', ', '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['{0} (logfiles: log/bidsapp-{0}.{{err,out}}).'.format(k) for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n'
                             '\t%s', '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if not self.group_cmd:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return True

        JOB_LOG.info('Kicking off reduce operation')
        group_wrapper = 'group-wrapper.sh'
        conf = Template(self.GROUP_TEMPLATE)
        conf.generate_conf({
            'modules': self._settings.get('modules', []),
            'cmdline': ' '.join(self.group_cmd)
        }, group_wrapper)

        if _run_cmd(['/bin/bash', group_wrapper]):
            JOB_LOG.info('Group level finished successfully.')
            return True
        return False
<commit_msg>fix forgotten --participant_label in group command<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
from os import path as op
import re
from time import sleep
import logging
from pprint import pformat as pf
from pkg_resources import resource_filename as pkgrf
from builtins import object

from cappat import AGAVE_JOB_LOGS, AGAVE_JOB_OUTPUT
from ..tpl import Template
from ..utils import check_folder

from .tools import (
    time_fraction as _tf,
    format_modules as _format_modules,
    run_cmd as _run_cmd)

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    _settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = None
    GROUP_TEMPLATE = op.abspath(pkgrf('cappat', 'tpl/group-wrapper.jnj2'))

    def __init__(self, task_list, settings=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list
        self._jobs = {}

        if settings is not None:
            self._settings.update(settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self._settings['child_runtime'] = _tf(self._settings['max_runtime'])

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self._settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )

        self._group_cmd = [self._settings['executable'], self._settings['bids_dir'],
                           AGAVE_JOB_OUTPUT, 'group']

        if self._settings.get('participant_label', None):
            part_labels = self._settings['participant_label']
            if not isinstance(part_labels, list):
                part_labels = [part_labels]
            self._group_cmd += ['--participant_label']
            self._group_cmd += ['%s' % part for part in part_labels]

        if self._settings.get('group_args'):
            self._group_cmd += [self._settings.get('group_args')]

        JOB_LOG.info('Automatically inferred group level command: "%s"',
                     ' '.join(self.group_cmd))

        self._settings['modules'] = _format_modules(self._settings.get('modules', []))
        JOB_LOG.info('Created TaskManager type "%s" with default settings: \n\t%s',
                     self.__class__.__name__, pf(self._settings))

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    @group_cmd.setter
    def group_cmd(self, value):
        if not isinstance(value, list):
            value = [value]

        self._group_cmd = value

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        regexp = re.compile('(?P<jobid>\\d*) +(?P<status>\\w*)\\+? +'
                            '(?P<exit_code>\\d+):\\d+')
        exit_codes = []
        for line in results.split('\n'):
            m = regexp.search(line)
            if m is not None and all(m.groups()):
                self._jobs[m.group('jobid')] = m.group('status')
                if not m.group('status').startswith('CANCEL'):
                    exit_codes.append(int(m.group('exit_code')))
                else:
                    exit_codes.append(128)

        return exit_codes

    def _get_jobs_status(self):
        squeue = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if squeue is None:
            JOB_LOG.warn('Command "squeue" was empty: jobs are completed.')
            return True

        if 'Invalid job id specified' in squeue:
            JOB_LOG.warn('Jobs completed - squeue: %s', squeue)
            return True

        pending = []
        sqexp = re.compile('(?P<jobid>\\d*),(?P<jobstatus>[' +
                           '|'.join(SLURM_WAIT_STATUS + SLURM_FAIL_STATUS) + ']*)')
        statuses = squeue.split('\n')
        for line in statuses:
            m = sqexp.search(line)
            if m is not None and all(m.groups()):
                status = m.groups()
                self._jobs[status[0]] = status[1]

                if status[1] in SLURM_FAIL_STATUS:
                    JOB_LOG.warn('Job id %s failed (%s).', *status)
                else:
                    pending.append(status[0])

        if pending:
            JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
            return False

        JOB_LOG.info('Jobs %s not present in squeue list, finishing polling.',
                     ', '.join(self.job_ids))
        return True

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        sbatch_files = self._generate_sbatch()
        for i, task in enumerate(sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s', ', '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['{0} (logfiles: log/bidsapp-{0}.{{err,out}}).'.format(k) for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n'
                             '\t%s', '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if not self.group_cmd:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return True

        JOB_LOG.info('Kicking off reduce operation')
        group_wrapper = 'group-wrapper.sh'
        conf = Template(self.GROUP_TEMPLATE)
        conf.generate_conf({
            'modules': self._settings.get('modules', []),
            'cmdline': ' '.join(self.group_cmd)
        }, group_wrapper)

        if _run_cmd(['/bin/bash', group_wrapper]):
            JOB_LOG.info('Group level finished successfully.')
            return True
        return False
<|endoftext|>


<commit_before>import os
from nose.tools import *

class TempDir(object):
    'temporary directory that is automatically deleted when object is released'
    def __init__(self):
        import tempfile
        self.path = tempfile.mkdtemp()
    def __str__(self):
        return self.path
    def __del__(self):
        'recursively delete the temp dir and its subdirs'
        for dirpath,subdirs,files in os.walk(self.path):
            for name in files: # DELETE ALL FILES IN dirpath
                os.remove(os.path.join(dirpath,name))
            os.rmdir(dirpath) # FINALLY DELETE dirpath DIRECTORY
    def subfile(self,name):
        'return full path by appending name to temp dir path'
        return os.path.join(self.path,name)
    def copyFile(self,path):
        'copy file into the temp dir and return its new path'
        infile = file(path)
        filename = self.subfile(os.path.basename(path))
        outfile = file(filename,'w')
        outfile.write(infile.read())
        infile.close()
        outfile.close()
        return filename

def get_pygr_data_path(newpath=''):
    'force pygr.Data to use newpath, without side-effects on environment'
    import pygr.Data
    pygr.Data.pygrDataPath = newpath
    reload(pygr.Data)
    del pygr.Data.pygrDataPath
    return pygr.Data

class TempPygrData(TempDir):
    'restrict pygr.Data to an initially empty temp directory'
    def __init__(self):
        TempDir.__init__(self)
        self.force_reload(str(self))
    def force_reload(self,newpath=None):
        if newpath is None:
            newpath = self.pygrdatapath
        else:
            self.pygrdatapath = newpath
        return get_pygr_data_path(newpath)
    def __del__(self):
        get_pygr_data_path(None)

class TempPygrDataMySQL(TempPygrData):
    'restrict pygr.Data to an initially empty MySQL resource database'
    def __init__(self,dbname='test',args=''):
        TempDir.__init__(self) # GENERATE A TEMPORARY TABLENAME
        import random
        l = [c for c in 'TeMpBiGdAcDy']
        random.shuffle(l)
        tablename = dbname+'.'+''.join(l)
        import pygr.Data
        pygr.Data.ResourceDBMySQL(tablename+args,createLayer='temp') # CREATE TABLE
        self.tablename = tablename
        self.force_reload('mysql:'+tablename+args) # RELOAD PYGR.DATA USING NEW TABLE
    def __del__(self):
        'drop the temporary resource database table'
        TempDir.__del__(self)
        try:
            t = self.tablename
        except AttributeError: # APPARENTLY NO TABLE CREATED, SO NOTHING TO DO.
            pass
        else:
            import pygr.Data
            cursor = pygr.Data.getResource.db[0].cursor
            cursor.execute('drop table if exists %s' % self.tablename)
            cursor.execute('drop table if exists %s_schema' % self.tablename)
            try:
                del pygr.Data.getResource.layer['temp'] # REMOVE FROM LAYER INDEX
            except KeyError:
                pass
        get_pygr_data_path(None)

def skiptest():
    'cause nose to skip the current test case'
    import nose
    raise nose.SkipTest

def skip_errors(*skipErrors):
    'decorator will force skipping of tests on specified error types'
    def decorate(f):
        def new_f(*args,**kwargs):
            try:
                return f(*args,**kwargs)
            except skipErrors:
                skiptest()
        return new_f
    return decorate

class PygrDataTextFile(object):
    '''dict interface to a text file storage that is pygr.Data-smart,
    i.e. it uses pygr.Data.getResource.loads(), so data will be saved
    and loaded in terms of pygr.Data resource IDs, which will be loaded
    from pygr.Data in the usual way.  Intended for storing test results
    in a platform-independent text format.'''
    def __init__(self,path,mode='r'):
        'open in mode r, a or w'
        self.path = path
        self.mode = mode
        if mode=='r' or mode=='a':
            ifile = file(path)
            import pickle
            self.d = pickle.load(ifile)
            ifile.close()
        elif mode=='w':
            self.d = {}
        else:
            raise ValueError('unknown file mode %s.  Use r, w, or a.' % mode)
    def __getitem__(self,k):
        s = self.d[k]
        import pygr.Data
        return pygr.Data.getResource.loads(s)
    def __setitem__(self,k,obj):
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        import pygr.Data
        s = pygr.Data.getResource.dumps(obj)
        self.d[k] = s
        self.save()
    def __delitem__(self,k):
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        del self.d[k]
        self.save()
    def __iter__(self): return iter(self.d)
    def save(self):
        'save our dictionary to text file by pickling'
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        ifile = file(self.path,'w')
        import pickle
        pickle.dump(self.d,ifile)
        ifile.close()

def find_unused_port(port=5123):
    'look for an unused port begining at the specified port number.'
    import xmlrpclib,socket
    while port<9999:
        s = xmlrpclib.ServerProxy('http://localhost:%d' %port)
        try:
            s.listMethods()
            port += 1
        except socket.error:
            return port
    raise OSError('unable to find any open port')

class TestXMLRPCServer(object):
    """runs XMLRPC server in the background with a list of pygr.Data resources
    Makes server exit when this object is released.
    Because we want this to work even on Windows (gag! choke!),
    we can't use fork, backgrounding or any other quasi-sensible method for
    running the server process in the background.  So we just use a separate
    thread to keep our caller from blocking..."""
    def __init__(self,*pygrDataNames):
        'starts server, returns without blocking'
        self.port = find_unused_port()
        import pygr.Data
        for name in pygrDataNames: # ENSURE ALL RES FOR THE TEST ARE AVAILABLE
            obj = pygr.Data.getResource(name)
        self.pygrDataNames = pygrDataNames
        from threading import Thread
        t = Thread(target=self.run_server)
        t.start()
        import time
        time.sleep(1) # WAIT TO MAKE SURE THE CHILD IS STARTED
    def run_server(self):
        'this method blocks, so run it in a separate thread'
        print 'starting server on port',self.port
        os.system('python pygrdata_server.py %d %s'
                  %(self.port,' '.join(self.pygrDataNames)))
        print 'server exited.'
    def access_server(self):
        'force pygr.Data to only use the XMLRPC server'
        return get_pygr_data_path('http://localhost:%d' % self.port)
    def close(self):
        import xmlrpclib
        s = xmlrpclib.ServerProxy('http://localhost:%d' % self.port)
        s.exit_now() # TELL THE SERVER TO EXIT

class TestBase(object):
    '''base class for tests that can skip on setup errors.
       You can subclass the following attributes:

       Class attribute _skipSetupErrors gives tuple of
       setup error types that will cause the test to be skipped.

       Class attribute _testLevel, if provided, should be an
       integer indicating the intensity level of the test,
       starting from 0 (lowest).  This will be compared against
       the environment variable PYGR_TEST_LEVEL, and if greater,
       will force skipping of this test class.'''
    _skipSetupErrors = (KeyError,AttributeError,IOError)
    def setup(self):
        if not self.is_approved():
            skiptest()
        try:
            m = self.try_setup
        except AttributeError:
            return
        try:
            m()
        except self._skipSetupErrors:
            skiptest()
    def is_approved(self):
        'True if this test class is approved'
        try:
            level = self._testLevel
        except AttributeError:
            return True # NO LEVEL, SO NO APPROVAL REQUIRED
        import os
        try:
            approved = int(os.environ['PYGR_TEST_LEVEL'])
        except (KeyError,ValueError):
            approved = 0
        if level>approved:
            return False
        else:
            return True

##     def teardown(self):
##         try:
##             self.tear_me_down()
##         except KeyError:
##             skiptest()

<commit_msg>added approximate_cmp() function<commit_after>import os
from nose.tools import *

class TempDir(object):
    'temporary directory that is automatically deleted when object is released'
    def __init__(self):
        import tempfile
        self.path = tempfile.mkdtemp()
    def __str__(self):
        return self.path
    def __del__(self):
        'recursively delete the temp dir and its subdirs'
        for dirpath,subdirs,files in os.walk(self.path):
            for name in files: # DELETE ALL FILES IN dirpath
                os.remove(os.path.join(dirpath,name))
            os.rmdir(dirpath) # FINALLY DELETE dirpath DIRECTORY
    def subfile(self,name):
        'return full path by appending name to temp dir path'
        return os.path.join(self.path,name)
    def copyFile(self,path):
        'copy file into the temp dir and return its new path'
        infile = file(path)
        filename = self.subfile(os.path.basename(path))
        outfile = file(filename,'w')
        outfile.write(infile.read())
        infile.close()
        outfile.close()
        return filename

def get_pygr_data_path(newpath=''):
    'force pygr.Data to use newpath, without side-effects on environment'
    import pygr.Data
    pygr.Data.pygrDataPath = newpath
    reload(pygr.Data)
    del pygr.Data.pygrDataPath
    return pygr.Data

class TempPygrData(TempDir):
    'restrict pygr.Data to an initially empty temp directory'
    def __init__(self):
        TempDir.__init__(self)
        self.force_reload(str(self))
    def force_reload(self,newpath=None):
        if newpath is None:
            newpath = self.pygrdatapath
        else:
            self.pygrdatapath = newpath
        return get_pygr_data_path(newpath)
    def __del__(self):
        get_pygr_data_path(None)

class TempPygrDataMySQL(TempPygrData):
    'restrict pygr.Data to an initially empty MySQL resource database'
    def __init__(self,dbname='test',args=''):
        TempDir.__init__(self) # GENERATE A TEMPORARY TABLENAME
        import random
        l = [c for c in 'TeMpBiGdAcDy']
        random.shuffle(l)
        tablename = dbname+'.'+''.join(l)
        import pygr.Data
        pygr.Data.ResourceDBMySQL(tablename+args,createLayer='temp') # CREATE TABLE
        self.tablename = tablename
        self.force_reload('mysql:'+tablename+args) # RELOAD PYGR.DATA USING NEW TABLE
    def __del__(self):
        'drop the temporary resource database table'
        TempDir.__del__(self)
        try:
            t = self.tablename
        except AttributeError: # APPARENTLY NO TABLE CREATED, SO NOTHING TO DO.
            pass
        else:
            import pygr.Data
            cursor = pygr.Data.getResource.db[0].cursor
            cursor.execute('drop table if exists %s' % self.tablename)
            cursor.execute('drop table if exists %s_schema' % self.tablename)
            try:
                del pygr.Data.getResource.layer['temp'] # REMOVE FROM LAYER INDEX
            except KeyError:
                pass
        get_pygr_data_path(None)

def skiptest():
    'cause nose to skip the current test case'
    import nose
    raise nose.SkipTest

def skip_errors(*skipErrors):
    'decorator will force skipping of tests on specified error types'
    def decorate(f):
        def new_f(*args,**kwargs):
            try:
                return f(*args,**kwargs)
            except skipErrors:
                skiptest()
        return new_f
    return decorate

class PygrDataTextFile(object):
    '''dict interface to a text file storage that is pygr.Data-smart,
    i.e. it uses pygr.Data.getResource.loads(), so data will be saved
    and loaded in terms of pygr.Data resource IDs, which will be loaded
    from pygr.Data in the usual way.  Intended for storing test results
    in a platform-independent text format.'''
    def __init__(self,path,mode='r'):
        'open in mode r, a or w'
        self.path = path
        self.mode = mode
        if mode=='r' or mode=='a':
            ifile = file(path)
            import pickle
            self.d = pickle.load(ifile)
            ifile.close()
        elif mode=='w':
            self.d = {}
        else:
            raise ValueError('unknown file mode %s.  Use r, w, or a.' % mode)
    def __getitem__(self,k):
        s = self.d[k]
        import pygr.Data
        return pygr.Data.getResource.loads(s)
    def __setitem__(self,k,obj):
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        import pygr.Data
        s = pygr.Data.getResource.dumps(obj)
        self.d[k] = s
        self.save()
    def __delitem__(self,k):
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        del self.d[k]
        self.save()
    def __iter__(self): return iter(self.d)
    def save(self):
        'save our dictionary to text file by pickling'
        if self.mode=='r':
            raise ValueError('this PygrDataTextFile was opened read-only! Use append mode')
        ifile = file(self.path,'w')
        import pickle
        pickle.dump(self.d,ifile)
        ifile.close()

def find_unused_port(port=5123):
    'look for an unused port begining at the specified port number.'
    import xmlrpclib,socket
    while port<9999:
        s = xmlrpclib.ServerProxy('http://localhost:%d' %port)
        try:
            s.listMethods()
            port += 1
        except socket.error:
            return port
    raise OSError('unable to find any open port')

class TestXMLRPCServer(object):
    """runs XMLRPC server in the background with a list of pygr.Data resources
    Makes server exit when this object is released.
    Because we want this to work even on Windows (gag! choke!),
    we can't use fork, backgrounding or any other quasi-sensible method for
    running the server process in the background.  So we just use a separate
    thread to keep our caller from blocking..."""
    def __init__(self,*pygrDataNames):
        'starts server, returns without blocking'
        self.port = find_unused_port()
        import pygr.Data
        for name in pygrDataNames: # ENSURE ALL RES FOR THE TEST ARE AVAILABLE
            obj = pygr.Data.getResource(name)
        self.pygrDataNames = pygrDataNames
        from threading import Thread
        t = Thread(target=self.run_server)
        t.start()
        import time
        time.sleep(1) # WAIT TO MAKE SURE THE CHILD IS STARTED
    def run_server(self):
        'this method blocks, so run it in a separate thread'
        print 'starting server on port',self.port
        os.system('python pygrdata_server.py %d %s'
                  %(self.port,' '.join(self.pygrDataNames)))
        print 'server exited.'
    def access_server(self):
        'force pygr.Data to only use the XMLRPC server'
        return get_pygr_data_path('http://localhost:%d' % self.port)
    def close(self):
        import xmlrpclib
        s = xmlrpclib.ServerProxy('http://localhost:%d' % self.port)
        s.exit_now() # TELL THE SERVER TO EXIT

def approximate_cmp(x,y,delta):
    '''expects two lists of tuples.  Performs comparison as usual,
    except that numeric types are considered equal if they differ by
    less than delta'''
    diff = cmp(len(x),len(y))
    if diff != 0:
        return diff
    x.sort() # SORT TO ENSURE IN SAME ORDER...
    y.sort()
    for i in range(len(x)):
        s = x[i]
        t = y[i]
        diff = cmp(len(s),len(t))
        if diff != 0:
            return diff
        for j in range(len(s)):
            u = s[j]
            v = t[j]
            if isinstance(u,int) or isinstance(u,float):
                diff = u - v
                if diff < -delta:
                    return -1
                elif diff >delta:
                    return 1
            else:
                diff = cmp(u,v)
                if diff != 0:
                    return diff
    return 0

class TestBase(object):
    '''base class for tests that can skip on setup errors.
       You can subclass the following attributes:

       Class attribute _skipSetupErrors gives tuple of
       setup error types that will cause the test to be skipped.

       Class attribute _testLevel, if provided, should be an
       integer indicating the intensity level of the test,
       starting from 0 (lowest).  This will be compared against
       the environment variable PYGR_TEST_LEVEL, and if greater,
       will force skipping of this test class.'''
    _skipSetupErrors = (KeyError,AttributeError,IOError)
    def setup(self):
        if not self.is_approved():
            skiptest()
        try:
            m = self.try_setup
        except AttributeError:
            return
        try:
            m()
        except self._skipSetupErrors:
            skiptest()
    def is_approved(self):
        'True if this test class is approved'
        try:
            level = self._testLevel
        except AttributeError:
            return True # NO LEVEL, SO NO APPROVAL REQUIRED
        import os
        try:
            approved = int(os.environ['PYGR_TEST_LEVEL'])
        except (KeyError,ValueError):
            approved = 0
        if level>approved:
            return False
        else:
            return True

##     def teardown(self):
##         try:
##             self.tear_me_down()
##         except KeyError:
##             skiptest()

<|endoftext|>


<commit_before>import os
import configparser
import shutil
import subprocess

# Setup
print("Setting up...")
shutil.copyfile("../halite.ini", "temp.ini")
shutil.copyfile("tests.ini", "../halite.ini")
parser = configparser.ConfigParser()
parser.read("../halite.ini")

# Website tests
print("Beginning website backend tests")
os.system("mysql -u "+parser["database"]["username"]+" -p"+parser["database"]["password"]+" < ../website/sql/Database.sql")
subprocess.call(["phpunit", "--stderr", "UserTest", "WebsiteTests.php"])

# Tear down
print("Almost done...")
shutil.copyfile("temp.ini", "../halite.ini")
<commit_msg>Added env tests to runTests.py<commit_after>import os
import configparser
import shutil
import subprocess

# Setup
print("Setting up...")
shutil.copyfile("../halite.ini", "temp.ini")
shutil.copyfile("tests.ini", "../halite.ini")
parser = configparser.ConfigParser()
parser.read("../halite.ini")

# Website tests
print("Beginning website backend tests")
os.system("mysql -u "+parser["database"]["username"]+" -p"+parser["database"]["password"]+" < ../website/sql/Database.sql")
subprocess.call(["phpunit", "--stderr", "UserTest", "WebsiteTests.php"])

# Environment tests.
print(subprocess.Popen('cd environment; python3 testenv.py', stdout=subprocess.PIPE, shell = True).stdout.read().decode('utf-8'))

# Tear down
print("Almost done...")
shutil.copyfile("temp.ini", "../halite.ini")<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, sys, traceback
import unittest

import django.contrib as contrib

try:
    set
except NameError:
    from sets import Set as set     # For Python 2.3

CONTRIB_DIR_NAME = 'django.contrib'
MODEL_TESTS_DIR_NAME = 'modeltests'
REGRESSION_TESTS_DIR_NAME = 'regressiontests'

TEST_TEMPLATE_DIR = 'templates'

CONTRIB_DIR = os.path.dirname(contrib.__file__)
MODEL_TEST_DIR = os.path.join(os.path.dirname(__file__), MODEL_TESTS_DIR_NAME)
REGRESSION_TEST_DIR = os.path.join(os.path.dirname(__file__), REGRESSION_TESTS_DIR_NAME)

ALWAYS_INSTALLED_APPS = [
    'django.contrib.contenttypes',
    'django.contrib.auth',
    'django.contrib.sites',
    'django.contrib.flatpages',
    'django.contrib.redirects',
    'django.contrib.sessions',
    'django.contrib.comments',
    'django.contrib.admin',
]

def get_test_models():
    models = []
    for loc, dirpath in (MODEL_TESTS_DIR_NAME, MODEL_TEST_DIR), (REGRESSION_TESTS_DIR_NAME, REGRESSION_TEST_DIR), (CONTRIB_DIR_NAME, CONTRIB_DIR):
        for f in os.listdir(dirpath):
            if f.startswith('__init__') or f.startswith('.') or f.startswith('sql') or f.startswith('invalid'):
                continue
            models.append((loc, f))
    return models

def get_invalid_models():
    models = []
    for loc, dirpath in (MODEL_TESTS_DIR_NAME, MODEL_TEST_DIR), (REGRESSION_TESTS_DIR_NAME, REGRESSION_TEST_DIR), (CONTRIB_DIR_NAME, CONTRIB_DIR):
        for f in os.listdir(dirpath):
            if f.startswith('__init__') or f.startswith('.') or f.startswith('sql'):
                continue
            if f.startswith('invalid'):
                models.append((loc, f))
    return models

class InvalidModelTestCase(unittest.TestCase):
    def __init__(self, model_label):
        unittest.TestCase.__init__(self)
        self.model_label = model_label

    def runTest(self):
        from django.core.management.validation import get_validation_errors
        from django.db.models.loading import load_app
        from cStringIO import StringIO

        try:
            module = load_app(self.model_label)
        except Exception, e:
            self.fail('Unable to load invalid model module')

        # Make sure sys.stdout is not a tty so that we get errors without
        # coloring attached (makes matching the results easier). We restore
        # sys.stderr afterwards.
        orig_stdout = sys.stdout
        s = StringIO()
        sys.stdout = s
        count = get_validation_errors(s, module)
        sys.stdout = orig_stdout
        s.seek(0)
        error_log = s.read()
        actual = error_log.split('\n')
        expected = module.model_errors.split('\n')

        unexpected = [err for err in actual if err not in expected]
        missing = [err for err in expected if err not in actual]

        self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
        self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))

def django_tests(verbosity, interactive, test_labels):
    from django.conf import settings

    old_installed_apps = settings.INSTALLED_APPS
    old_test_database_name = settings.TEST_DATABASE_NAME
    old_root_urlconf = settings.ROOT_URLCONF
    old_template_dirs = settings.TEMPLATE_DIRS
    old_use_i18n = settings.USE_I18N
    old_language_code = settings.LANGUAGE_CODE
    old_middleware_classes = settings.MIDDLEWARE_CLASSES

    # Redirect some settings for the duration of these tests.
    settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
    settings.ROOT_URLCONF = 'urls'
    settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), TEST_TEMPLATE_DIR),)
    settings.USE_I18N = True
    settings.LANGUAGE_CODE = 'en'
    settings.MIDDLEWARE_CLASSES = (
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.middleware.common.CommonMiddleware',
    )
    settings.SITE_ID = 1

    # Load all the ALWAYS_INSTALLED_APPS.
    # (This import statement is intentionally delayed until after we
    # access settings because of the USE_I18N dependency.)
    from django.db.models.loading import get_apps, load_app
    get_apps()

    # Load all the test model apps.
    test_models = []
    for model_dir, model_name in get_test_models():
        model_label = '.'.join([model_dir, model_name])
        try:
            # if the model was named on the command line, or
            # no models were named (i.e., run all), import
            # this model and add it to the list to test.
            if not test_labels or model_name in set([label.split('.')[0] for label in test_labels]):
                if verbosity >= 1:
                    print "Importing model %s" % model_name
                mod = load_app(model_label)
                if mod:
                    if model_label not in settings.INSTALLED_APPS:
                        settings.INSTALLED_APPS.append(model_label)
        except Exception, e:
            sys.stderr.write("Error while importing %s:" % model_name + ''.join(traceback.format_exception(*sys.exc_info())[1:]))
            continue

    # Add tests for invalid models.
    extra_tests = []
    for model_dir, model_name in get_invalid_models():
        model_label = '.'.join([model_dir, model_name])
        if not test_labels or model_name in test_labels:
            extra_tests.append(InvalidModelTestCase(model_label))

    # Run the test suite, including the extra validation tests.
    from django.test.simple import run_tests
    failures = run_tests(test_labels, verbosity=verbosity, interactive=interactive, extra_tests=extra_tests)
    if failures:
        sys.exit(failures)

    # Restore the old settings.
    settings.INSTALLED_APPS = old_installed_apps
    settings.ROOT_URLCONF = old_root_urlconf
    settings.TEMPLATE_DIRS = old_template_dirs
    settings.USE_I18N = old_use_i18n
    settings.LANGUAGE_CODE = old_language_code
    settings.MIDDLEWARE_CLASSES = old_middleware_classes

if __name__ == "__main__":
    from optparse import OptionParser
    usage = "%prog [options] [model model model ...]"
    parser = OptionParser(usage=usage)
    parser.add_option('-v','--verbosity', action='store', dest='verbosity', default='0',
        type='choice', choices=['0', '1', '2'],
        help='Verbosity level; 0=minimal output, 1=normal output, 2=all output')
    parser.add_option('--noinput', action='store_false', dest='interactive', default=True,
        help='Tells Django to NOT prompt the user for input of any kind.')
    parser.add_option('--settings',
        help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
    options, args = parser.parse_args()
    if options.settings:
        os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
    elif "DJANGO_SETTINGS_MODULE" not in os.environ:
        parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
                      "Set it or use --settings.")
    django_tests(int(options.verbosity), options.interactive, args)
<commit_msg>Fixed #6200 -- Made the core tests independent of the value of LOGIN_URL, since they are sensitive to its setting. Thanks, Jason Yan.<commit_after>#!/usr/bin/env python

import os, sys, traceback
import unittest

import django.contrib as contrib

try:
    set
except NameError:
    from sets import Set as set     # For Python 2.3

CONTRIB_DIR_NAME = 'django.contrib'
MODEL_TESTS_DIR_NAME = 'modeltests'
REGRESSION_TESTS_DIR_NAME = 'regressiontests'

TEST_TEMPLATE_DIR = 'templates'

CONTRIB_DIR = os.path.dirname(contrib.__file__)
MODEL_TEST_DIR = os.path.join(os.path.dirname(__file__), MODEL_TESTS_DIR_NAME)
REGRESSION_TEST_DIR = os.path.join(os.path.dirname(__file__), REGRESSION_TESTS_DIR_NAME)

ALWAYS_INSTALLED_APPS = [
    'django.contrib.contenttypes',
    'django.contrib.auth',
    'django.contrib.sites',
    'django.contrib.flatpages',
    'django.contrib.redirects',
    'django.contrib.sessions',
    'django.contrib.comments',
    'django.contrib.admin',
]

def get_test_models():
    models = []
    for loc, dirpath in (MODEL_TESTS_DIR_NAME, MODEL_TEST_DIR), (REGRESSION_TESTS_DIR_NAME, REGRESSION_TEST_DIR), (CONTRIB_DIR_NAME, CONTRIB_DIR):
        for f in os.listdir(dirpath):
            if f.startswith('__init__') or f.startswith('.') or f.startswith('sql') or f.startswith('invalid'):
                continue
            models.append((loc, f))
    return models

def get_invalid_models():
    models = []
    for loc, dirpath in (MODEL_TESTS_DIR_NAME, MODEL_TEST_DIR), (REGRESSION_TESTS_DIR_NAME, REGRESSION_TEST_DIR), (CONTRIB_DIR_NAME, CONTRIB_DIR):
        for f in os.listdir(dirpath):
            if f.startswith('__init__') or f.startswith('.') or f.startswith('sql'):
                continue
            if f.startswith('invalid'):
                models.append((loc, f))
    return models

class InvalidModelTestCase(unittest.TestCase):
    def __init__(self, model_label):
        unittest.TestCase.__init__(self)
        self.model_label = model_label

    def runTest(self):
        from django.core.management.validation import get_validation_errors
        from django.db.models.loading import load_app
        from cStringIO import StringIO

        try:
            module = load_app(self.model_label)
        except Exception, e:
            self.fail('Unable to load invalid model module')

        # Make sure sys.stdout is not a tty so that we get errors without
        # coloring attached (makes matching the results easier). We restore
        # sys.stderr afterwards.
        orig_stdout = sys.stdout
        s = StringIO()
        sys.stdout = s
        count = get_validation_errors(s, module)
        sys.stdout = orig_stdout
        s.seek(0)
        error_log = s.read()
        actual = error_log.split('\n')
        expected = module.model_errors.split('\n')

        unexpected = [err for err in actual if err not in expected]
        missing = [err for err in expected if err not in actual]

        self.assert_(not unexpected, "Unexpected Errors: " + '\n'.join(unexpected))
        self.assert_(not missing, "Missing Errors: " + '\n'.join(missing))

def django_tests(verbosity, interactive, test_labels):
    from django.conf import settings

    old_installed_apps = settings.INSTALLED_APPS
    old_test_database_name = settings.TEST_DATABASE_NAME
    old_root_urlconf = settings.ROOT_URLCONF
    old_template_dirs = settings.TEMPLATE_DIRS
    old_use_i18n = settings.USE_I18N
    old_login_url = settings.LOGIN_URL
    old_language_code = settings.LANGUAGE_CODE
    old_middleware_classes = settings.MIDDLEWARE_CLASSES

    # Redirect some settings for the duration of these tests.
    settings.INSTALLED_APPS = ALWAYS_INSTALLED_APPS
    settings.ROOT_URLCONF = 'urls'
    settings.TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), TEST_TEMPLATE_DIR),)
    settings.USE_I18N = True
    settings.LANGUAGE_CODE = 'en'
    settings.LOGIN_URL = '/accounts/login/'
    settings.MIDDLEWARE_CLASSES = (
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.middleware.common.CommonMiddleware',
    )
    settings.SITE_ID = 1

    # Load all the ALWAYS_INSTALLED_APPS.
    # (This import statement is intentionally delayed until after we
    # access settings because of the USE_I18N dependency.)
    from django.db.models.loading import get_apps, load_app
    get_apps()

    # Load all the test model apps.
    test_models = []
    for model_dir, model_name in get_test_models():
        model_label = '.'.join([model_dir, model_name])
        try:
            # if the model was named on the command line, or
            # no models were named (i.e., run all), import
            # this model and add it to the list to test.
            if not test_labels or model_name in set([label.split('.')[0] for label in test_labels]):
                if verbosity >= 1:
                    print "Importing model %s" % model_name
                mod = load_app(model_label)
                if mod:
                    if model_label not in settings.INSTALLED_APPS:
                        settings.INSTALLED_APPS.append(model_label)
        except Exception, e:
            sys.stderr.write("Error while importing %s:" % model_name + ''.join(traceback.format_exception(*sys.exc_info())[1:]))
            continue

    # Add tests for invalid models.
    extra_tests = []
    for model_dir, model_name in get_invalid_models():
        model_label = '.'.join([model_dir, model_name])
        if not test_labels or model_name in test_labels:
            extra_tests.append(InvalidModelTestCase(model_label))

    # Run the test suite, including the extra validation tests.
    from django.test.simple import run_tests
    failures = run_tests(test_labels, verbosity=verbosity, interactive=interactive, extra_tests=extra_tests)
    if failures:
        sys.exit(failures)

    # Restore the old settings.
    settings.INSTALLED_APPS = old_installed_apps
    settings.ROOT_URLCONF = old_root_urlconf
    settings.TEMPLATE_DIRS = old_template_dirs
    settings.USE_I18N = old_use_i18n
    settings.LANGUAGE_CODE = old_language_code
    settings.LOGIN_URL = old_login_url
    settings.MIDDLEWARE_CLASSES = old_middleware_classes

if __name__ == "__main__":
    from optparse import OptionParser
    usage = "%prog [options] [model model model ...]"
    parser = OptionParser(usage=usage)
    parser.add_option('-v','--verbosity', action='store', dest='verbosity', default='0',
        type='choice', choices=['0', '1', '2'],
        help='Verbosity level; 0=minimal output, 1=normal output, 2=all output')
    parser.add_option('--noinput', action='store_false', dest='interactive', default=True,
        help='Tells Django to NOT prompt the user for input of any kind.')
    parser.add_option('--settings',
        help='Python path to settings module, e.g. "myproject.settings". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.')
    options, args = parser.parse_args()
    if options.settings:
        os.environ['DJANGO_SETTINGS_MODULE'] = options.settings
    elif "DJANGO_SETTINGS_MODULE" not in os.environ:
        parser.error("DJANGO_SETTINGS_MODULE is not set in the environment. "
                      "Set it or use --settings.")
    django_tests(int(options.verbosity), options.interactive, args)
<|endoftext|>


<commit_before>import os
import sys
import django

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

sys.path.insert(0, os.path.join(BASE_DIR, 'src'))

DEBUG = True

ADMINS = (
    ('admin', 'admin@location'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'test_db.sqlite3',
    }
}

TIME_ZONE = 'America/Chicago'

LANGUAGE_CODE = 'en-us'
USE_I18N = True
USE_L10N = True

ROOT_URLCONF = 'tests.url'

SITE_ID = 1

MEDIA_ROOT = ''

MEDIA_URL = ''

SECRET_KEY = '#($gm0!3&=yy^-3aka=0#y#2b1i$qn51tc$vpmplum1_$az8-='

if django.VERSION >= (1, 10):
    MIDDLEWARE = [
        'django.middleware.security.SecurityMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
    ]
else:
    MIDDLEWARE_CLASSES = [
        'django.middleware.security.SecurityMiddleware',
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
    ]

ROOT_URLCONF = 'tests.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'notification',
    'notification.tests',
)

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
)

TEST_RUNNER = 'notification.tests.compatibility.TestRunner'

STATIC_ROOT = 'staticfiles'
STATIC_URL = '/static/'
<commit_msg>Modify django version's judge<commit_after>import os
import sys
import django

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

sys.path.insert(0, os.path.join(BASE_DIR, 'src'))

DEBUG = True

ADMINS = (
    ('admin', 'admin@location'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': 'test_db.sqlite3',
    }
}

TIME_ZONE = 'America/Chicago'

LANGUAGE_CODE = 'en-us'
USE_I18N = True
USE_L10N = True

ROOT_URLCONF = 'tests.url'

SITE_ID = 1

MEDIA_ROOT = ''

MEDIA_URL = ''

SECRET_KEY = '#($gm0!3&=yy^-3aka=0#y#2b1i$qn51tc$vpmplum1_$az8-='

# if django.VERSION >= (1, 10):
#     MIDDLEWARE = [
#         'django.middleware.security.SecurityMiddleware',
#         'django.contrib.sessions.middleware.SessionMiddleware',
#         'django.middleware.common.CommonMiddleware',
#         'django.middleware.csrf.CsrfViewMiddleware',
#         'django.contrib.auth.middleware.AuthenticationMiddleware',
#         'django.contrib.messages.middleware.MessageMiddleware',
#         'django.middleware.clickjacking.XFrameOptionsMiddleware',
#     ]
# elif django.VERSION <= (1, 7):
#     MIDDLEWARE_CLASSES = [
#         'django.contrib.sessions.middleware.SessionMiddleware',
#         'django.middleware.common.CommonMiddleware',
#         'django.middleware.csrf.CsrfViewMiddleware',
#         'django.contrib.auth.middleware.AuthenticationMiddleware',
#         'django.contrib.messages.middleware.MessageMiddleware',
#         'django.middleware.clickjacking.XFrameOptionsMiddleware',
#     ]
# else:
#     MIDDLEWARE_CLASSES = [
#         'django.middleware.security.SecurityMiddleware',
#         'django.contrib.sessions.middleware.SessionMiddleware',
#         'django.middleware.common.CommonMiddleware',
#         'django.middleware.csrf.CsrfViewMiddleware',
#         'django.contrib.auth.middleware.AuthenticationMiddleware',
#         'django.contrib.messages.middleware.MessageMiddleware',
#         'django.middleware.clickjacking.XFrameOptionsMiddleware',
#     ]
MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
)

ROOT_URLCONF = 'tests.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [os.path.join(BASE_DIR, 'templates')],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'notification',
    'notification.tests',
)

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
)

TEST_RUNNER = 'notification.tests.compatibility.TestRunner'

STATIC_ROOT = 'staticfiles'
STATIC_URL = '/static/'
<|endoftext|>


<commit_before>import re
from mock import call
from common import HandyTestCase

def collapse(s):
    return re.sub('\s+', ' ', s).strip()

class TekNginxTest(HandyTestCase):

    def setUp(self):
        self.reload_ = self.patch('tek.nginx.NginxTek.reload_')

    def nginx_tek(self):
        from tek.nginx import NginxTek
        return NginxTek(sites_dir=str(self.tmp))

    def test_nginx_configure_with_empty_urlmap_creates_blank_site(self):
        self.nginx_tek().configure('zz.example.com:8080', {})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:8080').text()),
                         'server { listen 8080; server_name zz.example.com; }')

    def test_nginx_configure_triggers_nginx_reload(self):
        self.nginx_tek().configure('zz.example.com:8080', {})
        self.assertEqual(self.reload_.mock_calls, [call()])

    def test_nginx_delete_removes_site(self):
        nginx = self.nginx_tek()
        nginx.configure('zz.example.com:8080', {})
        nginx.delete('zz.example.com:8080')
        self.assertEqual(self.tmp.listdir(), [])

    def test_nginx_delete_with_no_site_obeys_nofail_flag(self):
        self.nginx_tek().delete('zz.example.com:8080', nofail=True)

    def test_nginx_delete_triggers_nginx_reload(self):
        nginx = self.nginx_tek()
        nginx.configure('zz.example.com:8080', {})
        self.reload_.reset_mock()
        nginx.delete('zz.example.com:8080')
        self.assertEqual(self.reload_.mock_calls, [call()])

    def test_arbitrary_nginx_options_are_written_to_config(self):
        self.nginx_tek().configure('zz.example.com:80',
                                   {'options': {'send_timeout': '2m'}})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()),
                         ('server { '
                            'listen 80; '
                            'send_timeout 2m; '
                            'server_name zz.example.com; '
                          '}'))

    def test_static_folder_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/media', 'type': 'static', 'path': '/var/local/x'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()),
                         ('server { '
                            'listen 80; '
                            'server_name zz.example.com; '
                            'location /media { '
                              'alias /var/local/x; '
                            '} '
                          '}'))

    def test_fcgi_route_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/app', 'type': 'fcgi', 'socket': 'localhost:24637'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()), (
            'server { '
              'listen 80; '
              'server_name zz.example.com; '
              'location /app { '
                'include /etc/nginx/fastcgi_params; '
                'fastcgi_param PATH_INFO $fastcgi_script_name; '
                'fastcgi_param SCRIPT_NAME ""; '
                'fastcgi_pass localhost:24637; '
              '} '
            '}'))

    def test_http_proxy_route_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/other', 'type': 'proxy',
             'upstream_url': 'http://backend:4912/some/path'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()), (
            'server { '
              'listen 80; '
              'server_name zz.example.com; '
              'location /other { '
                'proxy_pass http://backend:4912/some/path; '
                'proxy_redirect off; '
                'proxy_set_header Host $host; '
                'proxy_set_header X-Real-IP $remote_addr; '
                'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; '
              '} '
            '}'))

    def test_shell_configure_invocation_calls_configure_with_parameters(self):
        configure = self.patch('tek.nginx.NginxTek.configure')
        self.nginx_tek().main(['configure', 'zzz', '-p', '8080', '{"a": "b"}'])
        self.assertEqual(configure.mock_calls,
                         [call(site_name='zzz', port=8080, config={'a': 'b'})])

    def test_shell_delete_invocation_calls_delete_with_parameters(self):
        delete = self.patch('tek.nginx.NginxTek.delete')
        self.nginx_tek().main(['delete', 'zzz', '-p', '8080', '-f'])
        self.assertEqual(delete.mock_calls,
                         [call(site_name='zzz', port=8080, nofail=True)])
<commit_msg>update test to match reality<commit_after>import re
from mock import call
from common import HandyTestCase

def collapse(s):
    return re.sub('\s+', ' ', s).strip()

class TekNginxTest(HandyTestCase):

    def setUp(self):
        self.reload_ = self.patch('tek.nginx.NginxTek.reload_')

    def nginx_tek(self):
        from tek.nginx import NginxTek
        return NginxTek(sites_dir=str(self.tmp))

    def test_nginx_configure_with_empty_urlmap_creates_blank_site(self):
        self.nginx_tek().configure('zz.example.com:8080', {})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:8080').text()),
                         'server { listen 8080; server_name zz.example.com; }')

    def test_nginx_configure_triggers_nginx_reload(self):
        self.nginx_tek().configure('zz.example.com:8080', {})
        self.assertEqual(self.reload_.mock_calls, [call()])

    def test_nginx_delete_removes_site(self):
        nginx = self.nginx_tek()
        nginx.configure('zz.example.com:8080', {})
        nginx.delete('zz.example.com:8080')
        self.assertEqual(self.tmp.listdir(), [])

    def test_nginx_delete_with_no_site_obeys_nofail_flag(self):
        self.nginx_tek().delete('zz.example.com:8080', nofail=True)

    def test_nginx_delete_triggers_nginx_reload(self):
        nginx = self.nginx_tek()
        nginx.configure('zz.example.com:8080', {})
        self.reload_.reset_mock()
        nginx.delete('zz.example.com:8080')
        self.assertEqual(self.reload_.mock_calls, [call()])

    def test_arbitrary_nginx_options_are_written_to_config(self):
        self.nginx_tek().configure('zz.example.com:80',
                                   {'options': {'send_timeout': '2m'}})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()),
                         ('server { '
                            'listen 80; '
                            'send_timeout 2m; '
                            'server_name zz.example.com; '
                          '}'))

    def test_static_folder_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/media', 'type': 'static', 'path': '/var/local/x'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()),
                         ('server { '
                            'listen 80; '
                            'server_name zz.example.com; '
                            'location /media { '
                              'alias /var/local/x; '
                            '} '
                          '}'))

    def test_fcgi_route_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/app', 'type': 'fcgi', 'socket': 'localhost:24637'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()), (
            'server { '
              'listen 80; '
              'server_name zz.example.com; '
              'location /app { '
                'include /etc/nginx/fastcgi_params; '
                'fastcgi_param PATH_INFO $fastcgi_script_name; '
                'fastcgi_param SCRIPT_NAME ""; '
                'fastcgi_pass localhost:24637; '
              '} '
            '}'))

    def test_http_proxy_route_is_configured_in_nginx(self):
        self.nginx_tek().configure('zz.example.com:80', {'urlmap': [
            {'url': '/other', 'type': 'proxy',
             'upstream_url': 'http://backend:4912/some/path'}]})
        self.assertEqual(collapse((self.tmp / 'zz.example.com:80').text()), (
            'server { '
              'listen 80; '
              'server_name zz.example.com; '
              'location /other { '
                'proxy_pass http://backend:4912/some/path; '
                'proxy_redirect off; '
                'proxy_set_header Host $host; '
                'proxy_set_header X-Real-IP $remote_addr; '
                'proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; '
              '} '
            '}'))

    def test_shell_configure_invocation_calls_configure_with_parameters(self):
        configure = self.patch('tek.nginx.NginxTek.configure')
        self.nginx_tek().main(['configure', 'zzz:8080', '{"a": "b"}'])
        self.assertEqual(configure.mock_calls,
                         [call(site_name='zzz:8080', config={'a': 'b'})])

    def test_shell_delete_invocation_calls_delete_with_parameters(self):
        delete = self.patch('tek.nginx.NginxTek.delete')
        self.nginx_tek().main(['delete', 'zzz:8080', '-f'])
        self.assertEqual(delete.mock_calls,
                         [call(site_name='zzz:8080', nofail=True)])
<|endoftext|>