<commit_before>#!/usr/bin/env python
# Nom nom nom nom

# TODO: future feature: colored cli prompts to make packets vs. crashes
#       vs. whatever easy to distinguish

# TODO: rather than just prompting "Continue to next round? [Yn]", allow
#       the user to examine the state of the network interactively (i.e.,
#       provide them with the normal POX cli + the simulated events

import pox.openflow.libopenflow_01 as of
from pox.lib.revent import EventMixin
from debugger_entities import Link, Host

from traffic_generator import TrafficGenerator
from invariant_checker import InvariantChecker

import sys
import threading
import signal
import subprocess
import socket
import time
import random
import os
import logging
import pickle

from sts.console import msg

log = logging.getLogger("debugger")

class FuzzTester (EventMixin):
  """
  This is part of a testing framework for controller applications. It
  acts as a replacement for pox.topology.

  Given a set of event handlers (registered by a controller application),
  it will inject intelligently chosen mock events (and observe
  their responses?)
  """
  def __init__(self, fuzzer_params="fuzzer_params.cfg", interactive=True,
               check_interval=35, trace_interval=10, random_seed=0.0,
               delay=0.1, dataplane_trace=None, control_socket=None):
    self.interactive = interactive
    self.check_interval = check_interval
    self.trace_interval = trace_interval
    # Format of trace file is a pickled array of DataplaneEvent objects
    self.dataplane_trace = None
    if dataplane_trace:
      self.dataplane_trace = pickle.load(file(dataplane_trace))
    self.running = False
    self.panel = None
    self.switch_impls = []

    self.delay = delay

    self._load_fuzzer_params(fuzzer_params)

    # Logical time (round #) for the simulation execution
    self.logical_time = 0

    # Metatdata for simulated failures
    # sts.debugger_entities.Link objects
    self.cut_links = set()
    # SwitchOutDpEvent objects
    self.dropped_dp_events = []
    # SwitchImpl objects
    self.failed_switches = set()
    # topology.Controller objects
    self.failed_controllers = set()
    # ?
    self.cancelled_timeouts = set()

    # Statistics to print on exit
    self.packets_sent = 0

    # Make execution deterministic to allow the user to easily replay
    self.seed = random_seed
    self.random = random.Random(self.seed)
    self.traffic_generator = TrafficGenerator(self.random)
    self.invariant_checker = InvariantChecker(control_socket)

    # TODO: future feature: log all events, and allow user to (interactively)
    # replay the execution
    # self.replay_logger = ReplayLogger()
    # Write the seed out to the replay log as the first 4 bytes

    # TODO: future feature: allow the user to interactively choose the order
    # events occur for each round, whether to delay, drop packets, fail nodes,
    # etc.
    # self.failure_lvl = [
    #   NOTHING,    # Everything is handled by the random number generator
    #   CRASH,      # The user only controls node crashes and restarts
    #   DROP,       # The user also controls message dropping
    #   DELAY,      # The user also controls message delays
    #   EVERYTHING  # The user controls everything, including message ordering
    # ]

    # TODO: need a mechanism for signaling  when the distributed controller handshake has completed

  def _load_fuzzer_params(self, fuzzer_params):
    if os.path.exists(fuzzer_params):
      # TODO: more pythonic way to read lines (currently on a plane...)
      # TODO: even better: there is probably a library for parsing config files
      for line in file(fuzzer_params).read().splitlines():
        if line == "[fuzzer]":
          # TODO: handle more directives other than [fuzzer]
          continue
        (kw, eq, val) = line.split()
        val = float(val)
        setattr(self, kw, val)
    else:
      # TODO: default values in case fuzzer_config is not present / missing directives
      raise IOError("Could not find logging config file: %s" % fuzzer_params)

  def type_check_dataplane_trace(self):
    if self.dataplane_trace is not None:
      for dp_event in self.dataplane_trace:
        if dp_event.interface not in self.interface2host:
          raise RuntimeError("Dataplane trace does not type check (%s)" % str(dp_event.interface))

  def simulate(self, panel, switch_impls, network_links, hosts, access_links, steps=None):
    """
    Start the fuzzer loop!
    """
    log.debug("Starting fuzz loop")
    self.panel = panel
    self.switch_impls = set(switch_impls)
    self.dataplane_links = set(network_links)
    self.hosts = hosts
    self.interface2host = {}
    for host in hosts:
      for interface in host.interfaces:
        self.interface2host[interface] = host
    self.access_links = set(access_links)
    self.type_check_dataplane_trace()
    self.loop(steps)

  def loop(self, steps=None):
    self.running = True
    end_time = self.logical_time + steps if steps else sys.maxint
    while self.running and self.logical_time < end_time:
      self.logical_time += 1
      self.trigger_events()
      msg.event("Round %d completed." % self.logical_time)

      if self.interactive:
        # TODO: print out the state of the network at each timestep? Take a
        # verbose flag..
        self.invariant_check_prompt()
        self.dataplane_trace_prompt()
        answer = msg.raw_input('Continue to next round? [Yn]').strip()
        if answer != '' and answer.lower() != 'y':
          self.stop()
          break
      else: # not self.interactive
        if (self.logical_time % self.check_interval) == 0:
          # Time to run correspondence!
          # spawn a thread for running correspondence. Make sure the controller doesn't
          # think we've gone idle though: send OFP_ECHO_REQUESTS every few seconds
          # TODO: this is a HACK
          def do_correspondence():
            any_policy_violations = self.invariant_checker.check_correspondence(self.live_switches, self.live_links, self.access_links)
            if any_policy_violations:
              msg.fail("There were policy-violations!")
            else:
              msg.interactive("No policy-violations!")
          thread = threading.Thread(target=do_correspondence)
          thread.start()
          while thread.isAlive():
            for switch in self.live_switches:
              # connection -> deferred io worker -> io worker
              switch.send(of.ofp_echo_request().pack())
            thread.join(2.0)

        if self.dataplane_trace and (self.logical_time % self.trace_interval) == 0:
          self.inject_trace_event()

        time.sleep(self.delay)

  def stop(self):
    self.running = False

  def invariant_check_prompt(self):
    answer = msg.raw_input('Check Invariants? [Ny]')
    if answer != '' and answer.lower() != 'n':
      msg.interactive("Which one?")
      msg.interactive("  'l' - loops")
      msg.interactive("  'b' - blackholes")
      msg.interactive("  'r' - routing consistency")
      msg.interactive("  'c' - connectivity")
      msg.interactive("  'o' - omega")
      answer = msg.raw_input("  ")
      result = None
      if answer.lower() == 'l':
        result = self.invariant_checker.check_loops()
      elif answer.lower() == 'b':
        result = self.invariant_checker.check_blackholes()
      elif answer.lower() == 'r':
        result = self.invariant_checker.check_routing_consistency()
      elif answer.lower() == 'c':
        result = self.invariant_checker.check_connectivity()
      elif answer.lower() == 'o':
        result = self.invariant_checker.check_correspondence(self.live_switches,
                                                             self.live_links,
                                                             self.access_links)
      else:
        log.warn("Unknown input...")

      if result is None:
        return
      else:
        msg.interactive("Result: %s" % str(result))

  def dataplane_trace_prompt(self):
    if self.dataplane_trace:
      while True:
        answer = msg.raw_input('Feed in next dataplane event? [Ny]')
        if answer != '' and answer.lower() != 'n':
          self.inject_trace_event()
        else:
          break

  # ============================================ #
  #     Bookkeeping methods                      #
  # ============================================ #
  @property
  def live_switches(self):
    """ Return the switch_impls which are currently up """
    return self.switch_impls - self.failed_switches

  @property
  def live_links(self):
    return self.dataplane_links - self.cut_links

  # ============================================ #
  #      Methods to trigger events               #
  # ============================================ #
  def trigger_events(self):
    self.check_dataplane()
    self.check_controlplane()
    self.check_switch_crashes()
    self.check_timeouts()
    self.fuzz_traffic()

  def check_dataplane(self):
    ''' Decide whether to delay, drop, or deliver packets '''
    for dp_event in set(self.panel.get_buffered_dp_events()):
      if self.random.random() < self.dataplane_delay_rate:
        msg.event("Delaying dataplane event")
        # (Monkey patch on a delay counter)
        if not hasattr(dp_event, "delayed_rounds"):
          dp_event.delayed_rounds = 0
        dp_event.delayed_rounds += 1
      elif self.random.random() < self.dataplane_drop_rate:
        msg.event("Dropping dataplane event")
        # Drop the message
        self.panel.drop_dp_event(dp_event)
        self.dropped_dp_events.append(dp_event)
      else:
        (next_hop, next_port) = self.panel.get_connected_port(dp_event.switch, dp_event.port)
        if type(dp_event.node) == Host or type(next_hop) == Host:
          # TODO: model access link failures:
          self.panel.permit_dp_event(dp_event)
        else:
          link = Link(dp_event.switch, dp_event.port, next_hop, next_port)
          if not link in self.cut_links:
            msg.event("Forwarding dataplane event")
            # Forward the message
            self.panel.permit_dp_event(dp_event)

  def check_controlplane(self):
    ''' Decide whether to delay or deliver packets '''
    def check_deliver(switch_impl, type, give_permission):
      if self.random.random() < self.controlplane_delay_rate:
        log.debug("Delaying control plane %s for %s" % (type, str(switch_impl)))
      else:
        log.debug("Giving permission for control plane %s for %s" % (type, str(switch_impl)))
        give_permission()

    for switch_impl in self.live_switches:
      # Check reads
      # TODO: shouldn't be sticking our hands into switch_impl._connection
      for c in switch_impl.connections:
        if c.io_worker.has_pending_receives():
          check_deliver(switch_impl, "receive", c.io_worker.permit_receive)

      # Check writes
      for c in switch_impl.connections:
        if c.io_worker.has_pending_sends():
          check_deliver(switch_impl, "send", c.io_worker.permit_send)

  def check_switch_crashes(self):
    ''' Decide whether to crash or restart switches, links and controllers '''
    def crash_switches():
      crashed_this_round = set()
      for switch_impl in self.live_switches:
        if self.random.random() < self.switch_failure_rate:
          msg.event("Crashing switch_impl %s" % str(switch_impl))
          switch_impl.fail()
          crashed_this_round.add(switch_impl)
          self.failed_switches.add(switch_impl)
      return crashed_this_round

    def restart_switches(crashed_this_round):
      for switch_impl in set(self.failed_switches):
        if switch_impl in crashed_this_round:
          continue
        if self.random.random() < self.switch_recovery_rate:
          msg.event("Rebooting switch_impl %s" % str(switch_impl))
          switch_impl.recover()
          self.failed_switches.remove(switch_impl)

    def sever_links():
      # TODO: model administratively down links? (OFPPC_PORT_DOWN)
      cut_this_round = set()
      for link in self.live_links:
        if self.random.random() < self.link_failure_rate:
          msg.event("Cutting link %s" % str(link))
          self.cut_links.add(link)
          link.start_switch_impl.take_port_down(link.start_port)
          cut_this_round.add(link)
      return cut_this_round

    def repair_links(cut_this_round):
      for link in set(self.cut_links):
        if link in cut_this_round:
          continue
        if self.random.random() < self.link_recovery_rate:
          msg.event("Restoring link %s" % str(link))
          link.start_switch_impl.bring_port_up(link.start_port)
          self.cut_links.remove(link)

    crashed_this_round = crash_switches()
    restart_switches(crashed_this_round)
    cut_this_round = sever_links()
    repair_links(cut_this_round)

  def check_timeouts(self):
    # Interpose on timeouts
    pass

  def inject_trace_event(self):
    ''' Precondition: --trace is set '''
    if len(self.dataplane_trace) == 0:
      log.warn("No more trace inputs to inject!")
      return
    else:
      log.info("Injecting trace input")
      dp_event = self.dataplane_trace.pop(0)
      host = self.interface2host[dp_event.interface]
      if not host:
        log.warn("Host %s not present" % str(host))
        return
      host.send(dp_event.interface, dp_event.packet)

  def fuzz_traffic(self):
    if not self.dataplane_trace:
      # randomly generate messages from switches
      for switch_impl in self.live_switches:
        if self.random.random() < self.traffic_generation_rate:
          if len(switch_impl.ports) > 0:
            msg.event("injecting a random packet")
            traffic_type = "icmp_ping"
            # Generates a packet, and feeds it to the switch_impl
            self.traffic_generator.generate(traffic_type, switch_impl)
<commit_msg>adding better prompt for interactive mode<commit_after>#!/usr/bin/env python
# Nom nom nom nom

# TODO: future feature: colored cli prompts to make packets vs. crashes
#       vs. whatever easy to distinguish

# TODO: rather than just prompting "Continue to next round? [Yn]", allow
#       the user to examine the state of the network interactively (i.e.,
#       provide them with the normal POX cli + the simulated events

import pox.openflow.libopenflow_01 as of
from pox.lib.revent import EventMixin
from debugger_entities import Link, Host

from traffic_generator import TrafficGenerator
from invariant_checker import InvariantChecker

import sys
import threading
import signal
import subprocess
import socket
import time
import random
import os
import logging
import pickle

from sts.console import msg

log = logging.getLogger("debugger")

class FuzzTester (EventMixin):
  """
  This is part of a testing framework for controller applications. It
  acts as a replacement for pox.topology.

  Given a set of event handlers (registered by a controller application),
  it will inject intelligently chosen mock events (and observe
  their responses?)
  """
  def __init__(self, fuzzer_params="fuzzer_params.cfg", interactive=True,
               check_interval=35, trace_interval=10, random_seed=0.0,
               delay=0.1, dataplane_trace=None, control_socket=None):
    self.interactive = interactive
    self.check_interval = check_interval
    self.trace_interval = trace_interval
    # Format of trace file is a pickled array of DataplaneEvent objects
    self.dataplane_trace = None
    if dataplane_trace:
      self.dataplane_trace = pickle.load(file(dataplane_trace))
    self.running = False
    self.panel = None
    self.switch_impls = []

    self.delay = delay

    self._load_fuzzer_params(fuzzer_params)

    # Logical time (round #) for the simulation execution
    self.logical_time = 0

    # Metatdata for simulated failures
    # sts.debugger_entities.Link objects
    self.cut_links = set()
    # SwitchOutDpEvent objects
    self.dropped_dp_events = []
    # SwitchImpl objects
    self.failed_switches = set()
    # topology.Controller objects
    self.failed_controllers = set()
    # ?
    self.cancelled_timeouts = set()

    # Statistics to print on exit
    self.packets_sent = 0

    # Make execution deterministic to allow the user to easily replay
    self.seed = random_seed
    self.random = random.Random(self.seed)
    self.traffic_generator = TrafficGenerator(self.random)
    self.invariant_checker = InvariantChecker(control_socket)

    # TODO: future feature: log all events, and allow user to (interactively)
    # replay the execution
    # self.replay_logger = ReplayLogger()
    # Write the seed out to the replay log as the first 4 bytes

    # TODO: future feature: allow the user to interactively choose the order
    # events occur for each round, whether to delay, drop packets, fail nodes,
    # etc.
    # self.failure_lvl = [
    #   NOTHING,    # Everything is handled by the random number generator
    #   CRASH,      # The user only controls node crashes and restarts
    #   DROP,       # The user also controls message dropping
    #   DELAY,      # The user also controls message delays
    #   EVERYTHING  # The user controls everything, including message ordering
    # ]

    # TODO: need a mechanism for signaling  when the distributed controller handshake has completed

  def _load_fuzzer_params(self, fuzzer_params):
    if os.path.exists(fuzzer_params):
      # TODO: more pythonic way to read lines (currently on a plane...)
      # TODO: even better: there is probably a library for parsing config files
      for line in file(fuzzer_params).read().splitlines():
        if line == "[fuzzer]":
          # TODO: handle more directives other than [fuzzer]
          continue
        (kw, eq, val) = line.split()
        val = float(val)
        setattr(self, kw, val)
    else:
      # TODO: default values in case fuzzer_config is not present / missing directives
      raise IOError("Could not find logging config file: %s" % fuzzer_params)

  def type_check_dataplane_trace(self):
    if self.dataplane_trace is not None:
      for dp_event in self.dataplane_trace:
        if dp_event.interface not in self.interface2host:
          raise RuntimeError("Dataplane trace does not type check (%s)" % str(dp_event.interface))

  def simulate(self, panel, switch_impls, network_links, hosts, access_links, steps=None):
    """
    Start the fuzzer loop!
    """
    log.debug("Starting fuzz loop")
    self.panel = panel
    self.switch_impls = set(switch_impls)
    self.dataplane_links = set(network_links)
    self.hosts = hosts
    self.interface2host = {}
    for host in hosts:
      for interface in host.interfaces:
        self.interface2host[interface] = host
    self.access_links = set(access_links)
    self.type_check_dataplane_trace()
    self.loop(steps)

  def loop(self, steps=None):
    self.running = True
    end_time = self.logical_time + steps if steps else sys.maxint
    while self.running and self.logical_time < end_time:
      self.logical_time += 1
      self.trigger_events()
      msg.event("Round %d completed." % self.logical_time)

      if self.interactive:
        # TODO: print out the state of the network at each timestep? Take a
        # verbose flag..
        self.invariant_check_prompt()
        self.dataplane_trace_prompt()
        answer = msg.raw_input('Continue to next round? [Yn]').strip()
        if answer != '' and answer.lower() != 'y':
          self.stop()
          break
      else: # not self.interactive
        if (self.logical_time % self.check_interval) == 0:
          # Time to run correspondence!
          # spawn a thread for running correspondence. Make sure the controller doesn't
          # think we've gone idle though: send OFP_ECHO_REQUESTS every few seconds
          # TODO: this is a HACK
          def do_correspondence():
            any_policy_violations = self.invariant_checker.check_correspondence(self.live_switches, self.live_links, self.access_links)
            if any_policy_violations:
              msg.fail("There were policy-violations!")
            else:
              msg.interactive("No policy-violations!")
          thread = threading.Thread(target=do_correspondence)
          thread.start()
          while thread.isAlive():
            for switch in self.live_switches:
              # connection -> deferred io worker -> io worker
              switch.send(of.ofp_echo_request().pack())
            thread.join(2.0)

        if self.dataplane_trace and (self.logical_time % self.trace_interval) == 0:
          self.inject_trace_event()

        time.sleep(self.delay)

  def stop(self):
    self.running = False

  def invariant_check_prompt(self):
    answer = msg.raw_input('Check Invariants? [Ny]')
    if answer != '' and answer.lower() != 'n':
      msg.interactive("Which one?")
      msg.interactive("  'l' - loops")
      msg.interactive("  'b' - blackholes")
      msg.interactive("  'r' - routing consistency")
      msg.interactive("  'c' - connectivity")
      msg.interactive("  'o' - omega")
      answer = msg.raw_input("> ")
      result = None
      if answer.lower() == 'l':
        result = self.invariant_checker.check_loops()
      elif answer.lower() == 'b':
        result = self.invariant_checker.check_blackholes()
      elif answer.lower() == 'r':
        result = self.invariant_checker.check_routing_consistency()
      elif answer.lower() == 'c':
        result = self.invariant_checker.check_connectivity()
      elif answer.lower() == 'o':
        result = self.invariant_checker.check_correspondence(self.live_switches,
                                                             self.live_links,
                                                             self.access_links)
      else:
        log.warn("Unknown input...")

      if result is None:
        return
      else:
        msg.interactive("Result: %s" % str(result))

  def dataplane_trace_prompt(self):
    if self.dataplane_trace:
      while True:
        answer = msg.raw_input('Feed in next dataplane event? [Ny]')
        if answer != '' and answer.lower() != 'n':
          self.inject_trace_event()
        else:
          break

  # ============================================ #
  #     Bookkeeping methods                      #
  # ============================================ #
  @property
  def live_switches(self):
    """ Return the switch_impls which are currently up """
    return self.switch_impls - self.failed_switches

  @property
  def live_links(self):
    return self.dataplane_links - self.cut_links

  # ============================================ #
  #      Methods to trigger events               #
  # ============================================ #
  def trigger_events(self):
    self.check_dataplane()
    self.check_controlplane()
    self.check_switch_crashes()
    self.check_timeouts()
    self.fuzz_traffic()

  def check_dataplane(self):
    ''' Decide whether to delay, drop, or deliver packets '''
    for dp_event in set(self.panel.get_buffered_dp_events()):
      if self.random.random() < self.dataplane_delay_rate:
        msg.event("Delaying dataplane event")
        # (Monkey patch on a delay counter)
        if not hasattr(dp_event, "delayed_rounds"):
          dp_event.delayed_rounds = 0
        dp_event.delayed_rounds += 1
      elif self.random.random() < self.dataplane_drop_rate:
        msg.event("Dropping dataplane event")
        # Drop the message
        self.panel.drop_dp_event(dp_event)
        self.dropped_dp_events.append(dp_event)
      else:
        (next_hop, next_port) = self.panel.get_connected_port(dp_event.switch, dp_event.port)
        if type(dp_event.node) == Host or type(next_hop) == Host:
          # TODO: model access link failures:
          self.panel.permit_dp_event(dp_event)
        else:
          link = Link(dp_event.switch, dp_event.port, next_hop, next_port)
          if not link in self.cut_links:
            msg.event("Forwarding dataplane event")
            # Forward the message
            self.panel.permit_dp_event(dp_event)

  def check_controlplane(self):
    ''' Decide whether to delay or deliver packets '''
    def check_deliver(switch_impl, type, give_permission):
      if self.random.random() < self.controlplane_delay_rate:
        log.debug("Delaying control plane %s for %s" % (type, str(switch_impl)))
      else:
        log.debug("Giving permission for control plane %s for %s" % (type, str(switch_impl)))
        give_permission()

    for switch_impl in self.live_switches:
      # Check reads
      # TODO: shouldn't be sticking our hands into switch_impl._connection
      for c in switch_impl.connections:
        if c.io_worker.has_pending_receives():
          check_deliver(switch_impl, "receive", c.io_worker.permit_receive)

      # Check writes
      for c in switch_impl.connections:
        if c.io_worker.has_pending_sends():
          check_deliver(switch_impl, "send", c.io_worker.permit_send)

  def check_switch_crashes(self):
    ''' Decide whether to crash or restart switches, links and controllers '''
    def crash_switches():
      crashed_this_round = set()
      for switch_impl in self.live_switches:
        if self.random.random() < self.switch_failure_rate:
          msg.event("Crashing switch_impl %s" % str(switch_impl))
          switch_impl.fail()
          crashed_this_round.add(switch_impl)
          self.failed_switches.add(switch_impl)
      return crashed_this_round

    def restart_switches(crashed_this_round):
      for switch_impl in set(self.failed_switches):
        if switch_impl in crashed_this_round:
          continue
        if self.random.random() < self.switch_recovery_rate:
          msg.event("Rebooting switch_impl %s" % str(switch_impl))
          switch_impl.recover()
          self.failed_switches.remove(switch_impl)

    def sever_links():
      # TODO: model administratively down links? (OFPPC_PORT_DOWN)
      cut_this_round = set()
      for link in self.live_links:
        if self.random.random() < self.link_failure_rate:
          msg.event("Cutting link %s" % str(link))
          self.cut_links.add(link)
          link.start_switch_impl.take_port_down(link.start_port)
          cut_this_round.add(link)
      return cut_this_round

    def repair_links(cut_this_round):
      for link in set(self.cut_links):
        if link in cut_this_round:
          continue
        if self.random.random() < self.link_recovery_rate:
          msg.event("Restoring link %s" % str(link))
          link.start_switch_impl.bring_port_up(link.start_port)
          self.cut_links.remove(link)

    crashed_this_round = crash_switches()
    restart_switches(crashed_this_round)
    cut_this_round = sever_links()
    repair_links(cut_this_round)

  def check_timeouts(self):
    # Interpose on timeouts
    pass

  def inject_trace_event(self):
    ''' Precondition: --trace is set '''
    if len(self.dataplane_trace) == 0:
      log.warn("No more trace inputs to inject!")
      return
    else:
      log.info("Injecting trace input")
      dp_event = self.dataplane_trace.pop(0)
      host = self.interface2host[dp_event.interface]
      if not host:
        log.warn("Host %s not present" % str(host))
        return
      host.send(dp_event.interface, dp_event.packet)

  def fuzz_traffic(self):
    if not self.dataplane_trace:
      # randomly generate messages from switches
      for switch_impl in self.live_switches:
        if self.random.random() < self.traffic_generation_rate:
          if len(switch_impl.ports) > 0:
            msg.event("injecting a random packet")
            traffic_type = "icmp_ping"
            # Generates a packet, and feeds it to the switch_impl
            self.traffic_generator.generate(traffic_type, switch_impl)
<|endoftext|>


<commit_before>from django.http import HttpResponse
import urllib2
import simplejson

def index(request):

    html_str = 'Winter sucks.'
    return HttpResponse(html_str)
<commit_msg>mac vm updates<commit_after>from django.http import HttpResponse

def index(request):

    html_str = 'Winter sucks.'
    return HttpResponse(html_str)
<|endoftext|>


<commit_before># The version of Review Board.
#
# This is in the format of:
#
#   (Major, Minor, Micro, Patch, alpha/beta/rc/final, Release Number, Released)
#
VERSION = (1, 5, 0, 0, 'final', 0, True)

def get_version_string():
    version = '%s.%s' % (VERSION[0], VERSION[1])

    if VERSION[2]:
        version += ".%s" % VERSION[2]

    if VERSION[3]:
        version += ".%s" % VERSION[3]

    if VERSION[4] != 'final':
        if VERSION[4] == 'rc':
            version += ' RC%s' % VERSION[5]
        else:
            version += ' %s %s' % (VERSION[4], VERSION[5])

    if not is_release():
        version += " (dev)"

    return version

def get_package_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])

    if VERSION[2]:
        version += ".%s" % VERSION[2]

    if VERSION[3]:
        version += ".%s" % VERSION[3]

    if VERSION[4] != 'final':
        version += '%s%s' % (VERSION[4], VERSION[5])

    return version

def is_release():
    return VERSION[6]

def initialize():
    """Begins initialization of Review Board.

    This sets up the logging, generates cache serial numbers, and then
    fires an initializing signal that other parts of the codebase can
    connect to. This must be called for such features as e-mail notification
    to work.
    """
    import logging
    import os

    from djblets.util.misc import generate_cache_serials
    from djblets import log

    from reviewboard import signals

    # Set up logging.
    log.init_logging()
    logging.info("Log file for Review Board v%s (PID %s)" %
                 (get_version_string(), os.getpid()))

    # Generate cache serials
    generate_cache_serials()

    signals.initializing.send(sender=None)

__version_info__ = VERSION[:-1]
__version__ = get_package_version()
<commit_msg>Bump to Review Board 1.6 dev.<commit_after># The version of Review Board.
#
# This is in the format of:
#
#   (Major, Minor, Micro, Patch, alpha/beta/rc/final, Release Number, Released)
#
VERSION = (1, 6, 0, 0, 'alpha', 0, False)

def get_version_string():
    version = '%s.%s' % (VERSION[0], VERSION[1])

    if VERSION[2]:
        version += ".%s" % VERSION[2]

    if VERSION[3]:
        version += ".%s" % VERSION[3]

    if VERSION[4] != 'final':
        if VERSION[4] == 'rc':
            version += ' RC%s' % VERSION[5]
        else:
            version += ' %s %s' % (VERSION[4], VERSION[5])

    if not is_release():
        version += " (dev)"

    return version

def get_package_version():
    version = '%s.%s' % (VERSION[0], VERSION[1])

    if VERSION[2]:
        version += ".%s" % VERSION[2]

    if VERSION[3]:
        version += ".%s" % VERSION[3]

    if VERSION[4] != 'final':
        version += '%s%s' % (VERSION[4], VERSION[5])

    return version

def is_release():
    return VERSION[6]

def initialize():
    """Begins initialization of Review Board.

    This sets up the logging, generates cache serial numbers, and then
    fires an initializing signal that other parts of the codebase can
    connect to. This must be called for such features as e-mail notification
    to work.
    """
    import logging
    import os

    from djblets.util.misc import generate_cache_serials
    from djblets import log

    from reviewboard import signals

    # Set up logging.
    log.init_logging()
    logging.info("Log file for Review Board v%s (PID %s)" %
                 (get_version_string(), os.getpid()))

    # Generate cache serials
    generate_cache_serials()

    signals.initializing.send(sender=None)

__version_info__ = VERSION[:-1]
__version__ = get_package_version()
<|endoftext|>


<commit_before># Rewind is an event store server written in Python that talks ZeroMQ.
# Copyright (C) 2012  Jens Rantil
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Configuration file utilities."""
import importlib
import logging

import rewind.server.eventstores as eventstores

_logger = logging.getLogger(__name__)

class ConfigurationError(Exception):

    """An error thrown when configuration of the application fails."""

    def __init__(self, what):
        self.what = what

    def __str__(self):
        return repr(self.what)

def construct_eventstore(config, args, section=None):
    """Construct the event store to write and write from/to.

    The event store is constructed from an optionally given configuration file
    and/or command line arguments. Command line arguments has higher
    presendence over configuration file attributes.

    If a defined event store is missing, `InMemoryEventStore` will be used and
    a warning will be logged to stderr.

    This function is considered public API since external event stores might
    use it to load other (usually, underlying) event stores.

    Arguments:
    config  -- a configuration dictionary. Derived from
                     `configparser.RawConfigParser`.
    args    -- the arguments given at command line to Rewind.
    section -- the config section to use for event store instantiation.

    returns -- a new event store.

    """
    DEFAULT_SECTION = 'general'
    ESTORE_CLASS_ATTRIBUTE = 'class'

    if config is None:
        _logger.warn("Using InMemoryEventStore. Events are not persisted."
                     " See example config file on how to persist them.")
        eventstore = eventstores.InMemoryEventStore()
        return eventstore

    if section is None:
        if DEFAULT_SECTION not in config.sections():
            raise ConfigurationError("Missing default section, `general`.")
        section = config.get(DEFAULT_SECTION, 'storage-backend')

    if section not in config.sections():
        msg = "The section for event store does not exist: {0}"
        raise ConfigurationError(msg.format(section))

    if not config.has_option(section, ESTORE_CLASS_ATTRIBUTE):
        errmsg = 'Configuration option `class` missing for section `{0}`.'
        raise ConfigurationError(errmsg.format(section))

    classpath = config.get(section, ESTORE_CLASS_ATTRIBUTE)
    classpath_pieces = classpath.split('.')
    classname = classpath_pieces[-1]
    modulepath = '.'.join(classpath_pieces[0:-1])

    module = importlib.import_module(modulepath)

    # Instantiating the event store in question using custom arguments
    options = config.options(section)
    if ESTORE_CLASS_ATTRIBUTE in options:
        # Unnecessary argument
        i = options.index(ESTORE_CLASS_ATTRIBUTE)
        del options[i]
    Class = getattr(module, classname)
    customargs = {option: config.get(section, option) for option in options}
    try:
        eventstore = Class.from_config(config, args, **customargs)
    except ConfigurationError as e:
        msg = "Could not instantiate `{0}`: {1}"
        raise ConfigurationError(msg.format(Class, e.what))

    return eventstore
<commit_msg>Bugfix: Remove circular imports<commit_after># Rewind is an event store server written in Python that talks ZeroMQ.
# Copyright (C) 2012  Jens Rantil
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Configuration file utilities."""
import importlib
import logging

_logger = logging.getLogger(__name__)

class ConfigurationError(Exception):

    """An error thrown when configuration of the application fails."""

    def __init__(self, what):
        self.what = what

    def __str__(self):
        return repr(self.what)

def construct_eventstore(config, args, section=None):
    """Construct the event store to write and write from/to.

    The event store is constructed from an optionally given configuration file
    and/or command line arguments. Command line arguments has higher
    presendence over configuration file attributes.

    If a defined event store is missing, `InMemoryEventStore` will be used and
    a warning will be logged to stderr.

    This function is considered public API since external event stores might
    use it to load other (usually, underlying) event stores.

    Arguments:
    config  -- a configuration dictionary. Derived from
                     `configparser.RawConfigParser`.
    args    -- the arguments given at command line to Rewind.
    section -- the config section to use for event store instantiation.

    returns -- a new event store.

    """
    DEFAULT_SECTION = 'general'
    ESTORE_CLASS_ATTRIBUTE = 'class'

    if config is None:
        _logger.warn("Using InMemoryEventStore. Events are not persisted."
                     " See example config file on how to persist them.")
        # XXX: This is to evade circular dependency between config and
        #      eventstores.
        import rewind.server.eventstores as eventstores
        eventstore = eventstores.InMemoryEventStore()
        return eventstore

    if section is None:
        if DEFAULT_SECTION not in config.sections():
            raise ConfigurationError("Missing default section, `general`.")
        section = config.get(DEFAULT_SECTION, 'storage-backend')

    if section not in config.sections():
        msg = "The section for event store does not exist: {0}"
        raise ConfigurationError(msg.format(section))

    if not config.has_option(section, ESTORE_CLASS_ATTRIBUTE):
        errmsg = 'Configuration option `class` missing for section `{0}`.'
        raise ConfigurationError(errmsg.format(section))

    classpath = config.get(section, ESTORE_CLASS_ATTRIBUTE)
    classpath_pieces = classpath.split('.')
    classname = classpath_pieces[-1]
    modulepath = '.'.join(classpath_pieces[0:-1])

    module = importlib.import_module(modulepath)

    # Instantiating the event store in question using custom arguments
    options = config.options(section)
    if ESTORE_CLASS_ATTRIBUTE in options:
        # Unnecessary argument
        i = options.index(ESTORE_CLASS_ATTRIBUTE)
        del options[i]
    Class = getattr(module, classname)
    customargs = {option: config.get(section, option) for option in options}
    try:
        eventstore = Class.from_config(config, args, **customargs)
    except ConfigurationError as e:
        msg = "Could not instantiate `{0}`: {1}"
        raise ConfigurationError(msg.format(Class, e.what))

    return eventstore
<|endoftext|>


<commit_before>from django.db import models
from django.core.exceptions import ValidationError

from core.network.models import Network
from core.mixins import ObjectUrlMixin
from core.keyvalue.base_option import CommonOption
from core.interface.static_intr.models import StaticInterface
from mozdns.ip.models import ipv6_to_longs
from mozdns.address_record.models import AddressRecord
from mozdns.ptr.models import PTR

import ipaddr

import pdb

class Range(models.Model, ObjectUrlMixin):
    """The Range class.

        >>> Range(start=start_ip, end=end_ip,
        >>>         defualt_domain=domain, network=network)

    Ranges live inside networks; their start ip address is greater than or
    equal to the the start of their network and their end ip address is less
    than or equal to the end of their network; both the Range and the network
    class enforce these requirements. Good practice says ranges should not
    start on the network address of their network and they should not end on
    the broadcast address of their network; the Range and Network classes do
    not enforce this.

    Chaning a Range

    Things that happen when a static range is changed:

        *   The new `start` and `end` values are checked against the range's
            network to ensure that the range still exists within the network.
        *   The new `start` and `end` values are checked against all other
            existing range's `start` and `end` values to make sure that the
            new range does not overlap.
    """
    id = models.AutoField(primary_key=True)

    start_upper = models.BigIntegerField(null=True)
    start_lower = models.BigIntegerField(null=True)
    start_str = models.CharField(max_length=39, editable=True)

    end_lower = models.BigIntegerField(null=True)
    end_upper = models.BigIntegerField(null=True)
    end_str = models.CharField(max_length=39, editable=True)

    dhcpd_raw_include = models.TextField(null=True, blank=True)

    network = models.ForeignKey(Network, null=False)

    class Meta:
        db_table = 'range'
        unique_together = ('start_upper', 'start_lower', 'end_upper', 'end_lower')

    def save(self, *args, **kwargs):
        self.clean()
        super(Range, self).save(*args, **kwargs)

    def clean(self):
        if not self.network:
            raise ValidationError("ERROR: No network found")
        try:
            if self.network.ip_type == '4':
                self.start_upper, self.start_lower = 0 ,int(
                        ipaddr.IPv4Address(self.start_str))
                self.end_upper, self.end_lower = 0 ,int(
                        ipaddr.IPv4Address(self.end_str))
            elif self.network.ip_type == '6':
                self.start_upper, self.start_lower = ipv6_to_longs(
                        self.start_str)
                self.end_upper, self.end_lower = ipv6_to_longs(self.end_str)
            else:
                raise ValidationError("ERROR: could not determine the ip type")
        except ipaddr.AddressValueError, e:
            raise ValidationError(str(e))

        """
        Some notes:
        start = s1 s2
        end = e1 e2

        if s1 > e1:
            start > end
            # Bad
        if e1 > s1:
            end > start
            # Good
        if s1 == e1 and s2 > e2:
            start > end
            # Bad
        if s1 == e1 and s2 < e2:
            end > start
            # Good
        if s1 == e1 and s2 == e2:
            end == start
            # Bad
        """
        fail = False
        if self.start_upper > self.end_upper:
            # start > end
            fail = True
        if (self.start_upper == self.end_upper and self.start_lower >
            self.end_lower):
            # start > end
            fail = True
        if (self.start_upper == self.end_upper and self.start_lower ==
            self.end_lower):
            # end == start
            fail = True

        if fail:
            raise ValidationError("The start of a range cannot be greater than"
                    " or equal to the end of the range.")

        self.network.update_network()
        if self.network.ip_type == '4':
            IPClass = ipaddr.IPv4Address
        else:
            IPClass = ipaddr.IPv6Address

        if IPClass(self.start_str) < self.network.network.network:
            #lol, network.network.network.network.network....
            raise ValidationError("The start of a range cannot be less than "
                "it's network's network address.")
        if IPClass(self.end_str) > self.network.network.broadcast:
            raise ValidationError("The end of a range cannot be more than "
                "it's network's broadcast address.")

        self.check_for_overlaps()

    def check_for_overlaps(self):
        """This function will look at all the other ranges and make sure we
        don't overlap with any of them.
        """
        for range_ in self.network.range_set.all():
            if range_.pk == self.pk:
                continue

            # start > end
            if self.start_upper > range_.end_upper:
                continue
            if (self.start_upper == range_.end_upper and self.start_lower >
                range_.end_lower):
                continue
            # end < start
            if self.end_upper < range_.start_upper:
                continue
            if (self.end_upper == range_.start_upper and self.end_lower <
                range_.start_lower):
                continue
            raise ValidationError("Ranges cannot exist inside of other "
                "ranges.")

    def __str__(self):
        x = "Site: {0} Vlan: {1} Network: {2} Range: Start - {3} End -  {4}"
        return x.format(self.network.site, self.network.vlan, self.network,
            self.start_str, self.end_str)

    def display(self):
        return "Range: {3} to {4}  {0} -- {2} -- {1}  ".format(
                self.network.site, self.network.vlan, self.network,
                self.start_str, self.end_str)

    def choice_display(self):
        return "{0} - {1} - ({2}) {3} to {4}".format(
                str(self.network.site).upper(), self.network.vlan, self.network,
                self.start_str, self.end_str)

    def __repr__(self):
        return "<Range: {0}>".format(str(self))

    def get_next_ip(self):
        """Find's the most appropriate ip address within a range. If it can't
        find an IP it returns None. If it finds an IP it returns an IPv4Address
        object.

            :returns: ipaddr.IPv4Address
        """
        if self.network.ip_type != '4':
            return None
        start = self.start_lower
        end = self.end_lower
        if start >= end - 1:
            return HttpResponse("Too small of a range.")
        ip = find_free_ip(start, end, ip_type = '4')
        if ip:
            return ip
        else:
            return None

def find_free_ip(start, end, ip_type='4'):
    """Given start and end numbers, find a free ip.
    :param start: The start number
    :type start: int
    :param end: The end number
    :type end: int
    :param ip_type: The type of IP you are looking for.
    :type ip_type: str either '4' or '6'
    """
    if ip_type == '4':
        records = AddressRecord.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        ptrs = PTR.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        intrs = StaticInterface.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        if not records and not intrs:
            ip = ipaddr.IPv4Address(start)
            return ip
        for i in xrange(start, end + 1):
            taken = False
            for record in records:
                if record.ip_lower == i:
                    taken = True
                    break
            for ptr in ptrs:
                if ptr.ip_lower == i:
                    taken = True
                    break
            if taken == False:
                for intr in intrs:
                    if intr.ip_lower == i:
                        taken = True
                        break
            if taken == False:
                ip = ipaddr.IPv4Address(i)
                return ip
    else:
        raise NotImplemented

class RangeKeyValue(CommonOption):
    range = models.ForeignKey(Range, null=False)

    class Meta:
        db_table = 'range_key_value'
        unique_together = ('key', 'value', 'range')

    def _aa_failover(self):
        self.is_statement = True
        self.is_option = False
        if self.value != "peer \"dhcp-failover\"":
            raise ValidationError("Invalid failover option. Try `peer "
                "\"dhcp-failover\"`")

    def _aa_routers(self):
        self._routers(self.range.network.ip_type)

    def _aa_ntp_servers(self):
        self._ntp_servers(self.range.network.ip_type)
<commit_msg>added range type<commit_after>from django.db import models
from django.core.exceptions import ValidationError

from core.network.models import Network
from core.mixins import ObjectUrlMixin
from core.keyvalue.base_option import CommonOption
from core.interface.static_intr.models import StaticInterface
from mozdns.ip.models import ipv6_to_longs
from mozdns.address_record.models import AddressRecord
from mozdns.ptr.models import PTR

import ipaddr

import pdb

class Range(models.Model, ObjectUrlMixin):
    """The Range class.

        >>> Range(start=start_ip, end=end_ip,
        >>>         defualt_domain=domain, network=network)

    Ranges live inside networks; their start ip address is greater than or
    equal to the the start of their network and their end ip address is less
    than or equal to the end of their network; both the Range and the network
    class enforce these requirements. Good practice says ranges should not
    start on the network address of their network and they should not end on
    the broadcast address of their network; the Range and Network classes do
    not enforce this.

    Chaning a Range

    Things that happen when a static range is changed:

        *   The new `start` and `end` values are checked against the range's
            network to ensure that the range still exists within the network.
        *   The new `start` and `end` values are checked against all other
            existing range's `start` and `end` values to make sure that the new
            range does not overlap.
    """
    id = models.AutoField(primary_key=True)

    start_upper = models.BigIntegerField(null=True)
    start_lower = models.BigIntegerField(null=True)
    start_str = models.CharField(max_length=39, editable=True)

    end_lower = models.BigIntegerField(null=True)
    end_upper = models.BigIntegerField(null=True)
    end_str = models.CharField(max_length=39, editable=True)

    dhcpd_raw_include = models.TextField(null=True, blank=True)

    network = models.ForeignKey(Network, null=False)

    STATIC = "st"
    DYNAMIC = "dy"
    RANGE_TYPE = (
        (STATIC, 'Static'),
        (DYNAMIC, 'Dynamic'),
    )
    models.CharField(max_length=2, choices=RANGE_TYPE, default=STATIC,
                        editable=False)

    class Meta:
        db_table = 'range'
        unique_together = ('start_upper', 'start_lower', 'end_upper',
                'end_lower')

    def save(self, *args, **kwargs):
        self.clean()
        super(Range, self).save(*args, **kwargs)

    def clean(self):
        if not self.network:
            raise ValidationError("ERROR: No network found")
        try:
            if self.network.ip_type == '4':
                self.start_upper, self.start_lower = 0 ,int(
                        ipaddr.IPv4Address(self.start_str))
                self.end_upper, self.end_lower = 0 ,int(
                        ipaddr.IPv4Address(self.end_str))
            elif self.network.ip_type == '6':
                self.start_upper, self.start_lower = ipv6_to_longs(
                        self.start_str)
                self.end_upper, self.end_lower = ipv6_to_longs(self.end_str)
            else:
                raise ValidationError("ERROR: could not determine the ip type")
        except ipaddr.AddressValueError, e:
            raise ValidationError(str(e))

        """
        Some notes:
        start = s1 s2
        end = e1 e2

        if s1 > e1:
            start > end
            # Bad
        if e1 > s1:
            end > start
            # Good
        if s1 == e1 and s2 > e2:
            start > end
            # Bad
        if s1 == e1 and s2 < e2:
            end > start
            # Good
        if s1 == e1 and s2 == e2:
            end == start
            # Bad
        """
        fail = False
        if self.start_upper > self.end_upper:
            # start > end
            fail = True
        if (self.start_upper == self.end_upper and self.start_lower >
            self.end_lower):
            # start > end
            fail = True
        if (self.start_upper == self.end_upper and self.start_lower ==
            self.end_lower):
            # end == start
            fail = True

        if fail:
            raise ValidationError("The start of a range cannot be greater than"
                    " or equal to the end of the range.")

        self.network.update_network()
        if self.network.ip_type == '4':
            IPClass = ipaddr.IPv4Address
        else:
            IPClass = ipaddr.IPv6Address

        if IPClass(self.start_str) < self.network.network.network:
            #lol, network.network.network.network.network....
            raise ValidationError("The start of a range cannot be less than "
                "it's network's network address.")
        if IPClass(self.end_str) > self.network.network.broadcast:
            raise ValidationError("The end of a range cannot be more than "
                "it's network's broadcast address.")

        self.check_for_overlaps()

    def check_for_overlaps(self):
        """This function will look at all the other ranges and make sure we
        don't overlap with any of them.
        """
        for range_ in self.network.range_set.all():
            if range_.pk == self.pk:
                continue

            # start > end
            if self.start_upper > range_.end_upper:
                continue
            if (self.start_upper == range_.end_upper and self.start_lower >
                range_.end_lower):
                continue
            # end < start
            if self.end_upper < range_.start_upper:
                continue
            if (self.end_upper == range_.start_upper and self.end_lower <
                range_.start_lower):
                continue
            raise ValidationError("Ranges cannot exist inside of other "
                "ranges.")

    def __str__(self):
        x = "Site: {0} Vlan: {1} Network: {2} Range: Start - {3} End -  {4}"
        return x.format(self.network.site, self.network.vlan, self.network,
            self.start_str, self.end_str)

    def display(self):
        return "Range: {3} to {4}  {0} -- {2} -- {1}  ".format(
                self.network.site, self.network.vlan, self.network,
                self.start_str, self.end_str)

    def choice_display(self):
        return "{0} - {1} - ({2}) {3} to {4}".format(
                str(self.network.site).upper(), self.network.vlan,
                self.network, self.start_str, self.end_str)

    def __repr__(self):
        return "<Range: {0}>".format(str(self))

    def get_next_ip(self):
        """Find's the most appropriate ip address within a range. If it can't
        find an IP it returns None. If it finds an IP it returns an IPv4Address
        object.

            :returns: ipaddr.IPv4Address
        """
        if self.network.ip_type != '4':
            return None
        start = self.start_lower
        end = self.end_lower
        if start >= end - 1:
            return HttpResponse("Too small of a range.")
        ip = find_free_ip(start, end, ip_type = '4')
        if ip:
            return ip
        else:
            return None

def find_free_ip(start, end, ip_type='4'):
    """Given start and end numbers, find a free ip.
    :param start: The start number
    :type start: int
    :param end: The end number
    :type end: int
    :param ip_type: The type of IP you are looking for.
    :type ip_type: str either '4' or '6'
    """
    if ip_type == '4':
        records = AddressRecord.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        ptrs = PTR.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        intrs = StaticInterface.objects.filter(ip_upper=0, ip_lower__gte=start,
                ip_lower__lte=end)
        if not records and not intrs:
            ip = ipaddr.IPv4Address(start)
            return ip
        for i in xrange(start, end + 1):
            taken = False
            for record in records:
                if record.ip_lower == i:
                    taken = True
                    break
            for ptr in ptrs:
                if ptr.ip_lower == i:
                    taken = True
                    break
            if taken == False:
                for intr in intrs:
                    if intr.ip_lower == i:
                        taken = True
                        break
            if taken == False:
                ip = ipaddr.IPv4Address(i)
                return ip
    else:
        raise NotImplemented

class RangeKeyValue(CommonOption):
    range = models.ForeignKey(Range, null=False)

    class Meta:
        db_table = 'range_key_value'
        unique_together = ('key', 'value', 'range')

    def _aa_failover(self):
        self.is_statement = True
        self.is_option = False
        if self.value != "peer \"dhcp-failover\"":
            raise ValidationError("Invalid failover option. Try `peer "
                "\"dhcp-failover\"`")

    def _aa_routers(self):
        self._routers(self.range.network.ip_type)

    def _aa_ntp_servers(self):
        self._ntp_servers(self.range.network.ip_type)
<|endoftext|>


<commit_before>import numpy as np
from .utils import epsilon

# Color manipulation functions

def sigmoidal(arr, contrast, bias):
    """
    Sigmoidal contrast is type of contrast control that
    adjusts the contrast without saturating highlights or shadows.
    It allows control over two factors:
    the contrast range from light to dark, and where the middle value
    of the mid-tones falls. The result is a non-linear and smooth
    contrast change.

    Parameters
    ----------
    contrast : Float. For example, 0 is none, 3 is typical and 20 is a lot.
    bias : Float, threshold level for the contrast function to
        center on (typically centered at '50%')

    Notes
    ----------

    Sigmoidal contrast is based on the sigmoidal transfer function:

    .. math:: g(u) = ( 1/(1 + e^{- \alpha * u + \beta)})

    This sigmoid function is scaled so that the output is bound by
    the interval [0, 1].

    .. math:: ( 1/(1 + e^(\beta * (\alpha - u))) - 1/(1 + e^(\beta * \alpha)) ) /
    ( 1/(1 + e^(\beta*(\alpha - 1))) - 1/(1 + e^(\beta * \alpha)) )

    Where :math: `\alpha` is the threshold level, and :math: `\beta` the contrast factor
    to be applied.

    References
    ----------
    .. [CT] Hany Farid "Fundamentals of Image Processing"
            http://www.cs.dartmouth.edu/farid/downloads/tutorials/fip.pdf

    """
    alpha, beta = bias, contrast
    # We use the names a and b to match documentation.

    if alpha == 0:
        alpha = epsilon

    if beta == 0:
        return arr

    np.seterr(divide='ignore', invalid='ignore')

    if beta > 0:
        numerator = 1 / (1 + np.exp(beta * (alpha - arr))) - \
            1 / (1 + np.exp(beta * alpha))
        denominator = 1 / (1 + np.exp(beta * (alpha - 1))) - \
            1 / (1 + np.exp(beta * alpha))
        return numerator / denominator
    else:
        # Inverse sigmoidal function:
        # todo: account for 0s
        # todo: formatting ;)
        return (
            (beta * alpha) - np.log(
                (
                    1 / (
                        (arr / (1 + np.exp(beta * alpha - beta)))
                        - (arr / (1 + np.exp(beta * alpha)))
                        + (1 / (1 + np.exp(beta * alpha)))
                    )
                )
                - 1)
        ) / beta

def gamma(arr, g):
    """
    Gamma correction is a nonlinear operation that
    adjusts the image's channel values pixel-by-pixel according
    to a power-law:

    .. math:: pixel_{out} = pixel_{in} ^ {\gamma}

    Setting gamma (:math:`\gamma`) to be less than 1.0 darkens the image and setting gamma
    to be greater than 1.0 lightens it.

    Parameters
    ----------
    Gamma (:math:`\gamma`): Float, Reasonable values range from 0.8 to 2.3.

    """
    return arr**(1.0 / g)

def saturation(arr, percent):
    """
    multiply saturation by percent in LCH color space
    Color saturation is used to describe the intensity of
    color in the image. As saturation increases, colors appear
    more "pure." As saturation decreases, colors appear more "washed-out."

    """
    lch = rgb2lch(arr)
    # Adjust chroma, band at index=1
    lch[1] = lch[1] * (percent / 100.0)
    return lch2rgb(lch)

# Utility functions

def rgb2lch(rgb):
    from skimage.color import rgb2lab, lab2lch
    # reshape for skimage (bands, cols, rows) -> (cols, rows, bands)
    srgb = np.swapaxes(rgb, 0, 2)
    # convert colorspace
    lch = lab2lch(rgb2lab(srgb))
    # return in (bands, cols, rows) order
    return np.swapaxes(lch, 2, 0)

def lch2rgb(lch):
    from skimage.color import lch2lab, lab2rgb
    # reshape for skimage (bands, cols, rows) -> (cols, rows, bands)
    slch = np.swapaxes(lch, 0, 2)
    # convert colorspace
    rgb = lab2rgb(lch2lab(slch))
    # return in (bands, cols, rows) order
    return np.swapaxes(rgb, 2, 0)

def simple_atmo(rgb, haze, contrast, bias):
    '''A simple, static (non-adaptive) atmospheric correction function.'''
    # bias assumed to be given in percent,
    # convert to proportion
    bias = bias / 100.0

    gamma_b = 1 - haze
    gamma_g = 1 - (haze * (1 / 3.0))

    rgb[1] = gamma(rgb[1], gamma_g)
    rgb[2] = gamma(rgb[2], gamma_b)

    return sigmoidal(rgb, contrast, bias)

def parse_operations(operations, count=3):
    """Takes an iterable of operations,
    each operation is expected to be a string with a specified syntax

    "OPERATION-NAME BANDS ARGS..."

    And yields a list of functions that take and return ndarrays
    """
    band_lookup = {'r': 1, 'g': 2, 'b': 3}

    opfuncs = {
        'saturation': saturation,
        'sigmoidal': sigmoidal,
        'gamma': gamma}

    opkwargs = {
        'saturation': ('percent',),
        'sigmoidal': ('contrast', 'bias'),
        'gamma': ('g',)}

    # Operations that assume RGB colorspace
    rgb_ops = ('saturation',)

    for op in operations:
        parts = op.split(" ")
        opname = parts[0]
        bandstr = parts[1]
        args = parts[2:]

        try:
            func = opfuncs[opname]
        except KeyError:
            raise ValueError("{} is not a valid operation".format(opname))

        if opname in rgb_ops:
            # ignore bands, assumed to be in rgb
            # push 2nd arg into args
            args = [bandstr] + args
            bands = (1, 2, 3)
        else:
            # 2nd arg is bands
            # parse r,g,b ~= 1,2,3
            bands = set()
            for bs in bandstr.split(","):
                try:
                    band = int(bs)
                except ValueError:
                    band = band_lookup[bs.lower()]
                if band < 1 or band > count:
                    raise ValueError(
                        "{} BAND must be between 1 and {}".format(opname, count))
                bands.add(band)

        # assume all args are float
        args = [float(arg) for arg in args]
        kwargs = dict(zip(opkwargs[opname], args))

        def f(arr, func=func, kwargs=kwargs):
            if opname in rgb_ops:
                # apply func to array assuming 3 band r,g,b
                newarr = func(arr, **kwargs)
            else:
                # apply func to array band at a time
                # Avoid mutation by copying (TODO evalutate)
                newarr = arr.copy()
                for b in bands:
                    newarr[b - 1] = func(arr[b - 1], **kwargs)
            return newarr

        yield f
<commit_msg>add validation logic<commit_after>import numpy as np
from .utils import epsilon

# Color manipulation functions
def sigmoidal(arr, contrast, bias):
    """
    Sigmoidal contrast is type of contrast control that
    adjusts the contrast without saturating highlights or shadows.
    It allows control over two factors:
    the contrast range from light to dark, and where the middle value
    of the mid-tones falls. The result is a non-linear and smooth
    contrast change.

    Parameters
    ----------
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float
        Threshold level for the contrast function to center on
        (typically centered at '50%')

    Notes
    ----------

    Sigmoidal contrast is based on the sigmoidal transfer function:

    .. math:: g(u) = ( 1/(1 + e^{- \alpha * u + \beta)})

    This sigmoid function is scaled so that the output is bound by
    the interval [0, 1].

    .. math:: ( 1/(1 + e^(\beta * (\alpha - u))) - 1/(1 + e^(\beta * \alpha)))/
        ( 1/(1 + e^(\beta*(\alpha - 1))) - 1/(1 + e^(\beta * \alpha)) )

    Where :math: `\alpha` is the threshold level, and :math: `\beta` the
    contrast factor to be applied.

    References
    ----------
    .. [CT] Hany Farid "Fundamentals of Image Processing"
            http://www.cs.dartmouth.edu/farid/downloads/tutorials/fip.pdf

    """
    alpha, beta = bias, contrast
    # We use the names a and b to match documentation.

    if alpha == 0:
        alpha = epsilon

    if beta == 0:
        return arr

    np.seterr(divide='ignore', invalid='ignore')

    if beta > 0:
        numerator = 1 / (1 + np.exp(beta * (alpha - arr))) - \
            1 / (1 + np.exp(beta * alpha))
        denominator = 1 / (1 + np.exp(beta * (alpha - 1))) - \
            1 / (1 + np.exp(beta * alpha))
        output = numerator / denominator

    else:
        # Inverse sigmoidal function:
        # todo: account for 0s
        # todo: formatting ;)
        output = (
            (beta * alpha) - np.log(
                (
                    1 / (
                        (arr / (1 + np.exp(beta * alpha - beta))) -
                        (arr / (1 + np.exp(beta * alpha))) +
                        (1 / (1 + np.exp(beta * alpha)))
                    )
                ) - 1)
            ) / beta

    if np.any(output < 0) or np.any(output > (1 + epsilon)):
        raise ValueError("Output is not within the range of [0,1]")
    else:
        return output

def gamma(arr, g):
    """
    Gamma correction is a nonlinear operation that
    adjusts the image's channel values pixel-by-pixel according
    to a power-law:

    .. math:: pixel_{out} = pixel_{in} ^ {\gamma}

    Setting gamma (:math:`\gamma`) to be less than 1.0 darkens the image and
    setting gamma to be greater than 1.0 lightens it.

    Parameters
    ----------
    gamma (:math:`\gamma`): float
        Reasonable values range from 0.8 to 2.3.

    """
    return arr**(1.0 / g)

def saturation(arr, percent):
    """
    Multiply saturation by percent in LCH color space to adjust the intensity
    of color in the image. As saturation increases, colors appear
    more "pure." As saturation decreases, colors appear more "washed-out."

    Parameters
    ----------
    percent: integer

    """
    lch = rgb2lch(arr)
    # Adjust chroma, band at index=1
    lch[1] = lch[1] * (percent / 100.0)
    return lch2rgb(lch)

# Utility functions

def rgb2lch(rgb):
    """
    Converts image array from RGB to LCH color space
    """
    from skimage.color import rgb2lab, lab2lch
    # reshape for skimage (bands, cols, rows) -> (cols, rows, bands)
    srgb = np.swapaxes(rgb, 0, 2)
    # convert colorspace
    lch = lab2lch(rgb2lab(srgb))
    # return in (bands, cols, rows) order
    return np.swapaxes(lch, 2, 0)

def lch2rgb(lch):
    """
    Converts image array from LCH color space to RGB
    """
    from skimage.color import lch2lab, lab2rgb
    # reshape for skimage (bands, cols, rows) -> (cols, rows, bands)
    slch = np.swapaxes(lch, 0, 2)
    # convert colorspace
    rgb = lab2rgb(lch2lab(slch))
    # return in (bands, cols, rows) order
    return np.swapaxes(rgb, 2, 0)

def simple_atmo(rgb, haze, contrast, bias):
    '''
    A simple, static (non-adaptive) atmospheric correction function.

    Parameters
    ----------
    haze: float
        Amount of haze to adjust for. For example, 0.03
    contrast : integer
        Enhances the intensity differences between the lighter and darker
        elements of the image. For example, 0 is none, 3 is typical and
        20 is a lot.
    bias : float
        Threshold level for the contrast function to center on
        (typically centered at '50%')

    '''
    # bias assumed to be given in percent,
    # convert to proportion
    bias = bias / 100.0

    gamma_b = 1 - haze
    gamma_g = 1 - (haze * (1 / 3.0))

    rgb[1] = gamma(rgb[1], gamma_g)
    rgb[2] = gamma(rgb[2], gamma_b)

    return sigmoidal(rgb, contrast, bias)

def parse_operations(operations, count=3):
    """
    Takes an iterable of operations,
    each operation is expected to be a string with a specified syntax

    "OPERATION-NAME BANDS ARGS..."

    And yields a list of functions that take and return ndarrays
    """
    band_lookup = {'r': 1, 'g': 2, 'b': 3}

    opfuncs = {
        'saturation': saturation,
        'sigmoidal': sigmoidal,
        'gamma': gamma}

    opkwargs = {
        'saturation': ('percent',),
        'sigmoidal': ('contrast', 'bias'),
        'gamma': ('g',)}

    # Operations that assume RGB colorspace
    rgb_ops = ('saturation',)

    for op in operations:
        parts = op.split(" ")
        opname = parts[0]
        bandstr = parts[1]
        args = parts[2:]

        try:
            func = opfuncs[opname]
        except KeyError:
            raise ValueError("{} is not a valid operation".format(opname))

        if opname in rgb_ops:
            # ignore bands, assumed to be in rgb
            # push 2nd arg into args
            args = [bandstr] + args
            bands = (1, 2, 3)
        else:
            # 2nd arg is bands
            # parse r,g,b ~= 1,2,3
            bands = set()
            for bs in bandstr.split(","):
                try:
                    band = int(bs)
                except ValueError:
                    band = band_lookup[bs.lower()]
                if band < 1 or band > count:
                    raise ValueError(
                        "{} BAND must be between 1 and {}"
                        .format(opname, count))
                bands.add(band)

        # assume all args are float
        args = [float(arg) for arg in args]
        kwargs = dict(zip(opkwargs[opname], args))

        def f(arr, func=func, kwargs=kwargs):
            if opname in rgb_ops:
                # apply func to array assuming 3 band r,g,b
                newarr = func(arr, **kwargs)
            else:
                # apply func to array band at a time
                # Avoid mutation by copying (TODO evalutate)
                newarr = arr.copy()
                for b in bands:
                    newarr[b - 1] = func(arr[b - 1], **kwargs)
            return newarr

        yield f
<|endoftext|>


<commit_before>#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import errno
import os
import socket
import sys
import tempfile
import time
import traceback

from conary import conaryclient
from conary.lib import util
from conary.repository import changeset

from rmake import errors

from rmake import failure
from rmake.build import subscriber

from rmake.lib import logfile
from rmake.lib import logger
from rmake.lib import server

from rmake.lib.apiutils import thaw, freeze

class Command(server.Server):
    name = 'command'
    # handles one job then exists.
    def __init__(self, cfg, commandId, jobId):
        server.Server.__init__(self)
        self.cfg = cfg
        self.commandId = commandId
        self.jobId = jobId
        self._isErrored = False
        self._errorMessage = ''
        self._output = []
        self.readPipe = None

    def _exit(self, exitRc):
        sys.exit(exitRc)

    def shouldFork(self):
        return True

    def setWritePipe(self, writePipe):
        self.writePipe = writePipe

    def setReadPipe(self, readPipe):
        self.readPipe = readPipe

    def fileno(self):
        return self.readPipe.fileno()

    def handleRead(self):
        data = self.readPipe.handle_read()
        if data:
            self._handleData(data)

    def flushInputBuffer(self):
        if not self.readPipe:
            return
        for data in self.readPipe.readUntilClosed():
            self._handleData(data)

    def handleWrite(self):
        self.writePipe.handle_write()

    def _handleData(self, data):
        self._output.append(data)

    def getCommandId(self):
        return self.commandId

    def getChrootFactory(self):
        return None

    def getLogPath(self):
        commandId = (self.getCommandId().replace('/', '_')
                                                    .replace('~', '')
                                                    .replace(' ', ''))
        base =  self.cfg.getBuildLogDir(self.jobId)
        return '%s/%s.log' % (base, commandId)

    def runCommandAndExit(self):
        self._installSignalHandlers()
        try:
            self._try('Command', self._runCommand)
            os._exit(0)
        except SystemExit, err:
            os._exit(err.args[0])
        except:
            # error occurred, but we'll let our parent node send out the
            # "errored" message.
            os._exit(1)

    def _serveLoopHook(self):
        self.writePipe.handleWriteIfReady()
        self._collectChildren()

    def runCommandNoExit(self):
        try:
            self._try('Command', self._runCommand)
        except SystemExit, err:
            return err.args[0]
        return 0

    def _runCommand(self):
        self.commandStarted()
        self.logger = logger.Logger(self.name, self.getLogPath())
        self.logFile = logfile.LogFile(self.getLogPath())
        self.logFile.redirectOutput()
        try:
            self.logger.info('Running Command... (pid %s)' % os.getpid())
            try:
                self.runCommand()
            except SystemExit, err:
                raise
            except Exception, err:
                if isinstance(err, SystemExit) and not err.args[0]:
                    self.commandFinished()
                else:
                    self.commandErrored(err, traceback.format_exc())
                raise
            else:
                self.commandFinished()
        finally:
            self.logFile.restoreOutput()

    def isErrored(self):
        return self._isErrored

    def getFailureReason(self):
        return self.failureReason

    def setError(self, msg, tb=''):
        self._isErrored = True
        self.failureReason = failure.CommandFailed(self.commandId, str(msg), tb)

    def commandDied(self, status):
        self.flushInputBuffer()
        exitRc = os.WEXITSTATUS(status)
        signalRc = os.WTERMSIG(status)
        commandId = self.getCommandId()
        if exitRc or signalRc:
            if exitRc:
                msg = 'unexpectedly died with exit code %s'
                msg = msg % (exitRc)
            else:
                msg = 'unexpectedly killed with signal %s'
                msg = msg % (signalRc)
            self.setError(msg)
        self.commandFinished()

    def commandStarted(self):
        pass

    def commandFinished(self):
        pass

    def commandErrored(self, msg, tb=''):
        self.setError(msg, tb)

class BuildCommand(Command):

    name = 'build-command'

    def __init__(self, serverCfg, commandId, jobId, eventHandler, buildCfg,
                 chrootFactory, trove, targetLabel, logHost='', logPort=0,
                 logPath=None, uri=None):
        Command.__init__(self, serverCfg, commandId, jobId)
        self.eventHandler = eventHandler
        self.buildCfg = buildCfg
        self.chrootFactory = chrootFactory
        self.trove = trove
        self.targetLabel = targetLabel
        self.logHost = logHost
        self.logPort = logPort
        self.logPath = logPath
        self.readPipe = None
        self.failureReason = None
        self.uri = None

    def _signalHandler(self, signal, frame):
        self.failureReason = failure.Stopped('Signal %s received' % signal)
        self.trove.troveFailed(self.failureReason)
        Command._signalHandler(self, signal, frame)

    def _shutDown(self):
        if self.failureReason:
            self.trove.troveFailed(self.failureReason)
        try:
            self.chroot.stop()
        except OSError, err:
            if err.errno != errno.ESRCH:
                raise
            else:
                return
        except errors.OpenError, err:
            pass
        self._killPid(self.chroot.pid)
        Command._shutDown(self)

    def _handleData(self, (jobId, eventList)):
        self.eventHandler._receiveEvents(*thaw('EventList', eventList))

    def getTrove(self):
        return self.trove

    def getChrootFactory(self):
        return self.chrootFactory

    def runCommand(self):
        try:
            try:
                trove = self.trove
                trove.getPublisher().reset()
                PipePublisher(self.writePipe).attach(trove)
                trove.creatingChroot(self.cfg.getName(),
                                     self.chrootFactory.getChrootName())
                self.chrootFactory.create()
                self.chroot = self.chrootFactory.start()
            except Exception, err:
                # sends off messages to all listeners that this trove failed.
                trove.chrootFailed(str(err), traceback.format_exc())
                return
            n,v,f = trove.getNameVersionFlavor()
            logPath, pid = self.chroot.buildTrove(self.buildCfg,
                                                  self.targetLabel,
                                                  n, v, f, self.logHost,
                                                  self.logPort)
            # sends off message that this trove is building.
            self.chroot.subscribeToBuild(n,v,f)
            if self.logPath:
                logPath = self.logPath
            trove.troveBuilding(logPath, pid)
            self.serve_forever()
        except SystemExit, err:
            self.writePipe.flush()
            raise
        except Exception, err:
            self.writePipe.flush()
            # even if there's an exception in here, we don't really
            # want to retry the build.
            reason = failure.InternalError(str(err), traceback.format_exc())
            trove.troveFailed(reason)
            raise

    def handleRequestIfReady(self, sleep):
        time.sleep(sleep)

    def _serveLoopHook(self):
        try:
            self.writePipe.handleWriteIfReady()
            if self.chroot.checkSubscription():
                self.getResults()
                self._halt = True
                return
        except SystemExit, err:
            raise
        except Exception, err:
            reason = failure.InternalError(str(err), traceback.format_exc())
            self.getTrove().troveFailed(reason)
            raise

    def getResults(self):
        try:
            trove = self.trove
            repos = conaryclient.ConaryClient(self.buildCfg).getRepos()
            buildResult = self.chroot.checkResults(
                                            *self.trove.getNameVersionFlavor())
            if buildResult.isBuildSuccess():
                csFile = buildResult.getChangeSetFile()
                cs = changeset.ChangeSetFromFile(csFile)
                repos.commitChangeSet(cs)
                # sends off message that this trove built successfully
                troveList = [x.getNewNameVersionFlavor() for
                             x in cs.iterNewTroveList() ]
                trove.troveBuilt(troveList)
                del cs # this makes sure the changeset closes the fd.
            else:
                reason = buildResult.getFailureReason()
                trove.troveFailed(reason)
                # passes through to killRoot at the bottom.
        except Exception, e:
            reason = failure.InternalError(str(e), traceback.format_exc())
            trove.troveFailed(reason)
        self.chroot.stop()

    def _shutDown(self):
        if self.buildCfg.cleanAfterCook and self.trove.isBuilt():
            self.chrootFactory.clean()
        else:
            self.chrootFactory.unmount()
        Command._shutDown(self)

class StopCommand(Command):

    name = 'stop-command'

    def __init__(self, cfg, commandId, targetCommand, killFn, hook):
        Command.__init__(self, cfg, commandId, targetCommand.jobId)
        self.targetCommand = targetCommand
        self.killFn = killFn
        self.hook = hook

    def runCommand(self):
        self.killFn(self.targetCommand.pid, hook=self.hook)

    def shouldFork(self):
        # Because this command runs kill, it must be the parent process
        # that runs kill, not some other child.  You can't kill your sibling.
        return False

    def commandDied(self, status):
        Command.commandDied(self, status)
        try:
            pid, status = os.waitpid(self.targetCommand.pid, os.WNOHANG)
        except OSError, err:
            if err.errno == errno.ECHILD:
                pass
            elif err.errno == errno.ESRCH:
                pass
            else:
                raise
        else:
            self.setError('%s did not die!' % self.targetCommand.getCommandId())

class SessionCommand(Command):

    name = 'session-command'

    def __init__(self, serverCfg, commandId, chrootFactory, command):
        Command.__init__(self, serverCfg, commandId, 0)
        self.chrootFactory = chrootFactory
        self.command = command
        self.hostInfo = []

    def getTrove(self):
        return self.trove

    def getChrootFactory(self):
        return self.chrootFactory

    def runCommand(self):
        self.chrootFactory.create()
        self.chroot = self.chrootFactory.start()
        port = self.chroot.startSession(self.command)
        self.writePipe.send((socket.getfqdn(), port))
        self.writePipe.flush()
        self.serve_forever()

    def _pidDied(self, pid, status, name=None):
        if pid == self.chroot.pid:
            self.chrootFactory.unmount()
            self._halt = True

    def _handleData(self, output):
        self.hostInfo = output

    def getHostInfo(self):
        return self.hostInfo

class PipePublisher(subscriber._RmakePublisherProxy):
    """
        Class that transmits events from internal build process -> rMake server.
    """

    # we override the _receiveEvents method to just pass these
    # events on, thus we just use listeners as a list of states we subscribe to

    def __init__(self, pipeWriter):
        self.pipeWriter = pipeWriter
        subscriber._RmakePublisherProxy.__init__(self)

    def _emitEvents(self, jobId, eventList):
        self.pipeWriter.send((jobId, freeze('EventList', eventList)))
        self.pipeWriter.flush()

def attach(trove, p):
    _RmakeBusPublisher(client).attach(trove)
<commit_msg>oops, combine two _shutDown methods<commit_after>#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import errno
import os
import socket
import sys
import tempfile
import time
import traceback

from conary import conaryclient
from conary.lib import util
from conary.repository import changeset

from rmake import errors

from rmake import failure
from rmake.build import subscriber

from rmake.lib import logfile
from rmake.lib import logger
from rmake.lib import server

from rmake.lib.apiutils import thaw, freeze

class Command(server.Server):
    name = 'command'
    # handles one job then exists.
    def __init__(self, cfg, commandId, jobId):
        server.Server.__init__(self)
        self.cfg = cfg
        self.commandId = commandId
        self.jobId = jobId
        self._isErrored = False
        self._errorMessage = ''
        self._output = []
        self.readPipe = None

    def _exit(self, exitRc):
        sys.exit(exitRc)

    def shouldFork(self):
        return True

    def setWritePipe(self, writePipe):
        self.writePipe = writePipe

    def setReadPipe(self, readPipe):
        self.readPipe = readPipe

    def fileno(self):
        return self.readPipe.fileno()

    def handleRead(self):
        data = self.readPipe.handle_read()
        if data:
            self._handleData(data)

    def flushInputBuffer(self):
        if not self.readPipe:
            return
        for data in self.readPipe.readUntilClosed():
            self._handleData(data)

    def handleWrite(self):
        self.writePipe.handle_write()

    def _handleData(self, data):
        self._output.append(data)

    def getCommandId(self):
        return self.commandId

    def getChrootFactory(self):
        return None

    def getLogPath(self):
        commandId = (self.getCommandId().replace('/', '_')
                                                    .replace('~', '')
                                                    .replace(' ', ''))
        base =  self.cfg.getBuildLogDir(self.jobId)
        return '%s/%s.log' % (base, commandId)

    def runCommandAndExit(self):
        self._installSignalHandlers()
        try:
            self._try('Command', self._runCommand)
            os._exit(0)
        except SystemExit, err:
            os._exit(err.args[0])
        except:
            # error occurred, but we'll let our parent node send out the
            # "errored" message.
            os._exit(1)

    def _serveLoopHook(self):
        self.writePipe.handleWriteIfReady()
        self._collectChildren()

    def runCommandNoExit(self):
        try:
            self._try('Command', self._runCommand)
        except SystemExit, err:
            return err.args[0]
        return 0

    def _runCommand(self):
        self.commandStarted()
        self.logger = logger.Logger(self.name, self.getLogPath())
        self.logFile = logfile.LogFile(self.getLogPath())
        self.logFile.redirectOutput()
        try:
            self.logger.info('Running Command... (pid %s)' % os.getpid())
            try:
                self.runCommand()
            except SystemExit, err:
                raise
            except Exception, err:
                if isinstance(err, SystemExit) and not err.args[0]:
                    self.commandFinished()
                else:
                    self.commandErrored(err, traceback.format_exc())
                raise
            else:
                self.commandFinished()
        finally:
            self.logFile.restoreOutput()

    def isErrored(self):
        return self._isErrored

    def getFailureReason(self):
        return self.failureReason

    def setError(self, msg, tb=''):
        self._isErrored = True
        self.failureReason = failure.CommandFailed(self.commandId, str(msg), tb)

    def commandDied(self, status):
        self.flushInputBuffer()
        exitRc = os.WEXITSTATUS(status)
        signalRc = os.WTERMSIG(status)
        commandId = self.getCommandId()
        if exitRc or signalRc:
            if exitRc:
                msg = 'unexpectedly died with exit code %s'
                msg = msg % (exitRc)
            else:
                msg = 'unexpectedly killed with signal %s'
                msg = msg % (signalRc)
            self.setError(msg)
        self.commandFinished()

    def commandStarted(self):
        pass

    def commandFinished(self):
        pass

    def commandErrored(self, msg, tb=''):
        self.setError(msg, tb)

class BuildCommand(Command):

    name = 'build-command'

    def __init__(self, serverCfg, commandId, jobId, eventHandler, buildCfg,
                 chrootFactory, trove, targetLabel, logHost='', logPort=0,
                 logPath=None, uri=None):
        Command.__init__(self, serverCfg, commandId, jobId)
        self.eventHandler = eventHandler
        self.buildCfg = buildCfg
        self.chrootFactory = chrootFactory
        self.trove = trove
        self.targetLabel = targetLabel
        self.logHost = logHost
        self.logPort = logPort
        self.logPath = logPath
        self.readPipe = None
        self.failureReason = None
        self.uri = None

    def _signalHandler(self, signal, frame):
        failureReason = failure.Stopped('Signal %s received' % signal)
        self.trove.troveFailed(failureReason)
        Command._signalHandler(self, signal, frame)

    def _handleData(self, (jobId, eventList)):
        self.eventHandler._receiveEvents(*thaw('EventList', eventList))

    def getTrove(self):
        return self.trove

    def getChrootFactory(self):
        return self.chrootFactory

    def runCommand(self):
        try:
            try:
                trove = self.trove
                trove.getPublisher().reset()
                PipePublisher(self.writePipe).attach(trove)
                trove.creatingChroot(self.cfg.getName(),
                                     self.chrootFactory.getChrootName())
                self.chrootFactory.create()
                self.chroot = self.chrootFactory.start()
            except Exception, err:
                # sends off messages to all listeners that this trove failed.
                trove.chrootFailed(str(err), traceback.format_exc())
                return
            n,v,f = trove.getNameVersionFlavor()
            logPath, pid = self.chroot.buildTrove(self.buildCfg,
                                                  self.targetLabel,
                                                  n, v, f, self.logHost,
                                                  self.logPort)
            # sends off message that this trove is building.
            self.chroot.subscribeToBuild(n,v,f)
            if self.logPath:
                logPath = self.logPath
            trove.troveBuilding(logPath, pid)
            self.serve_forever()
        except SystemExit, err:
            self.writePipe.flush()
            raise
        except Exception, err:
            self.writePipe.flush()
            # even if there's an exception in here, we don't really
            # want to retry the build.
            reason = failure.InternalError(str(err), traceback.format_exc())
            trove.troveFailed(reason)
            raise

    def handleRequestIfReady(self, sleep):
        time.sleep(sleep)

    def _serveLoopHook(self):
        try:
            self.writePipe.handleWriteIfReady()
            if self.chroot.checkSubscription():
                self.getResults()
                self._halt = True
                return
        except SystemExit, err:
            raise
        except Exception, err:
            reason = failure.InternalError(str(err), traceback.format_exc())
            self.getTrove().troveFailed(reason)
            raise

    def getResults(self):
        try:
            trove = self.trove
            repos = conaryclient.ConaryClient(self.buildCfg).getRepos()
            buildResult = self.chroot.checkResults(
                                            *self.trove.getNameVersionFlavor())
            if buildResult.isBuildSuccess():
                csFile = buildResult.getChangeSetFile()
                cs = changeset.ChangeSetFromFile(csFile)
                repos.commitChangeSet(cs)
                # sends off message that this trove built successfully
                troveList = [x.getNewNameVersionFlavor() for
                             x in cs.iterNewTroveList() ]
                trove.troveBuilt(troveList)
                del cs # this makes sure the changeset closes the fd.
            else:
                reason = buildResult.getFailureReason()
                trove.troveFailed(reason)
                # passes through to killRoot at the bottom.
        except Exception, e:
            reason = failure.InternalError(str(e), traceback.format_exc())
            trove.troveFailed(reason)
        self.chroot.stop()

    def _shutDown(self):
        try:
            self.chroot.stop()
        except OSError, err:
            if err.errno != errno.ESRCH:
                raise
            else:
                return
        except errors.OpenError, err:
            pass
        self._killPid(self.chroot.pid)

        if self.buildCfg.cleanAfterCook and self.trove.isBuilt():
            self.chrootFactory.clean()
        else:
            self.chrootFactory.unmount()
        Command._shutDown(self)

class StopCommand(Command):

    name = 'stop-command'

    def __init__(self, cfg, commandId, targetCommand, killFn, hook):
        Command.__init__(self, cfg, commandId, targetCommand.jobId)
        self.targetCommand = targetCommand
        self.killFn = killFn
        self.hook = hook

    def runCommand(self):
        self.killFn(self.targetCommand.pid, hook=self.hook)

    def shouldFork(self):
        # Because this command runs kill, it must be the parent process
        # that runs kill, not some other child.  You can't kill your sibling.
        return False

    def commandDied(self, status):
        Command.commandDied(self, status)
        try:
            pid, status = os.waitpid(self.targetCommand.pid, os.WNOHANG)
        except OSError, err:
            if err.errno == errno.ECHILD:
                pass
            elif err.errno == errno.ESRCH:
                pass
            else:
                raise
        else:
            self.setError('%s did not die!' % self.targetCommand.getCommandId())

class SessionCommand(Command):

    name = 'session-command'

    def __init__(self, serverCfg, commandId, chrootFactory, command):
        Command.__init__(self, serverCfg, commandId, 0)
        self.chrootFactory = chrootFactory
        self.command = command
        self.hostInfo = []

    def getTrove(self):
        return self.trove

    def getChrootFactory(self):
        return self.chrootFactory

    def runCommand(self):
        self.chrootFactory.create()
        self.chroot = self.chrootFactory.start()
        port = self.chroot.startSession(self.command)
        self.writePipe.send((socket.getfqdn(), port))
        self.writePipe.flush()
        self.serve_forever()

    def _pidDied(self, pid, status, name=None):
        if pid == self.chroot.pid:
            self.chrootFactory.unmount()
            self._halt = True

    def _handleData(self, output):
        self.hostInfo = output

    def getHostInfo(self):
        return self.hostInfo

class PipePublisher(subscriber._RmakePublisherProxy):
    """
        Class that transmits events from internal build process -> rMake server.
    """

    # we override the _receiveEvents method to just pass these
    # events on, thus we just use listeners as a list of states we subscribe to

    def __init__(self, pipeWriter):
        self.pipeWriter = pipeWriter
        subscriber._RmakePublisherProxy.__init__(self)

    def _emitEvents(self, jobId, eventList):
        self.pipeWriter.send((jobId, freeze('EventList', eventList)))
        self.pipeWriter.flush()

def attach(trove, p):
    _RmakeBusPublisher(client).attach(trove)
<|endoftext|>