<commit_before>""" Contains the base classes for drop targets and drop target banks."""

from mpf.core.delays import DelayManager
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

class DropTarget(SystemWideDevice):
    """Represents a single drop target in a pinball machine.

    Args: Same as the `Target` parent class"""

    config_section = 'drop_targets'
    collection = 'drop_targets'
    class_label = 'drop_target'

    def __init__(self, machine, name):
        self.reset_coil = None
        self.knockdown_coil = None
        self.banks = None
        super().__init__(machine, name)

        self._in_ball_search = False
        self.complete = False
        self.delay = DelayManager(machine.delayRegistry)

    def _initialize(self):
        self.reset_coil = self.config['reset_coil']
        self.knockdown_coil = self.config['knockdown_coil']
        self.banks = set()

        # can't read the switch until the switch controller is set up
        self.machine.events.add_handler('init_phase_4',
                                        self._update_state_from_switch)
        self.machine.events.add_handler('init_phase_4',
                                        self._register_switch_handlers)

        # TODO: make playfield name configureable
        self.machine.ball_devices['playfield'].ball_search.register(self.config['ball_search_order'], self._ball_search)

    def _ball_search_phase1(self):
        if not self.complete and self.reset_coil:
            self.reset_coil.pulse()
            return True
        # if down. knock down again
        elif self.complete and self.knockdown_coil:
            self.knockdown_coil.pulse()
            return True

    def _ball_search_phase2(self):
        if self.reset_coil and self.knockdown_coil:
            if self.complete:
                self._in_ball_search = True
                self.reset_coil.pulse()
                self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                self._in_ball_search = True
                self.knockdown_coil.pulse()
                self.delay.add(100, self._ball_search_reset)
                return True
        else:
            # fall back to phase1
            return self._ball_search_phase1()

    def _ball_search_phase3(self):
        if self.complete:
            if self.reset_coil:
                self.reset_coil.pulse()
                if self.knockdown_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                return self._ball_search_phase1()
        else:
            if self.knockdown_coil:
                self.knockdown_coil.pulse()
                if self.reset_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_reset)
                return True
            else:
                return self._ball_search_phase1()

    def _ball_search_iteration_finish(self):
        self._in_ball_search = False

    def _ball_search_knockdown(self):
        self.knockdown_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search_reset(self):
        self.reset_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search(self, phase, iteration):
        del iteration
        if phase == 1:
            # phase 1: do not change state.
            # if up. reset again
            return self._ball_search_phase1()
        elif phase == 2:
            # phase 2: if we can reset and knockdown the target we will do that
            return self._ball_search_phase2()
        else:
            # phase3: reset no matter what
            return self._ball_search_phase3()

    def _register_switch_handlers(self):
        # register for notification of switch state
        # this is in addition to the parent since drop targets track
        # self.complete in separately

        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 0)
        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 1)

    def knockdown(self, **kwargs):
        """Pulses the knockdown coil to knock down this drop target."""
        del kwargs
        if self.knockdown_coil:
            self.knockdown_coil.pulse()

    def _update_state_from_switch(self):
        if self._in_ball_search:
            return

        if self.machine.switch_controller.is_active(
                self.config['switch'].name):
            self._down()
        else:
            self._up()

        self._update_banks()

    def _down(self):
        self.complete = True
        self.machine.events.post(self.name + '_down')
        '''event: (drop_target_name)_down
        desc: The drop target with the name (drop_target_name) has just
        changed to the "down" state.'''

    def _up(self):
        self.complete = False
        self.machine.events.post(self.name + '_up')
        '''event: (drop_target_name)_up
        desc: The drop target with the name (drop_target_name) has just
        changed to the "up" state.'''

    def _update_banks(self):
        for bank in self.banks:
            bank.member_target_change()

    def add_to_bank(self, bank):
        """Adds this drop target to a drop target bank, which allows the bank to
        update its status based on state changes to this drop target.

        Args:
            bank: DropTargetBank object to add this drop target to.

        """
        self.banks.add(bank)

    def remove_from_bank(self, bank):
        """Remove the DropTarget from a bank

        Args:
            bank: DropTargetBank object to remove

        """
        self.banks.remove(bank)

    def reset(self, **kwargs):
        """Resets this drop target.

        If this drop target is configured with a reset coil, then this method
        will pulse that coil. If not, then it checks to see if this drop target
        is part of a drop target bank, and if so, it calls the reset() method of
        the drop target bank.

        This method does not reset the target profile, however, the switch event
        handler should reset the target profile on its own when the drop target
        physically moves back to the up position.

        """
        del kwargs

        if self.reset_coil:
            self.reset_coil.pulse()

class DropTargetBank(SystemWideDevice, ModeDevice):
    """Represents a bank of drop targets in a pinball machine by grouping
    together multiple `DropTarget` class devices.

    """
    config_section = 'drop_target_banks'
    collection = 'drop_target_banks'
    class_label = 'drop_target_bank'

    def __init__(self, machine, name):
        super().__init__(machine, name)

        self.drop_targets = list()
        self.reset_coil = None
        self.reset_coils = set()
        self.complete = False
        self.down = 0
        self.up = 0

    def _initialize(self):
        self.drop_targets = self.config['drop_targets']
        self.reset_coil = self.config['reset_coil']
        self.reset_coils = self.config['reset_coils']

        for target in self.drop_targets:
            target.add_to_bank(self)

        self.member_target_change()

        if self.debug:
            self.log.debug('Drop Targets: %s', self.drop_targets)

    def reset(self, **kwargs):
        """Resets this bank of drop targets.

        This method has some intelligence to figure out what coil(s) it should
        fire. It builds up a set by looking at its own reset_coil and
        reset_coils settings, and also scanning through all the member drop
        targets and collecting their coils. Then it pulses each of them. (This
        coil list is a "set" which means it only sends a single pulse to each
        coil, even if each drop target is configured with its own coil.)

        """
        del kwargs
        if self.debug:
            self.log.debug('Resetting')

        # figure out all the coils we need to pulse
        coils = set()

        for drop_target in self.drop_targets:
            if drop_target.reset_coil:
                coils.add(drop_target.reset_coil)

        for coil in self.reset_coils:
            coils.add(coil)

        if self.reset_coil:
            coils.add(self.reset_coil)

        # now pulse them
        for coil in coils:

            if self.debug:
                self.log.debug('Pulsing reset coils: %s', coils)

            coil.pulse()

    def member_target_change(self):
        """A member drop target has changed state.

        This method causes this group to update its down and up counts and
        complete status.

        """
        self.down = 0
        self.up = 0

        for target in self.drop_targets:
            if target.complete:
                self.down += 1
            else:
                self.up += 1

        if self.debug:
            self.log.debug(
                'Member drop target status change: Up: %s, Down: %s,'
                ' Total: %s', self.up, self.down,
                len(self.drop_targets))

        if self.down == len(self.drop_targets):
            self._bank_down()
        elif not self.down:
            self._bank_up()
        else:
            self._bank_mixed()

    def _bank_down(self):
        self.complete = True
        if self.debug:
            self.log.debug('All targets are down')

        self.machine.events.post(self.name + '_down')
        '''event: (drop_target_bank_name)_down
        desc: Every drop target in the drop target bank called
        (drop_target_bank_name) is now in the "down" state. This event is
        only posted once, when all the drop targets are down.'''

    def _bank_up(self):
        self.complete = False
        if self.debug:
            self.log.debug('All targets are up')
        self.machine.events.post(self.name + '_up')
        '''event: (drop_target_bank_name)_up
        desc: Every drop target in the drop target bank called
        (drop_target_bank_name) is now in the "up" state. This event is
        only posted once, when all the drop targets are up.'''

    def _bank_mixed(self):
        self.complete = False
        self.machine.events.post(self.name + '_mixed', down=self.down)
        '''event: (drop_target_bank_name)_mixed
        desc: The drop targets in the drop target bank
        (drop_target_bank_name) are in a "mixed" state, meaning that they're
        not all down or not all up. This event is posted every time a member
        drop target changes but the overall bank is not not complete.'''

    def device_removed_from_mode(self, mode):
        for target in self.drop_targets:
            target.remove_from_bank(self)
<commit_msg>update drop target events to be consistent with other event names<commit_after>""" Contains the base classes for drop targets and drop target banks."""

from mpf.core.delays import DelayManager
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

class DropTarget(SystemWideDevice):
    """Represents a single drop target in a pinball machine.

    Args: Same as the `Target` parent class"""

    config_section = 'drop_targets'
    collection = 'drop_targets'
    class_label = 'drop_target'

    def __init__(self, machine, name):
        self.reset_coil = None
        self.knockdown_coil = None
        self.banks = None
        super().__init__(machine, name)

        self._in_ball_search = False
        self.complete = False
        self.delay = DelayManager(machine.delayRegistry)

    def _initialize(self):
        self.reset_coil = self.config['reset_coil']
        self.knockdown_coil = self.config['knockdown_coil']
        self.banks = set()

        # can't read the switch until the switch controller is set up
        self.machine.events.add_handler('init_phase_4',
                                        self._update_state_from_switch)
        self.machine.events.add_handler('init_phase_4',
                                        self._register_switch_handlers)

        # TODO: make playfield name configurable
        self.machine.ball_devices['playfield'].ball_search.register(
            self.config['ball_search_order'], self._ball_search)

    def _ball_search_phase1(self):
        if not self.complete and self.reset_coil:
            self.reset_coil.pulse()
            return True
        # if down. knock down again
        elif self.complete and self.knockdown_coil:
            self.knockdown_coil.pulse()
            return True

    def _ball_search_phase2(self):
        if self.reset_coil and self.knockdown_coil:
            if self.complete:
                self._in_ball_search = True
                self.reset_coil.pulse()
                self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                self._in_ball_search = True
                self.knockdown_coil.pulse()
                self.delay.add(100, self._ball_search_reset)
                return True
        else:
            # fall back to phase1
            return self._ball_search_phase1()

    def _ball_search_phase3(self):
        if self.complete:
            if self.reset_coil:
                self.reset_coil.pulse()
                if self.knockdown_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                return self._ball_search_phase1()
        else:
            if self.knockdown_coil:
                self.knockdown_coil.pulse()
                if self.reset_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_reset)
                return True
            else:
                return self._ball_search_phase1()

    def _ball_search_iteration_finish(self):
        self._in_ball_search = False

    def _ball_search_knockdown(self):
        self.knockdown_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search_reset(self):
        self.reset_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search(self, phase, iteration):
        del iteration
        if phase == 1:
            # phase 1: do not change state.
            # if up. reset again
            return self._ball_search_phase1()
        elif phase == 2:
            # phase 2: if we can reset and knockdown the target we will do that
            return self._ball_search_phase2()
        else:
            # phase3: reset no matter what
            return self._ball_search_phase3()

    def _register_switch_handlers(self):
        # register for notification of switch state
        # this is in addition to the parent since drop targets track
        # self.complete in separately

        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 0)
        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 1)

    def knockdown(self, **kwargs):
        """Pulses the knockdown coil to knock down this drop target."""
        del kwargs
        if self.knockdown_coil:
            self.knockdown_coil.pulse()

    def _update_state_from_switch(self):
        if self._in_ball_search:
            return

        if self.machine.switch_controller.is_active(
                self.config['switch'].name):
            self._down()
        else:
            self._up()

        self._update_banks()

    def _down(self):
        self.complete = True
        self.machine.events.post('drop_target_' + self.name + '_down')
        '''event: drop_target_(name)_down
        desc: The drop target with the (name) has just changed to the "down"
        state.'''

    def _up(self):
        self.complete = False
        self.machine.events.post('drop_target_' + self.name + '_up')
        '''event: drop_target_(name)_up
        desc: The drop target (name) has just changed to the "up" state.'''

    def _update_banks(self):
        for bank in self.banks:
            bank.member_target_change()

    def add_to_bank(self, bank):
        """Adds this drop target to a drop target bank, which allows the bank to
        update its status based on state changes to this drop target.

        Args:
            bank: DropTargetBank object to add this drop target to.

        """
        self.banks.add(bank)

    def remove_from_bank(self, bank):
        """Remove the DropTarget from a bank

        Args:
            bank: DropTargetBank object to remove

        """
        self.banks.remove(bank)

    def reset(self, **kwargs):
        """Resets this drop target.

        If this drop target is configured with a reset coil, then this method
        will pulse that coil. If not, then it checks to see if this drop target
        is part of a drop target bank, and if so, it calls the reset() method of
        the drop target bank.

        This method does not reset the target profile, however, the switch event
        handler should reset the target profile on its own when the drop target
        physically moves back to the up position.

        """
        del kwargs

        if self.reset_coil:
            self.reset_coil.pulse()

class DropTargetBank(SystemWideDevice, ModeDevice):
    """Represents a bank of drop targets in a pinball machine by grouping
    together multiple `DropTarget` class devices.

    """
    config_section = 'drop_target_banks'
    collection = 'drop_target_banks'
    class_label = 'drop_target_bank'

    def __init__(self, machine, name):
        super().__init__(machine, name)

        self.drop_targets = list()
        self.reset_coil = None
        self.reset_coils = set()
        self.complete = False
        self.down = 0
        self.up = 0

    def _initialize(self):
        self.drop_targets = self.config['drop_targets']
        self.reset_coil = self.config['reset_coil']
        self.reset_coils = self.config['reset_coils']

        for target in self.drop_targets:
            target.add_to_bank(self)

        self.member_target_change()

        if self.debug:
            self.log.debug('Drop Targets: %s', self.drop_targets)

    def reset(self, **kwargs):
        """Resets this bank of drop targets.

        This method has some intelligence to figure out what coil(s) it should
        fire. It builds up a set by looking at its own reset_coil and
        reset_coils settings, and also scanning through all the member drop
        targets and collecting their coils. Then it pulses each of them. (This
        coil list is a "set" which means it only sends a single pulse to each
        coil, even if each drop target is configured with its own coil.)

        """
        del kwargs
        if self.debug:
            self.log.debug('Resetting')

        # figure out all the coils we need to pulse
        coils = set()

        for drop_target in self.drop_targets:
            if drop_target.reset_coil:
                coils.add(drop_target.reset_coil)

        for coil in self.reset_coils:
            coils.add(coil)

        if self.reset_coil:
            coils.add(self.reset_coil)

        # now pulse them
        for coil in coils:

            if self.debug:
                self.log.debug('Pulsing reset coils: %s', coils)

            coil.pulse()

    def member_target_change(self):
        """A member drop target has changed state.

        This method causes this group to update its down and up counts and
        complete status.

        """
        self.down = 0
        self.up = 0

        for target in self.drop_targets:
            if target.complete:
                self.down += 1
            else:
                self.up += 1

        if self.debug:
            self.log.debug(
                'Member drop target status change: Up: %s, Down: %s,'
                ' Total: %s', self.up, self.down,
                len(self.drop_targets))

        if self.down == len(self.drop_targets):
            self._bank_down()
        elif not self.down:
            self._bank_up()
        else:
            self._bank_mixed()

    def _bank_down(self):
        self.complete = True
        if self.debug:
            self.log.debug('All targets are down')

        self.machine.events.post('drop_target_bank_' + self.name + '_down')
        '''event: drop_target_bank_(name)_down
        desc: Every drop target in the drop target bank called
        (name) is now in the "down" state. This event is
        only posted once, when all the drop targets are down.'''

    def _bank_up(self):
        self.complete = False
        if self.debug:
            self.log.debug('All targets are up')
        self.machine.events.post('drop_target_bank_' + self.name + '_up')
        '''event: drop_target_bank_(name)_up
        desc: Every drop target in the drop target bank called
        (name) is now in the "up" state. This event is
        only posted once, when all the drop targets are up.'''

    def _bank_mixed(self):
        self.complete = False
        self.machine.events.post('drop_target_bank_' + self.name + '_mixed',
                                 down=self.down)
        '''event: drop_target_bank_(name)_mixed
        desc: The drop targets in the drop target bank
        (name) are in a "mixed" state, meaning that they're
        not all down or not all up. This event is posted every time a member
        drop target changes but the overall bank is not not complete.'''

    def device_removed_from_mode(self, mode):
        for target in self.drop_targets:
            target.remove_from_bank(self)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import collections
import os
import re
import time
import sys

import click
from netCDF4 import Dataset
import numpy as np
import numpy.ma as ma
import osr

import projections.geotools as geotools
import projections.utils as utils

BINS = 52

def sum_layers(ds, idx, layers, out):
  out.fill(0)
  for l in layers:
    out += ds.variables[l][idx]
  return

def find_frac(values, tot, frac):
  frac.fill(1)
  ## FIXME: need np.clip() because pos - neg < 0
  frac -= np.clip(ma.where(values[-1] > 0, tot / values[-1], 0), 0, 1)
  #frac -= np.clip(tot / values[-1], 0, 1)
  return

def dorem(values, remove, frac):
  find_frac(values, remove, frac)
  values[1:-1] *= np.broadcast_to(frac, values[1:-1].shape)
  return

def to_year(scenario, idx):
  if scenario == 'historical':
    return idx + 850
  return idx + 2015

def asserts(state, idx, name, values, atol):
  assert np.all(values[0] <= 1.0 + atol), 'current > 1'
  assert np.all(values[0] >= 0.0 - atol), 'current < 0'
  secd = state.variables[name][idx]
  assert np.allclose(values[-1] - secd, 0, atol=atol)

def write_data(out, fnf, idx, values):
  out.variables['secdy%s' % fnf][idx, :, :] = values[0:30].sum(axis=0)
  out.variables['secdi%s' % fnf][idx, :, :] = values[30:50].sum(axis=0)
  out.variables['secdm%s' % fnf][idx, :, :] = values[50]

def write_bins(out, vname, values):
  # FIXME: verify masked values are written and read correctly.
  out.variables[vname][:] = values
  return values

def init_values(state, vname, start_index, mask):
  shape = state.variables[vname].shape
  dtype = state.variables[vname].dtype
  values = ma.zeros((BINS, shape[1], shape[2]),
                    dtype = state.variables[vname].dtype,
                    fill_value=-9999)
  values.mask = np.broadcast_to(mask == 1.0, values.shape)
  values[-1] = state.variables[vname][start_index]
  values[-2] = state.variables[vname][start_index]
  return values

def roll_values(values):
  values[-3] += values[-2]
  values[-2] = values[0:-2].sum(axis=0)
  return np.roll(values, 1, 0)

def neg_re(fnf):
  return r'secd{fnf}_to_'.format(fnf=fnf)

def pos_re(fnf):
  return r'^(?!secd{fnf}).*_to_secd{fnf}$|prim{fnf}_harv$'.format(fnf=fnf)

@click.command()
@click.option('--scenario', type=click.Choice(utils.luh2_scenarios() +
                                              ('all', )),
              default='all',
              help='Which LUH2 scenario to run (default: all)')
@click.option('--outdir', type=click.Path(file_okay=False),
              default='/out/luh2',
              help='Output directory (default: /out/luh2)')
@click.option('--start-index', type=int, default=0,
              help='Start from given index skipping earlier years (default: 0)')
def doit(scenario, outdir, start_index=0):
  static = Dataset(os.path.join(utils.luh2_dir(), 'staticData_quarterdeg.nc'))
  icwtr = static.variables['icwtr'][:, :]
  fstnf = static.variables['fstnf'][:, :]
  atol = 5e-5

  variables = tuple([(x % fnf, 'f4', '1', -9999, 'time')
                     for fnf in ('f', 'n')
                     for x in ('secd%s%%s' % n for n in ('y', 'i', 'm'))] +
                    [('bins%s' % fnf, 'f4', '1', -9999, 'bins')
                     for fnf in ('f', 'n')])
  baselinef = None
  baselinen = None

  if scenario == 'all':
    # historical must be the first scenario processed
    scenarios = sorted(utils.luh2_scenarios())
  else:
    scenarios = [scenario]

  for scenario in scenarios:
    oname = os.path.join(outdir, 'secd-%s.nc' % scenario)
    tname = utils.luh2_transitions(scenario)
    sname = utils.luh2_states(scenario)
    if not (os.path.isfile(tname) and os.path.isfile(sname)):
      click.echo("skipping %s" % scenario)
      continue
    click.echo('%s -> %s' % (scenario, oname))

    with Dataset(oname, 'w') as out:
      click.echo(sname)
      click.echo(tname)
      with Dataset(tname) as trans:
        with Dataset(sname) as state:
          data = init_nc(out, state, variables)
          if scenario == 'historical':
            # Create a 3-D array to hold the last 50 years (plus 2)
            valuesf = init_values(state, 'secdf', start_index, icwtr)
            valuesn = init_values(state, 'secdn', start_index, icwtr)
          elif baselinef is None or baselinen is None:
            with Dataset(os.path.join(outdir, 'secd-historical.nc')) as hist:
              valuesf = hist.variables['binsf'][:]
              valuesn = hist.variables['binsn'][:]
          else:
            valuesf = baselinef.copy()
            valuesn = baselinen.copy()

          # Write initial data to output.
          valuesf[0].fill(0)
          valuesn[0].fill(0)
          write_data(out, 'f', start_index, valuesf)
          write_data(out, 'n', start_index, valuesn)

          remove = ma.empty_like(valuesf[0])
          frac = ma.empty_like(valuesf[0])
          posf = tuple(filter(lambda x: re.match(pos_re('f'), x),
                              trans.variables.keys()))
          posn = tuple(filter(lambda x: re.match(pos_re('n'), x),
                              trans.variables.keys()))
          negf = tuple(filter(lambda x: re.match(neg_re('f'), x),
                              trans.variables.keys()))
          negn = tuple(filter(lambda x: re.match(neg_re('n'), x),
                              trans.variables.keys()))
          click.echo("  " + ', '.join(posf))
          click.echo("  " + ', '.join(negf))
          click.echo("  " + ', '.join(posn))
          click.echo("  " + ', '.join(negn))
          for idx in range(start_index, trans.variables['time'].shape[0]):
            click.echo("  year %d" % to_year(scenario, idx))
            # Compute transitions from / to secondary.
            sum_layers(trans, idx, negf, remove)
            sum_layers(trans, idx, posf, valuesf[0])
            # Adjust secondary history
            dorem(valuesf, remove, frac)

            # Repeat for non-forested
            sum_layers(trans, idx, negn, remove)
            sum_layers(trans, idx, posn, valuesn[0])
            dorem(valuesn, remove, frac)

            # Check consistency of data.
            asserts(state, idx, 'secdf', valuesf, atol)
            asserts(state, idx, 'secdn', valuesn, atol)

            # Write data to output.
            write_data(out, 'f', idx + 1, valuesf)
            write_data(out, 'n', idx + 1, valuesn)

            # Rotate the array.
            valuesf = roll_values(valuesf)
            valuesn = roll_values(valuesn)

      if scenario == 'historical':
        baselinef = write_bins(out, 'binsf', valuesf).copy()
        baselinen = write_bins(out, 'binsn', valuesn).copy()
        start_index = 0

def id(x):
  # This function returns the memory
  # block address of an array.
  return x.__array_interface__['data'][0]

def init_nc(dst_ds, src_ds, variables):
  # Set attributes
  dst_ds.setncattr('Conventions', u'CF-1.5')
  dst_ds.setncattr('GDAL', u'GDAL 1.11.3, released 2015/09/16')

  # Create dimensions
  dst_ds.createDimension('time', None)
  dst_ds.createDimension('lat', len(src_ds.variables['lat']))
  dst_ds.createDimension('lon', len(src_ds.variables['lon']))
  dst_ds.createDimension('bins', BINS)

  # Create variables
  times = dst_ds.createVariable("time", "f8", ("time"), zlib=True,
                                least_significant_digit=3)
  latitudes = dst_ds.createVariable("lat", "f4", ("lat"), zlib=True,
                                    least_significant_digit = 3)
  longitudes = dst_ds.createVariable("lon", "f4", ("lon"), zlib=True,
                                     least_significant_digit=3)
  crs = dst_ds.createVariable('crs', "S1", ())

  # Add metadata
  dst_ds.history = "Created at " + time.ctime(time.time())
  dst_ds.source = "secd-dist.py"
  latitudes.units = "degrees_north"
  latitudes.long_name = 'latitude'
  longitudes.units = "degrees_east"
  longitudes.long_name = "longitude"
  times.units = "years since 850-01-01 00:00:00.0"
  times.calendar = "gregorian"
  times.standard_name = "time"
  times.axis = 'T'

  # Assign data to variables
  latitudes[:] = src_ds.variables['lat'][:]
  longitudes[:] = src_ds.variables['lon'][:]
  times[:] = src_ds.variables['time'][:]

  srs = osr.SpatialReference()
  srs.ImportFromWkt(geotools.WGS84_WKT)
  src_trans = (-180.0, 0.25, 0.0, 90.0, 0.0, -0.25)
  crs.grid_mapping_name = 'latitude_longitude'
  crs.spatial_ref = srs.ExportToWkt()
  crs.GetTransform = ' '.join(tuple(map(str, src_trans)))
  # FIXME: Attribute getters don't work in python3 or GDAL2
  crs.longitude_of_prime_meridian = geotools.srs_get_prime_meridian(srs)
  crs.semi_major_axis = geotools.srs_get_semi_major(srs)
  crs.inverse_flattening = geotools.srs_get_inv_flattening(srs)

  out = {}
  for name, dtype, units, fill, dimension in variables:
    dst_data = dst_ds.createVariable(name, dtype,
                                     (dimension, "lat","lon"), zlib = True,
                                     least_significant_digit = 4,
                                     fill_value = fill)
    dst_data.units = units
    dst_data.grid_mapping = 'crs'
    out[name] = dst_data
  return out

if __name__ == '__main__':
  doit()
  click.echo('done')

<commit_msg>Cleanup pylint errors<commit_after>#!/usr/bin/env python3

import os
import re
import time

import click
from netCDF4 import Dataset
import numpy as np
import numpy.ma as ma
import osr

import projections.geotools as geotools
import projections.utils as utils

BINS = 52

def sum_layers(ds, idx, layers, out):
  out.fill(0)
  for l in layers:
    out += ds.variables[l][idx]
  return

def find_frac(values, tot, frac):
  frac.fill(1)
  ## FIXME: need np.clip() because pos - neg < 0
  frac -= np.clip(ma.where(values[-1] > 0, tot / values[-1], 0), 0, 1)
  #frac -= np.clip(tot / values[-1], 0, 1)
  return

def dorem(values, remove, frac):
  find_frac(values, remove, frac)
  values[1:-1] *= np.broadcast_to(frac, values[1:-1].shape)
  return

def to_year(scenario, idx):
  if scenario == 'historical':
    return idx + 850
  return idx + 2015

def asserts(state, idx, name, values, atol):
  assert np.all(values[0] <= 1.0 + atol), 'current > 1'
  assert np.all(values[0] >= 0.0 - atol), 'current < 0'
  secd = state.variables[name][idx]
  assert np.allclose(values[-1] - secd, 0, atol=atol)

def write_data(out, fnf, idx, values):
  out.variables['secdy%s' % fnf][idx, :, :] = values[0:30].sum(axis=0)
  out.variables['secdi%s' % fnf][idx, :, :] = values[30:50].sum(axis=0)
  out.variables['secdm%s' % fnf][idx, :, :] = values[50]

def write_bins(out, vname, values):
  # FIXME: verify masked values are written and read correctly.
  out.variables[vname][:] = values
  return values

def init_values(state, vname, start_index, mask):
  shape = state.variables[vname].shape
  values = ma.zeros((BINS, shape[1], shape[2]),
                    dtype=state.variables[vname].dtype,
                    fill_value=-9999)
  values.mask = np.broadcast_to(mask == 1.0, values.shape)
  values[-1] = state.variables[vname][start_index]
  values[-2] = state.variables[vname][start_index]
  return values

def roll_values(values):
  values[-3] += values[-2]
  values[-2] = values[0:-2].sum(axis=0)
  return np.roll(values, 1, 0)

def neg_re(fnf):
  return r'secd{fnf}_to_'.format(fnf=fnf)

def pos_re(fnf):
  return r'^(?!secd{fnf}).*_to_secd{fnf}$|prim{fnf}_harv$'.format(fnf=fnf)

@click.command()
@click.option('--scenario', type=click.Choice(utils.luh2_scenarios() +
                                              ('all', )),
              default='all',
              help='Which LUH2 scenario to run (default: all)')
@click.option('--outdir', type=click.Path(file_okay=False),
              default='/out/luh2',
              help='Output directory (default: /out/luh2)')
@click.option('--start-index', type=int, default=0,
              help='Start from given index skipping earlier years (default: 0)')
def doit(scenario, outdir, start_index=0):
  static = Dataset(os.path.join(utils.luh2_dir(), 'staticData_quarterdeg.nc'))
  icwtr = static.variables['icwtr'][:, :]
  atol = 5e-5

  variables = tuple([(x % fnf, 'f4', '1', -9999, 'time')
                     for fnf in ('f', 'n')
                     for x in ('secd%s%%s' % n for n in ('y', 'i', 'm'))] +
                    [('bins%s' % fnf, 'f4', '1', -9999, 'bins')
                     for fnf in ('f', 'n')])
  baselinef = None
  baselinen = None

  if scenario == 'all':
    # historical must be the first scenario processed
    scenarios = sorted(utils.luh2_scenarios())
  else:
    scenarios = [scenario]

  for scenario in scenarios:
    oname = os.path.join(outdir, 'secd-%s.nc' % scenario)
    tname = utils.luh2_transitions(scenario)
    sname = utils.luh2_states(scenario)
    if not (os.path.isfile(tname) and os.path.isfile(sname)):
      click.echo("skipping %s" % scenario)
      continue
    click.echo('%s -> %s' % (scenario, oname))

    with Dataset(oname, 'w') as out:
      click.echo(sname)
      click.echo(tname)
      with Dataset(tname) as trans:
        with Dataset(sname) as state:
          _ = init_nc(out, state, variables)
          if scenario == 'historical':
            # Create a 3-D array to hold the last 50 years (plus 2)
            valuesf = init_values(state, 'secdf', start_index, icwtr)
            valuesn = init_values(state, 'secdn', start_index, icwtr)
          elif baselinef is None or baselinen is None:
            with Dataset(os.path.join(outdir, 'secd-historical.nc')) as hist:
              valuesf = hist.variables['binsf'][:]
              valuesn = hist.variables['binsn'][:]
          else:
            valuesf = baselinef.copy()
            valuesn = baselinen.copy()

          # Write initial data to output.
          valuesf[0].fill(0)
          valuesn[0].fill(0)
          write_data(out, 'f', start_index, valuesf)
          write_data(out, 'n', start_index, valuesn)

          remove = ma.empty_like(valuesf[0])
          frac = ma.empty_like(valuesf[0])
          posf = tuple(filter(lambda x: re.match(pos_re('f'), x),
                              trans.variables.keys()))
          posn = tuple(filter(lambda x: re.match(pos_re('n'), x),
                              trans.variables.keys()))
          negf = tuple(filter(lambda x: re.match(neg_re('f'), x),
                              trans.variables.keys()))
          negn = tuple(filter(lambda x: re.match(neg_re('n'), x),
                              trans.variables.keys()))
          click.echo("  " + ', '.join(posf))
          click.echo("  " + ', '.join(negf))
          click.echo("  " + ', '.join(posn))
          click.echo("  " + ', '.join(negn))
          for idx in range(start_index, trans.variables['time'].shape[0]):
            click.echo("  year %d" % to_year(scenario, idx))
            # Compute transitions from / to secondary.
            sum_layers(trans, idx, negf, remove)
            sum_layers(trans, idx, posf, valuesf[0])
            # Adjust secondary history
            dorem(valuesf, remove, frac)

            # Repeat for non-forested
            sum_layers(trans, idx, negn, remove)
            sum_layers(trans, idx, posn, valuesn[0])
            dorem(valuesn, remove, frac)

            # Check consistency of data.
            asserts(state, idx, 'secdf', valuesf, atol)
            asserts(state, idx, 'secdn', valuesn, atol)

            # Write data to output.
            write_data(out, 'f', idx + 1, valuesf)
            write_data(out, 'n', idx + 1, valuesn)

            # Rotate the array.
            valuesf = roll_values(valuesf)
            valuesn = roll_values(valuesn)

      if scenario == 'historical':
        baselinef = write_bins(out, 'binsf', valuesf).copy()
        baselinen = write_bins(out, 'binsn', valuesn).copy()
        start_index = 0

def init_nc(dst_ds, src_ds, variables):
  # Set attributes
  dst_ds.setncattr('Conventions', u'CF-1.5')
  dst_ds.setncattr('GDAL', u'GDAL 1.11.3, released 2015/09/16')

  # Create dimensions
  dst_ds.createDimension('time', None)
  dst_ds.createDimension('lat', len(src_ds.variables['lat']))
  dst_ds.createDimension('lon', len(src_ds.variables['lon']))
  dst_ds.createDimension('bins', BINS)

  # Create variables
  times = dst_ds.createVariable("time", "f8", ("time"), zlib=True,
                                least_significant_digit=3)
  latitudes = dst_ds.createVariable("lat", "f4", ("lat"), zlib=True,
                                    least_significant_digit=3)
  longitudes = dst_ds.createVariable("lon", "f4", ("lon"), zlib=True,
                                     least_significant_digit=3)
  crs = dst_ds.createVariable('crs', "S1", ())

  # Add metadata
  dst_ds.history = "Created at " + time.ctime(time.time())
  dst_ds.source = "secd-dist.py"
  latitudes.units = "degrees_north"
  latitudes.long_name = 'latitude'
  longitudes.units = "degrees_east"
  longitudes.long_name = "longitude"
  times.units = "years since 850-01-01 00:00:00.0"
  times.calendar = "gregorian"
  times.standard_name = "time"
  times.axis = 'T'

  # Assign data to variables
  latitudes[:] = src_ds.variables['lat'][:]
  longitudes[:] = src_ds.variables['lon'][:]
  times[:] = src_ds.variables['time'][:]

  srs = osr.SpatialReference()
  srs.ImportFromWkt(geotools.WGS84_WKT)
  src_trans = (-180.0, 0.25, 0.0, 90.0, 0.0, -0.25)
  crs.grid_mapping_name = 'latitude_longitude'
  crs.spatial_ref = srs.ExportToWkt()
  crs.GetTransform = ' '.join(tuple(map(str, src_trans)))
  # FIXME: Attribute getters don't work in python3 or GDAL2
  crs.longitude_of_prime_meridian = geotools.srs_get_prime_meridian(srs)
  crs.semi_major_axis = geotools.srs_get_semi_major(srs)
  crs.inverse_flattening = geotools.srs_get_inv_flattening(srs)

  out = {}
  for name, dtype, units, fill, dimension in variables:
    dst_data = dst_ds.createVariable(name, dtype,
                                     (dimension, "lat", "lon"), zlib=True,
                                     least_significant_digit=4,
                                     fill_value=fill)
    dst_data.units = units
    dst_data.grid_mapping = 'crs'
    out[name] = dst_data
  return out

if __name__ == '__main__':
#pylint: disable-msg=no-value-for-parameter
  doit()
#pylint: enable-msg=no-value-for-parameter
  click.echo('done')
<|endoftext|>


<commit_before>from . import about
from . import ev
from . import gs
from collections import namedtuple
import os
import re
import string
import subprocess
import time

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

Proc = namedtuple('Proc', 'p input orig_cmd cmd_lst env wd ok exc')
Result = namedtuple('Result', 'out cmd_lst err ok exc')

class _command(object):
	def __init__(self):
		self.input = None
		self.stdin = subprocess.PIPE
		self.stdout = subprocess.PIPE
		self.stderr = subprocess.PIPE
		self.startupinfo = STARTUP_INFO
		self.wd = None
		self.env = {}

	def proc(self):
		if self.wd:
			wd = self.wd
			try:
				os.makedirs(wd)
			except Exception:
				pass
		else:
			wd = None

		if self.input:
			input = gs.astr(self.input)
		else:
			input = None

		try:
			setsid = os.setsid
		except Exception:
			setsid = None

		out = ''
		err = ''
		exc = None

		nv = env(self.env)
		cmd_lst = self.cmd(nv)
		orig_cmd = cmd_lst[0]
		cmd_lst[0] = _which(orig_cmd, nv.get('PATH'))

		try:
			if not cmd_lst[0]:
				raise Exception('Cannot find command `%s`' % orig_cmd)

			p = subprocess.Popen(
				cmd_lst,
				stdout=self.stdout,
				stderr=self.stderr,
				stdin=self.stdin,
				startupinfo=gs.STARTUP_INFO,
				shell=False,
				env=nv,
				cwd=wd,
				preexec_fn=setsid,
				bufsize=0
			)
		except Exception as e:
			exc = e
			p = None

		return Proc(
			p=p,
			input=input,
			orig_cmd=orig_cmd,
			cmd_lst=cmd_lst,
			env=nv,
			wd=wd,
			ok=(not exc),
			exc=exc
		)

	def run(self):
		out = ''
		err = ''
		exc = None

		pr = self.proc()

		ev.debug('sh.run', pr)

		try:
			out, err = pr.p.communicate(input=pr.input)
		except Exception as e:
			exc = e

		return Result(
			out=gs.ustr(out),
			err=gs.ustr(err),
			cmd_lst=pr.cmd_lst,
			ok=(not exc),
			exc=exc
		)

class ShellCommand(_command):
	def __init__(self, cmd_str):
		_command.__init__(self)
		self.cmd_str = gs.astr(cmd_str)

	def cmd(self, e):
		return _cmd(self.cmd_str, e)

class Command(_command):
	def __init__(self, cmd_lst):
		_command.__init__(self)
		self.cmd_lst = [gs.astr(s) for s in cmd_lst]

	def cmd(self, e):
		return self.cmd_lst

def shl(m={}):
	return _shl(env(m))

def _shl(e):
	l = gs.setting('shell', [])
	if not l:
		fn = e.get('SHELL') or e.get('COMSPEC')
		if fn:
			name, _ = os.path.splitext(os.path.basename(fn))
			f = globals().get('_shl_%s' % name)
			if f:
				l = f(fn)

	if not l:
		if gs.os_is_windows():
			l = _shl_cmd('cmd')
		else:
			l = _shl_sh('sh')

	return l

def _shl_cmd(fn):
	return [fn, '/C', '${CMD}']

def _shl_sh(fn):
	return [fn, '-l', '-c', '${CMD}']

_shl_fish = _shl_sh
_shl_bash = _shl_sh
_shl_zsh = _shl_sh
_shl_rc = _shl_sh

def cmd(cmd_str, m={}):
	return _cmd(cmd_str, env(m))

def _cmd(cmd_str, e):
	cmdm = {'CMD': cmd_str}
	cmdl = []
	for s in _shl(e):
		s = string.Template(s).safe_substitute(cmdm)
		s = gs.astr(s)
		if s:
			cmdl.append(s)

	return cmdl

def init():
	global _env_ext
	global GO_VERSION

	start = time.time()

	vars = [
		'PATH',
		'GOARCH',
		'GOBIN',
		'GOCHAR',
		'GOEXE',
		'GOGCCFLAGS',
		'GOHOSTARCH',
		'GOHOSTOS',
		'GOOS',
		'GOPATH',
		'GOROOT',
		'GOTOOLDIR',
		'CGO_ENABLED',
	]

	cmdl = []
	for k in vars:
		cmdl.append('[[[$'+k+']]'+k+'[[%'+k+'%]]]')
	cmd_str = 'echo "%s"' % ' '.join(cmdl)

	cr = ShellCommand(cmd_str).run()
	if cr.exc:
		_print('error loading env vars: %s' % cr.exc)

	out = cr.out + cr.err

	mats = re.findall(r'\[\[\[(.*?)\]\](%s)\[\[(.*?)\]\]\]' % '|'.join(vars), out)
	for m in mats:
		a, k, b = m
		v = ''
		if a != '$'+k:
			v = a
		elif b != '%'+k+'%':
			v = b

		if v:
			_env_ext[k] = v

	cr_go = ShellCommand('go version').run()
	m = about.GO_VERSION_OUTPUT_PAT.search(cr_go.out + cr_go.err)
	if m:
		GO_VERSION = '-'.join(s for s in m.groups() if s)

	dur = (time.time() - start)

	ev.debug('sh.init', {
		'cr.init': cr,
		'cr.go': cr_go,
		'go_version': GO_VERSION,
		'env': _env_ext,
		'dur': dur,
	})

	cmd_lst = []
	for v in cr.cmd_lst:
		if v == cmd_str:
			cmd_lst.append('echo "..."')
		else:
			cmd_lst.append(v)

	_print('load env vars: (%s): %0.3fs' % (cmd_lst, dur))

def _print(s):
	print('GoSblime %s sh: %s' % (about.VERSION, s))

def _shell_pathsep():
	return gs.setting('shell_pathsep') or os.pathsep

def _sj_path(p):
	return _shell_pathsep().join(p.split(os.pathsep))

def getenv(name, default='', m={}):
	return env(m).get(name, default)

def env(m={}):
	"""
	Assemble environment information needed for correct operation. In particular,
	ensure that directories containing binaries are included in PATH.
	"""
	e = os.environ.copy()

	# the system's env may be compatible with the shell
	# so try to fix the env vars that depend on shell_pathsep
	e['PATH'] = _sj_path(e.get('PATH', ''))
	e['GOPATH'] = _sj_path(e.get('GOPATH', ''))

	e.update(_env_ext)
	e.update(m)

	roots = gs.lst(e.get('GOPATH', '').split(os.pathsep), e.get('GOROOT', ''))
	lfn = gs.attr('last_active_go_fn', '')

	comps = lfn.split(os.sep)
	gs_gopath = []
	for i, s in enumerate(comps):
		if s.lower() == "src":
			p = os.sep.join(comps[:i])
			if p not in roots:
				gs_gopath.append(p)
	gs_gopath.reverse()
	e['GS_GOPATH'] = os.pathsep.join(gs_gopath)

	uenv = gs.setting('env', {})
	for k in uenv:
		try:
			uenv[k] = string.Template(uenv[k]).safe_substitute(e)
		except Exception as ex:
			gs.println('%s: Cannot expand env var `%s`: %s' % (NAME, k, ex))

	e.update(uenv)
	e.update(m)

	# For custom values of GOPATH, installed binaries via go install
	# will go into the "bin" dir of the corresponding GOPATH path.
	# Therefore, make sure these paths are included in PATH.

	add_path = [gs.home_dir_path('bin')]

	for s in gs.lst(e.get('GOROOT', ''), e.get('GOPATH', '').split(os.pathsep)):
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)

	gobin = e.get('GOBIN', '')
	if gobin and gobin not in add_path:
		add_path.append(gobin)

	if gs.os_is_windows():
		l = [
			'~\\bin',
			'~\\go\\bin',
			'C:\\Go\\bin',
		]
	else:
		l = [
			'~/bin',
			'~/go/bin',
			'/usr/local/go/bin',
			'/usr/local/opt/go/bin',
			'/usr/local/bin',
			'/usr/bin',
		]

	for s in l:
		s = os.path.expanduser(s)
		if s not in add_path:
			add_path.append(s)

	psep = _shell_pathsep()

	for s in e.get('PATH', '').split(psep):
		if s and s not in add_path:
			add_path.append(s)

	e['PATH'] = psep.join(add_path)

	# Ensure no unicode objects leak through. The reason is twofold:
	# 	* On Windows, Python 2.6 (used by Sublime Text) subprocess.Popen
	# 	  can only take bytestrings as environment variables in the
	#	  "env"	parameter. Reference:
	# 	  https://github.com/DisposaBoy/GoSublime/issues/112
	# 	  http://stackoverflow.com/q/12253014/1670
	#   * Avoids issues with networking too.
	clean_env = {}
	for k, v in e.items():
		try:
			clean_env[gs.astr(k)] = gs.astr(v)
		except Exception as ex:
			gs.println('%s: Bad env: %s' % (NAME, ex))

	return clean_env

def which_ok(fn):
	try:
		return os.path.isfile(fn) and os.access(fn, os.X_OK)
	except Exception:
		return False

def which(cmd):
	return _which(cmd, getenv('PATH', ''))

def _which(cmd, env_path):
	if os.path.isabs(cmd):
		return cmd if which_ok(cmd) else ''

	# not supporting PATHEXT. period.
	if gs.os_is_windows() and not cmd.endswith('.exe'):
		cmd = '%s.exe' % cmd

	seen = {}
	for p in env_path.split(_shell_pathsep()):
		p = os.path.join(p, cmd)
		if p not in seen and which_ok(p):
			return p

		seen[p] = True

	return ''

def go(subcmd_str):
	cr = ShellCommand('go '+subcmd_str).run()
	out = cr.out.strip() + '\n' + cr.err.strip()
	return out.strip()

GO_VERSION = about.DEFAULT_GO_VERSION
_env_ext = {}
init()
<commit_msg>* use the correct STARTUP_INFO<commit_after>from . import about
from . import ev
from . import gs
from collections import namedtuple
import os
import re
import string
import subprocess
import time

try:
	STARTUP_INFO = subprocess.STARTUPINFO()
	STARTUP_INFO.dwFlags |= subprocess.STARTF_USESHOWWINDOW
	STARTUP_INFO.wShowWindow = subprocess.SW_HIDE
except (AttributeError):
	STARTUP_INFO = None

Proc = namedtuple('Proc', 'p input orig_cmd cmd_lst env wd ok exc')
Result = namedtuple('Result', 'out cmd_lst err ok exc')

class _command(object):
	def __init__(self):
		self.input = None
		self.stdin = subprocess.PIPE
		self.stdout = subprocess.PIPE
		self.stderr = subprocess.PIPE
		self.startupinfo = STARTUP_INFO
		self.wd = None
		self.env = {}

	def proc(self):
		if self.wd:
			wd = self.wd
			try:
				os.makedirs(wd)
			except Exception:
				pass
		else:
			wd = None

		if self.input:
			input = gs.astr(self.input)
		else:
			input = None

		try:
			setsid = os.setsid
		except Exception:
			setsid = None

		out = ''
		err = ''
		exc = None

		nv = env(self.env)
		cmd_lst = self.cmd(nv)
		orig_cmd = cmd_lst[0]
		cmd_lst[0] = _which(orig_cmd, nv.get('PATH'))

		try:
			if not cmd_lst[0]:
				raise Exception('Cannot find command `%s`' % orig_cmd)

			p = subprocess.Popen(
				cmd_lst,
				stdout=self.stdout,
				stderr=self.stderr,
				stdin=self.stdin,
				startupinfo=self.STARTUP_INFO,
				shell=False,
				env=nv,
				cwd=wd,
				preexec_fn=setsid,
				bufsize=0
			)
		except Exception as e:
			exc = e
			p = None

		return Proc(
			p=p,
			input=input,
			orig_cmd=orig_cmd,
			cmd_lst=cmd_lst,
			env=nv,
			wd=wd,
			ok=(not exc),
			exc=exc
		)

	def run(self):
		out = ''
		err = ''
		exc = None

		pr = self.proc()

		ev.debug('sh.run', pr)

		try:
			out, err = pr.p.communicate(input=pr.input)
		except Exception as e:
			exc = e

		return Result(
			out=gs.ustr(out),
			err=gs.ustr(err),
			cmd_lst=pr.cmd_lst,
			ok=(not exc),
			exc=exc
		)

class ShellCommand(_command):
	def __init__(self, cmd_str):
		_command.__init__(self)
		self.cmd_str = gs.astr(cmd_str)

	def cmd(self, e):
		return _cmd(self.cmd_str, e)

class Command(_command):
	def __init__(self, cmd_lst):
		_command.__init__(self)
		self.cmd_lst = [gs.astr(s) for s in cmd_lst]

	def cmd(self, e):
		return self.cmd_lst

def shl(m={}):
	return _shl(env(m))

def _shl(e):
	l = gs.setting('shell', [])
	if not l:
		fn = e.get('SHELL') or e.get('COMSPEC')
		if fn:
			name, _ = os.path.splitext(os.path.basename(fn))
			f = globals().get('_shl_%s' % name)
			if f:
				l = f(fn)

	if not l:
		if gs.os_is_windows():
			l = _shl_cmd('cmd')
		else:
			l = _shl_sh('sh')

	return l

def _shl_cmd(fn):
	return [fn, '/C', '${CMD}']

def _shl_sh(fn):
	return [fn, '-l', '-c', '${CMD}']

_shl_fish = _shl_sh
_shl_bash = _shl_sh
_shl_zsh = _shl_sh
_shl_rc = _shl_sh

def cmd(cmd_str, m={}):
	return _cmd(cmd_str, env(m))

def _cmd(cmd_str, e):
	cmdm = {'CMD': cmd_str}
	cmdl = []
	for s in _shl(e):
		s = string.Template(s).safe_substitute(cmdm)
		s = gs.astr(s)
		if s:
			cmdl.append(s)

	return cmdl

def init():
	global _env_ext
	global GO_VERSION

	start = time.time()

	vars = [
		'PATH',
		'GOARCH',
		'GOBIN',
		'GOCHAR',
		'GOEXE',
		'GOGCCFLAGS',
		'GOHOSTARCH',
		'GOHOSTOS',
		'GOOS',
		'GOPATH',
		'GOROOT',
		'GOTOOLDIR',
		'CGO_ENABLED',
	]

	cmdl = []
	for k in vars:
		cmdl.append('[[[$'+k+']]'+k+'[[%'+k+'%]]]')
	cmd_str = 'echo "%s"' % ' '.join(cmdl)

	cr = ShellCommand(cmd_str).run()
	if cr.exc:
		_print('error loading env vars: %s' % cr.exc)

	out = cr.out + cr.err

	mats = re.findall(r'\[\[\[(.*?)\]\](%s)\[\[(.*?)\]\]\]' % '|'.join(vars), out)
	for m in mats:
		a, k, b = m
		v = ''
		if a != '$'+k:
			v = a
		elif b != '%'+k+'%':
			v = b

		if v:
			_env_ext[k] = v

	cr_go = ShellCommand('go version').run()
	m = about.GO_VERSION_OUTPUT_PAT.search(cr_go.out + cr_go.err)
	if m:
		GO_VERSION = '-'.join(s for s in m.groups() if s)

	dur = (time.time() - start)

	ev.debug('sh.init', {
		'cr.init': cr,
		'cr.go': cr_go,
		'go_version': GO_VERSION,
		'env': _env_ext,
		'dur': dur,
	})

	cmd_lst = []
	for v in cr.cmd_lst:
		if v == cmd_str:
			cmd_lst.append('echo "..."')
		else:
			cmd_lst.append(v)

	_print('load env vars: (%s): %0.3fs' % (cmd_lst, dur))

def _print(s):
	print('GoSblime %s sh: %s' % (about.VERSION, s))

def _shell_pathsep():
	return gs.setting('shell_pathsep') or os.pathsep

def _sj_path(p):
	return _shell_pathsep().join(p.split(os.pathsep))

def getenv(name, default='', m={}):
	return env(m).get(name, default)

def env(m={}):
	"""
	Assemble environment information needed for correct operation. In particular,
	ensure that directories containing binaries are included in PATH.
	"""
	e = os.environ.copy()

	# the system's env may be compatible with the shell
	# so try to fix the env vars that depend on shell_pathsep
	e['PATH'] = _sj_path(e.get('PATH', ''))
	e['GOPATH'] = _sj_path(e.get('GOPATH', ''))

	e.update(_env_ext)
	e.update(m)

	roots = gs.lst(e.get('GOPATH', '').split(os.pathsep), e.get('GOROOT', ''))
	lfn = gs.attr('last_active_go_fn', '')

	comps = lfn.split(os.sep)
	gs_gopath = []
	for i, s in enumerate(comps):
		if s.lower() == "src":
			p = os.sep.join(comps[:i])
			if p not in roots:
				gs_gopath.append(p)
	gs_gopath.reverse()
	e['GS_GOPATH'] = os.pathsep.join(gs_gopath)

	uenv = gs.setting('env', {})
	for k in uenv:
		try:
			uenv[k] = string.Template(uenv[k]).safe_substitute(e)
		except Exception as ex:
			gs.println('%s: Cannot expand env var `%s`: %s' % (NAME, k, ex))

	e.update(uenv)
	e.update(m)

	# For custom values of GOPATH, installed binaries via go install
	# will go into the "bin" dir of the corresponding GOPATH path.
	# Therefore, make sure these paths are included in PATH.

	add_path = [gs.home_dir_path('bin')]

	for s in gs.lst(e.get('GOROOT', ''), e.get('GOPATH', '').split(os.pathsep)):
		if s:
			s = os.path.join(s, 'bin')
			if s not in add_path:
				add_path.append(s)

	gobin = e.get('GOBIN', '')
	if gobin and gobin not in add_path:
		add_path.append(gobin)

	if gs.os_is_windows():
		l = [
			'~\\bin',
			'~\\go\\bin',
			'C:\\Go\\bin',
		]
	else:
		l = [
			'~/bin',
			'~/go/bin',
			'/usr/local/go/bin',
			'/usr/local/opt/go/bin',
			'/usr/local/bin',
			'/usr/bin',
		]

	for s in l:
		s = os.path.expanduser(s)
		if s not in add_path:
			add_path.append(s)

	psep = _shell_pathsep()

	for s in e.get('PATH', '').split(psep):
		if s and s not in add_path:
			add_path.append(s)

	e['PATH'] = psep.join(add_path)

	# Ensure no unicode objects leak through. The reason is twofold:
	# 	* On Windows, Python 2.6 (used by Sublime Text) subprocess.Popen
	# 	  can only take bytestrings as environment variables in the
	#	  "env"	parameter. Reference:
	# 	  https://github.com/DisposaBoy/GoSublime/issues/112
	# 	  http://stackoverflow.com/q/12253014/1670
	#   * Avoids issues with networking too.
	clean_env = {}
	for k, v in e.items():
		try:
			clean_env[gs.astr(k)] = gs.astr(v)
		except Exception as ex:
			gs.println('%s: Bad env: %s' % (NAME, ex))

	return clean_env

def which_ok(fn):
	try:
		return os.path.isfile(fn) and os.access(fn, os.X_OK)
	except Exception:
		return False

def which(cmd):
	return _which(cmd, getenv('PATH', ''))

def _which(cmd, env_path):
	if os.path.isabs(cmd):
		return cmd if which_ok(cmd) else ''

	# not supporting PATHEXT. period.
	if gs.os_is_windows() and not cmd.endswith('.exe'):
		cmd = '%s.exe' % cmd

	seen = {}
	for p in env_path.split(_shell_pathsep()):
		p = os.path.join(p, cmd)
		if p not in seen and which_ok(p):
			return p

		seen[p] = True

	return ''

def go(subcmd_str):
	cr = ShellCommand('go '+subcmd_str).run()
	out = cr.out.strip() + '\n' + cr.err.strip()
	return out.strip()

GO_VERSION = about.DEFAULT_GO_VERSION
_env_ext = {}
init()
<|endoftext|>


<commit_before>"""Contains the base classes for drop targets and drop target banks."""

from mpf.core.delays import DelayManager
from mpf.core.device_monitor import DeviceMonitor
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

@DeviceMonitor("complete")
class DropTarget(SystemWideDevice):

    """Represents a single drop target in a pinball machine.

    Args: Same as the `Target` parent class
    """

    config_section = 'drop_targets'
    collection = 'drop_targets'
    class_label = 'drop_target'

    def __init__(self, machine, name):
        """Initialise drop target."""
        self.reset_coil = None
        self.knockdown_coil = None
        self.banks = None
        super().__init__(machine, name)

        self._in_ball_search = False
        self.complete = False
        self.delay = DelayManager(machine.delayRegistry)

        self._ignore_switch_hits = False

    def _initialize(self):
        self.reset_coil = self.config['reset_coil']
        self.knockdown_coil = self.config['knockdown_coil']
        self.banks = set()

        # can't read the switch until the switch controller is set up
        self.machine.events.add_handler('init_phase_4',
                                        self._update_state_from_switch)
        self.machine.events.add_handler('init_phase_4',
                                        self._register_switch_handlers)

        if self.config['ball_search_order']:
            self.config['playfield'].ball_search.register(
                self.config['ball_search_order'], self._ball_search, self.name)

    def _ignore_switch_hits_for(self, ms):
        """Ignore switch hits for ms."""
        self._ignore_switch_hits = True
        self.delay.reset(name="ignore_switch", callback=self._restore_switch_hits, ms=ms)

    def _restore_switch_hits(self):
        self._ignore_switch_hits = False

    def _ball_search_phase1(self):
        if not self.complete and self.reset_coil:
            self.reset_coil.pulse()
            return True
        # if down. knock down again
        elif self.complete and self.knockdown_coil:
            self.knockdown_coil.pulse()
            return True

    def _ball_search_phase2(self):
        if self.reset_coil and self.knockdown_coil:
            if self.complete:
                self._in_ball_search = True
                self.reset_coil.pulse()
                self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                self._in_ball_search = True
                self.knockdown_coil.pulse()
                self.delay.add(100, self._ball_search_reset)
                return True
        else:
            # fall back to phase1
            return self._ball_search_phase1()

    def _ball_search_phase3(self):
        if self.complete:
            if self.reset_coil:
                self.reset_coil.pulse()
                if self.knockdown_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                return self._ball_search_phase1()
        else:
            if self.knockdown_coil:
                self.knockdown_coil.pulse()
                if self.reset_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_reset)
                return True
            else:
                return self._ball_search_phase1()

    def _ball_search_iteration_finish(self):
        self._in_ball_search = False

    def _ball_search_knockdown(self):
        self.knockdown_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search_reset(self):
        self.reset_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search(self, phase, iteration):
        del iteration
        if phase == 1:
            # phase 1: do not change state.
            # if up. reset again
            return self._ball_search_phase1()
        elif phase == 2:
            # phase 2: if we can reset and knockdown the target we will do that
            return self._ball_search_phase2()
        else:
            # phase3: reset no matter what
            return self._ball_search_phase3()

    def _register_switch_handlers(self, **kwargs):
        del kwargs
        # register for notification of switch state
        # this is in addition to the parent since drop targets track
        # self.complete in separately

        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 0)
        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 1)

    def enable_keep_up(self, **kwargs):
        """Keep the target up by enabling the coil."""
        del kwargs
        if self.reset_coil:
            self.reset_coil.enable()

    def disable_keep_up(self, **kwargs):
        """No longer keep up the target up."""
        del kwargs
        if self.reset_coil:
            self.reset_coil.disable()

    def knockdown(self, **kwargs):
        """Pulse the knockdown coil to knock down this drop target."""
        del kwargs
        if self.knockdown_coil and not self.machine.switch_controller.is_active(self.config['switch'].name):
            self.knockdown_coil.pulse()
            self._ignore_switch_hits_for(ms=self.config['ignore_switch_ms'])

    def _update_state_from_switch(self, **kwargs):
        del kwargs
        if self._in_ball_search:
            return

        if not self._ignore_switch_hits:
            self.config['playfield'].mark_playfield_active_from_device_action()

        if self.machine.switch_controller.is_active(
                self.config['switch'].name):
            self._down()
        else:
            self._up()

        self._update_banks()

    def _down(self):
        self.complete = True
        self.machine.events.post('drop_target_' + self.name + '_down', device=self)
        '''event: drop_target_(name)_down
        desc: The drop target with the (name) has just changed to the "down"
        state.'''

    def _up(self):
        self.complete = False
        self.machine.events.post('drop_target_' + self.name + '_up', device=self)
        '''event: drop_target_(name)_up
        desc: The drop target (name) has just changed to the "up" state.'''

    def _update_banks(self):
        for bank in self.banks:
            bank.member_target_change()

    def add_to_bank(self, bank):
        """Add this drop target to a drop target bank.

         This allows the bank to update its status based on state changes to this drop target.

        Args:
            bank: DropTargetBank object to add this drop target to.
        """
        self.banks.add(bank)

    def remove_from_bank(self, bank):
        """Remove the DropTarget from a bank.

        Args:
            bank: DropTargetBank object to remove
        """
        self.banks.remove(bank)

    def reset(self, **kwargs):
        """Reset this drop target.

        If this drop target is configured with a reset coil, then this method
        will pulse that coil. If not, then it checks to see if this drop target
        is part of a drop target bank, and if so, it calls the reset() method of
        the drop target bank.

        This method does not reset the target profile, however, the switch event
        handler should reset the target profile on its own when the drop target
        physically moves back to the up position.
        """
        del kwargs

        if self.reset_coil and self.machine.switch_controller.is_active(self.config['switch'].name):
            self.reset_coil.pulse()
            self._ignore_switch_hits_for(ms=self.config['ignore_switch_ms'])

@DeviceMonitor("complete", "down", "up")
class DropTargetBank(SystemWideDevice, ModeDevice):

    """A bank of drop targets in a pinball machine by grouping together multiple `DropTarget` class devices."""

    config_section = 'drop_target_banks'
    collection = 'drop_target_banks'
    class_label = 'drop_target_bank'

    def __init__(self, machine, name):
        """Initialise drop target bank."""
        super().__init__(machine, name)

        self.drop_targets = list()
        self.reset_coil = None
        self.reset_coils = set()
        self.complete = False
        self.down = 0
        self.up = 0
        self.delay = DelayManager(machine.delayRegistry)
        self._ignore_switch_hits = False

    @property
    def can_exist_outside_of_game(self):
        """Return true if this device can exist outside of a game."""
        return True

    def _initialize(self):
        self.drop_targets = self.config['drop_targets']
        self.reset_coil = self.config['reset_coil']
        self.reset_coils = self.config['reset_coils']

        for target in self.drop_targets:
            target.add_to_bank(self)

        self.member_target_change()

        self.debug_log('Drop Targets: %s', self.drop_targets)

    def reset(self, **kwargs):
        """Reset this bank of drop targets.

        This method has some intelligence to figure out what coil(s) it should
        fire. It builds up a set by looking at its own reset_coil and
        reset_coils settings, and also scanning through all the member drop
        targets and collecting their coils. Then it pulses each of them. (This
        coil list is a "set" which means it only sends a single pulse to each
        coil, even if each drop target is configured with its own coil.)
        """
        del kwargs
        self.debug_log('Resetting')

        # figure out all the coils we need to pulse
        coils = set()

        for drop_target in self.drop_targets:
            if drop_target.reset_coil:
                coils.add(drop_target.reset_coil)

        for coil in self.reset_coils:
            coils.add(coil)

        if self.reset_coil:
            coils.add(self.reset_coil)

        if self.config['ignore_switch_ms']:
            self.machine.events.post('ignore')
            self._ignore_switch_hits = True
            self.delay.add(ms=self.config['ignore_switch_ms'],
                           callback=self._restore_switch_hits,
                           name='ignore_hits')

        # now pulse them
        for coil in coils:
            self.debug_log('Pulsing reset coils: %s', coils)
            coil.pulse()

    def _restore_switch_hits(self):
        self.machine.events.post('restore')
        self._ignore_switch_hits = False
        self.member_target_change()

    def member_target_change(self):
        """A member drop target has changed state.

        This method causes this group to update its down and up counts and
        complete status.
        """

        if self._ignore_switch_hits:
            return

        self.down = 0
        self.up = 0

        for target in self.drop_targets:
            if target.complete:
                self.down += 1
            else:
                self.up += 1

        self.debug_log(
            'Member drop target status change: Up: %s, Down: %s,'
            ' Total: %s', self.up, self.down,
            len(self.drop_targets))

        if self.down == len(self.drop_targets):
            self._bank_down()
        elif not self.down:
            self._bank_up()
        else:
            self._bank_mixed()

    def _bank_down(self):
        self.complete = True
        self.debug_log('All targets are down')

        if self.config['reset_on_complete']:
            self.debug_log("Reset on complete after %s", self.config['reset_on_complete'])
            self.delay.add(self.config['reset_on_complete'], self.reset)

        self.machine.events.post('drop_target_bank_' + self.name + '_down')
        '''event: drop_target_bank_(name)_down
        desc: Every drop target in the drop target bank called
        (name) is now in the "down" state. This event is
        only posted once, when all the drop targets are down.'''

    def _bank_up(self):
        self.complete = False
        self.debug_log('All targets are up')
        self.machine.events.post('drop_target_bank_' + self.name + '_up')
        '''event: drop_target_bank_(name)_up
        desc: Every drop target in the drop target bank called
        (name) is now in the "up" state. This event is
        only posted once, when all the drop targets are up.'''

    def _bank_mixed(self):
        self.complete = False
        self.machine.events.post('drop_target_bank_' + self.name + '_mixed',
                                 down=self.down)
        '''event: drop_target_bank_(name)_mixed
        desc: The drop targets in the drop target bank
        (name) are in a "mixed" state, meaning that they're
        not all down or not all up. This event is posted every time a member
        drop target changes but the overall bank is not not complete.'''

    def device_removed_from_mode(self, mode):
        """Remove targets which were added in this mode."""

        self.delay.remove('ignore_hits')

        for target in self.drop_targets:
            target.remove_from_bank(self)
<commit_msg>remove comically useless event<commit_after>"""Contains the base classes for drop targets and drop target banks."""

from mpf.core.delays import DelayManager
from mpf.core.device_monitor import DeviceMonitor
from mpf.core.mode_device import ModeDevice
from mpf.core.system_wide_device import SystemWideDevice

@DeviceMonitor("complete")
class DropTarget(SystemWideDevice):

    """Represents a single drop target in a pinball machine.

    Args: Same as the `Target` parent class
    """

    config_section = 'drop_targets'
    collection = 'drop_targets'
    class_label = 'drop_target'

    def __init__(self, machine, name):
        """Initialise drop target."""
        self.reset_coil = None
        self.knockdown_coil = None
        self.banks = None
        super().__init__(machine, name)

        self._in_ball_search = False
        self.complete = False
        self.delay = DelayManager(machine.delayRegistry)

        self._ignore_switch_hits = False

    def _initialize(self):
        self.reset_coil = self.config['reset_coil']
        self.knockdown_coil = self.config['knockdown_coil']
        self.banks = set()

        # can't read the switch until the switch controller is set up
        self.machine.events.add_handler('init_phase_4',
                                        self._update_state_from_switch)
        self.machine.events.add_handler('init_phase_4',
                                        self._register_switch_handlers)

        if self.config['ball_search_order']:
            self.config['playfield'].ball_search.register(
                self.config['ball_search_order'], self._ball_search, self.name)

    def _ignore_switch_hits_for(self, ms):
        """Ignore switch hits for ms."""
        self._ignore_switch_hits = True
        self.delay.reset(name="ignore_switch", callback=self._restore_switch_hits, ms=ms)

    def _restore_switch_hits(self):
        self._ignore_switch_hits = False

    def _ball_search_phase1(self):
        if not self.complete and self.reset_coil:
            self.reset_coil.pulse()
            return True
        # if down. knock down again
        elif self.complete and self.knockdown_coil:
            self.knockdown_coil.pulse()
            return True

    def _ball_search_phase2(self):
        if self.reset_coil and self.knockdown_coil:
            if self.complete:
                self._in_ball_search = True
                self.reset_coil.pulse()
                self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                self._in_ball_search = True
                self.knockdown_coil.pulse()
                self.delay.add(100, self._ball_search_reset)
                return True
        else:
            # fall back to phase1
            return self._ball_search_phase1()

    def _ball_search_phase3(self):
        if self.complete:
            if self.reset_coil:
                self.reset_coil.pulse()
                if self.knockdown_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_knockdown)
                return True
            else:
                return self._ball_search_phase1()
        else:
            if self.knockdown_coil:
                self.knockdown_coil.pulse()
                if self.reset_coil:
                    self._in_ball_search = True
                    self.delay.add(100, self._ball_search_reset)
                return True
            else:
                return self._ball_search_phase1()

    def _ball_search_iteration_finish(self):
        self._in_ball_search = False

    def _ball_search_knockdown(self):
        self.knockdown_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search_reset(self):
        self.reset_coil.pulse()
        self.delay.add(100, self._ball_search_iteration_finish)

    def _ball_search(self, phase, iteration):
        del iteration
        if phase == 1:
            # phase 1: do not change state.
            # if up. reset again
            return self._ball_search_phase1()
        elif phase == 2:
            # phase 2: if we can reset and knockdown the target we will do that
            return self._ball_search_phase2()
        else:
            # phase3: reset no matter what
            return self._ball_search_phase3()

    def _register_switch_handlers(self, **kwargs):
        del kwargs
        # register for notification of switch state
        # this is in addition to the parent since drop targets track
        # self.complete in separately

        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 0)
        self.machine.switch_controller.add_switch_handler(
            self.config['switch'].name,
            self._update_state_from_switch, 1)

    def enable_keep_up(self, **kwargs):
        """Keep the target up by enabling the coil."""
        del kwargs
        if self.reset_coil:
            self.reset_coil.enable()

    def disable_keep_up(self, **kwargs):
        """No longer keep up the target up."""
        del kwargs
        if self.reset_coil:
            self.reset_coil.disable()

    def knockdown(self, **kwargs):
        """Pulse the knockdown coil to knock down this drop target."""
        del kwargs
        if self.knockdown_coil and not self.machine.switch_controller.is_active(self.config['switch'].name):
            self.knockdown_coil.pulse()
            self._ignore_switch_hits_for(ms=self.config['ignore_switch_ms'])

    def _update_state_from_switch(self, **kwargs):
        del kwargs
        if self._in_ball_search:
            return

        if not self._ignore_switch_hits:
            self.config['playfield'].mark_playfield_active_from_device_action()

        if self.machine.switch_controller.is_active(
                self.config['switch'].name):
            self._down()
        else:
            self._up()

        self._update_banks()

    def _down(self):
        self.complete = True
        self.machine.events.post('drop_target_' + self.name + '_down', device=self)
        '''event: drop_target_(name)_down
        desc: The drop target with the (name) has just changed to the "down"
        state.'''

    def _up(self):
        self.complete = False
        self.machine.events.post('drop_target_' + self.name + '_up', device=self)
        '''event: drop_target_(name)_up
        desc: The drop target (name) has just changed to the "up" state.'''

    def _update_banks(self):
        for bank in self.banks:
            bank.member_target_change()

    def add_to_bank(self, bank):
        """Add this drop target to a drop target bank.

         This allows the bank to update its status based on state changes to this drop target.

        Args:
            bank: DropTargetBank object to add this drop target to.
        """
        self.banks.add(bank)

    def remove_from_bank(self, bank):
        """Remove the DropTarget from a bank.

        Args:
            bank: DropTargetBank object to remove
        """
        self.banks.remove(bank)

    def reset(self, **kwargs):
        """Reset this drop target.

        If this drop target is configured with a reset coil, then this method
        will pulse that coil. If not, then it checks to see if this drop target
        is part of a drop target bank, and if so, it calls the reset() method of
        the drop target bank.

        This method does not reset the target profile, however, the switch event
        handler should reset the target profile on its own when the drop target
        physically moves back to the up position.
        """
        del kwargs

        if self.reset_coil and self.machine.switch_controller.is_active(self.config['switch'].name):
            self.reset_coil.pulse()
            self._ignore_switch_hits_for(ms=self.config['ignore_switch_ms'])

@DeviceMonitor("complete", "down", "up")
class DropTargetBank(SystemWideDevice, ModeDevice):

    """A bank of drop targets in a pinball machine by grouping together multiple `DropTarget` class devices."""

    config_section = 'drop_target_banks'
    collection = 'drop_target_banks'
    class_label = 'drop_target_bank'

    def __init__(self, machine, name):
        """Initialise drop target bank."""
        super().__init__(machine, name)

        self.drop_targets = list()
        self.reset_coil = None
        self.reset_coils = set()
        self.complete = False
        self.down = 0
        self.up = 0
        self.delay = DelayManager(machine.delayRegistry)
        self._ignore_switch_hits = False

    @property
    def can_exist_outside_of_game(self):
        """Return true if this device can exist outside of a game."""
        return True

    def _initialize(self):
        self.drop_targets = self.config['drop_targets']
        self.reset_coil = self.config['reset_coil']
        self.reset_coils = self.config['reset_coils']

        for target in self.drop_targets:
            target.add_to_bank(self)

        self.member_target_change()

        self.debug_log('Drop Targets: %s', self.drop_targets)

    def reset(self, **kwargs):
        """Reset this bank of drop targets.

        This method has some intelligence to figure out what coil(s) it should
        fire. It builds up a set by looking at its own reset_coil and
        reset_coils settings, and also scanning through all the member drop
        targets and collecting their coils. Then it pulses each of them. (This
        coil list is a "set" which means it only sends a single pulse to each
        coil, even if each drop target is configured with its own coil.)
        """
        del kwargs
        self.debug_log('Resetting')

        # figure out all the coils we need to pulse
        coils = set()

        for drop_target in self.drop_targets:
            if drop_target.reset_coil:
                coils.add(drop_target.reset_coil)

        for coil in self.reset_coils:
            coils.add(coil)

        if self.reset_coil:
            coils.add(self.reset_coil)

        if self.config['ignore_switch_ms']:
            self._ignore_switch_hits = True
            self.delay.add(ms=self.config['ignore_switch_ms'],
                           callback=self._restore_switch_hits,
                           name='ignore_hits')

        # now pulse them
        for coil in coils:
            self.debug_log('Pulsing reset coils: %s', coils)
            coil.pulse()

    def _restore_switch_hits(self):
        self.machine.events.post('restore')
        self._ignore_switch_hits = False
        self.member_target_change()

    def member_target_change(self):
        """A member drop target has changed state.

        This method causes this group to update its down and up counts and
        complete status.
        """

        if self._ignore_switch_hits:
            return

        self.down = 0
        self.up = 0

        for target in self.drop_targets:
            if target.complete:
                self.down += 1
            else:
                self.up += 1

        self.debug_log(
            'Member drop target status change: Up: %s, Down: %s,'
            ' Total: %s', self.up, self.down,
            len(self.drop_targets))

        if self.down == len(self.drop_targets):
            self._bank_down()
        elif not self.down:
            self._bank_up()
        else:
            self._bank_mixed()

    def _bank_down(self):
        self.complete = True
        self.debug_log('All targets are down')

        if self.config['reset_on_complete']:
            self.debug_log("Reset on complete after %s", self.config['reset_on_complete'])
            self.delay.add(self.config['reset_on_complete'], self.reset)

        self.machine.events.post('drop_target_bank_' + self.name + '_down')
        '''event: drop_target_bank_(name)_down
        desc: Every drop target in the drop target bank called
        (name) is now in the "down" state. This event is
        only posted once, when all the drop targets are down.'''

    def _bank_up(self):
        self.complete = False
        self.debug_log('All targets are up')
        self.machine.events.post('drop_target_bank_' + self.name + '_up')
        '''event: drop_target_bank_(name)_up
        desc: Every drop target in the drop target bank called
        (name) is now in the "up" state. This event is
        only posted once, when all the drop targets are up.'''

    def _bank_mixed(self):
        self.complete = False
        self.machine.events.post('drop_target_bank_' + self.name + '_mixed',
                                 down=self.down)
        '''event: drop_target_bank_(name)_mixed
        desc: The drop targets in the drop target bank
        (name) are in a "mixed" state, meaning that they're
        not all down or not all up. This event is posted every time a member
        drop target changes but the overall bank is not not complete.'''

    def device_removed_from_mode(self, mode):
        """Remove targets which were added in this mode."""

        self.delay.remove('ignore_hits')

        for target in self.drop_targets:
            target.remove_from_bank(self)
<|endoftext|>


<commit_before>#!/usr/bin/env python
from argparse import ArgumentParser
from json import dump
import os
from os.path import (
    basename,
    dirname,
    join,
)
import subprocess

def main():
    scripts = dirname(__file__)
    parser = ArgumentParser()
    parser.add_argument('host', help='The machine to test on.')
    parser.add_argument('revision', help='The revision-build to test.')
    parser.add_argument('package', nargs='?', default='github.com/juju/juju',
                        help='The package to test.')
    args = parser.parse_args()

    juju_ci_path = join(scripts, 'jujuci.py')
    downloaded = subprocess.check_output([
        juju_ci_path, 'get', '-b', args.revision, 'build-revision', '*.tar.gz',
        './'])
    (tarfile,) = [basename(l) for l in downloaded.splitlines()]

    subprocess.check_call([
        juju_ci_path, 'get-build-vars', '--summary', args.revision])
    with open('temp-config.yaml', 'w') as temp_file:
        dump({
            'install': {'ci': [
                tarfile,
                join(scripts, 'gotesttarfile.py'),
                join(scripts, 'jujucharm.py'),
                join(scripts, 'utility.py'),
                ]},
            'command': [
                'python', 'ci/gotesttarfile.py', '-v', '-g', 'go.exe', '-p',
                args.package, '--remove', 'ci/{}'.format(tarfile)
                ]},
             temp_file)
    juju_home = os.environ.get('JUJU_HOME',
                               join(dirname(scripts), 'cloud-city'))
    subprocess.check_call([
        'workspace-run', '-v', '-i', join(juju_home, 'staging-juju-rsa'),
        'temp-config.yaml', 'Administrator@{}'.format(args.host)
        ])

if __name__ == '__main__':
    main()
<commit_msg>Upload local tarfile.<commit_after>#!/usr/bin/env python
from argparse import ArgumentParser
from json import dump
import os
from os.path import (
    basename,
    dirname,
    join,
)
import subprocess

def main():
    scripts = dirname(__file__)
    parser = ArgumentParser()
    parser.add_argument('host', help='The machine to test on.')
    parser.add_argument('revision',
                        help='The revision-build or tarfile to test.')
    parser.add_argument('package', nargs='?', default='github.com/juju/juju',
                        help='The package to test.')
    args = parser.parse_args()

    if args.revision.endswith('tar.gz'):
        downloaded = args.revision
    else:
        juju_ci_path = join(scripts, 'jujuci.py')
        downloaded = subprocess.check_output([
            juju_ci_path, 'get', '-b', args.revision, 'build-revision',
            '*.tar.gz', './'])
    (tarfile,) = [basename(l) for l in downloaded.splitlines()]

    subprocess.check_call([
        juju_ci_path, 'get-build-vars', '--summary', args.revision])
    with open('temp-config.yaml', 'w') as temp_file:
        dump({
            'install': {'ci': [
                tarfile,
                join(scripts, 'gotesttarfile.py'),
                join(scripts, 'jujucharm.py'),
                join(scripts, 'utility.py'),
                ]},
            'command': [
                'python', 'ci/gotesttarfile.py', '-v', '-g', 'go.exe', '-p',
                args.package, '--remove', 'ci/{}'.format(tarfile)
                ]},
             temp_file)
    juju_home = os.environ.get('JUJU_HOME',
                               join(dirname(scripts), 'cloud-city'))
    subprocess.check_call([
        'workspace-run', '-v', '-i', join(juju_home, 'staging-juju-rsa'),
        'temp-config.yaml', 'Administrator@{}'.format(args.host)
        ])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from nussl import ml, datasets
import tempfile
from torch import optim
import numpy as np
import logging
import os
import torch
from matplotlib import pyplot as plt

logging.basicConfig(
    format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
    datefmt='%Y-%m-%d:%H:%M:%S',
    level=logging.INFO
)

fix_dir = 'tests/local/trainer'

def test_overfit_a(mix_source_folder):
    tfms = datasets.transforms.Compose([
        datasets.transforms.PhaseSensitiveSpectrumApproximation(),
        datasets.transforms.ToSeparationModel(),
        datasets.transforms.Cache('~/.nussl/tests/cache', overwrite=True),
        datasets.transforms.GetExcerpt(400)
    ])
    dataset = datasets.MixSourceFolder(
        mix_source_folder, transform=tfms)

    ml.train.cache_dataset(dataset)
    dataset.cache_populated = True

    dataloader = torch.utils.data.DataLoader(
        dataset, shuffle=True, batch_size=len(dataset), num_workers=2)

    # create the model, based on the first item in the dataset
    # second bit of the shape is the number of features
    n_features = dataset[0]['mix_magnitude'].shape[1]
    mi_config = ml.networks.builders.build_recurrent_mask_inference(
        n_features, 50, 1, False, 0.0, 2, 'sigmoid',
    )

    model = ml.SeparationModel(mi_config)
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if device == 'cuda':
        epoch_length = 100
    else:
        epoch_length = 10
    model = model.to(device)
    # create optimizer
    optimizer = optim.Adam(model.parameters(), lr=1e-3)

    loss_dictionary = {
        'L1Loss': {
            'weight': 1.0
        }
    }

    train_closure = ml.train.closures.TrainClosure(
        loss_dictionary, optimizer, model)
    val_closure = ml.train.closures.ValidationClosure(
        loss_dictionary, model)

    with tempfile.TemporaryDirectory() as tmpdir:
        _dir = fix_dir if fix_dir else tmpdir
        trainer, validator = ml.train.create_train_and_validation_engines(
            train_closure, val_closure, device=device
        )

        # add handlers to engine
        ml.train.add_stdout_handler(trainer, validator)
        ml.train.add_validate_and_checkpoint(
            _dir, model, optimizer, dataset,
            trainer, val_data=dataloader, validator=validator)
        ml.train.add_tensorboard_handler(_dir, trainer)

        # run engine
        trainer.run(dataloader, max_epochs=5, epoch_length=epoch_length)

        model_path = os.path.join(
            trainer.state.output_folder, 'checkpoints', 'best.model.pth')
        state_dict = torch.load(
            model_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(state_dict['state_dict'])

        history = state_dict['metadata']['trainer.state.epoch_history']

        for key in history:
            plt.figure(figsize=(10, 4))
            plt.title(f"epoch:{key}")
            plt.plot(np.array(history[key]).reshape(-1,))
            plt.savefig(os.path.join(
                trainer.state.output_folder, 'plots',
                f"epoch:{key.replace('/', ':')}.png"))
<commit_msg>making directory<commit_after>from nussl import ml, datasets
import tempfile
from torch import optim
import numpy as np
import logging
import os
import torch
from matplotlib import pyplot as plt

logging.basicConfig(
    format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
    datefmt='%Y-%m-%d:%H:%M:%S',
    level=logging.INFO
)

fix_dir = 'tests/local/trainer'

def test_overfit_a(mix_source_folder):
    tfms = datasets.transforms.Compose([
        datasets.transforms.PhaseSensitiveSpectrumApproximation(),
        datasets.transforms.ToSeparationModel(),
        datasets.transforms.Cache('~/.nussl/tests/cache', overwrite=True),
        datasets.transforms.GetExcerpt(400)
    ])
    dataset = datasets.MixSourceFolder(
        mix_source_folder, transform=tfms)

    ml.train.cache_dataset(dataset)
    dataset.cache_populated = True

    dataloader = torch.utils.data.DataLoader(
        dataset, shuffle=True, batch_size=len(dataset), num_workers=2)

    # create the model, based on the first item in the dataset
    # second bit of the shape is the number of features
    n_features = dataset[0]['mix_magnitude'].shape[1]
    mi_config = ml.networks.builders.build_recurrent_mask_inference(
        n_features, 50, 1, False, 0.0, 2, 'sigmoid',
    )

    model = ml.SeparationModel(mi_config)
    device = 'cuda' if torch.cuda.is_available() else 'cpu'
    if device == 'cuda':
        epoch_length = 100
    else:
        epoch_length = 10
    model = model.to(device)
    # create optimizer
    optimizer = optim.Adam(model.parameters(), lr=1e-3)

    loss_dictionary = {
        'L1Loss': {
            'weight': 1.0
        }
    }

    train_closure = ml.train.closures.TrainClosure(
        loss_dictionary, optimizer, model)
    val_closure = ml.train.closures.ValidationClosure(
        loss_dictionary, model)

    with tempfile.TemporaryDirectory() as tmpdir:
        _dir = fix_dir if fix_dir else tmpdir
        os.makedirs(os.path.join(_dir, 'plots'), exist_ok=True)

        trainer, validator = ml.train.create_train_and_validation_engines(
            train_closure, val_closure, device=device
        )

        # add handlers to engine
        ml.train.add_stdout_handler(trainer, validator)
        ml.train.add_validate_and_checkpoint(
            _dir, model, optimizer, dataset,
            trainer, val_data=dataloader, validator=validator)
        ml.train.add_tensorboard_handler(_dir, trainer)

        # run engine
        trainer.run(dataloader, max_epochs=5, epoch_length=epoch_length)

        model_path = os.path.join(
            trainer.state.output_folder, 'checkpoints', 'best.model.pth')
        state_dict = torch.load(
            model_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(state_dict['state_dict'])

        history = state_dict['metadata']['trainer.state.epoch_history']

        for key in history:
            plt.figure(figsize=(10, 4))
            plt.title(f"epoch:{key}")
            plt.plot(np.array(history[key]).reshape(-1,))
            plt.savefig(os.path.join(
                trainer.state.output_folder, 'plots',
                f"epoch:{key.replace('/', ':')}.png"))
<|endoftext|>


<commit_before>from unittest.mock import MagicMock

from mpf.tests.MpfTestCase import MpfTestCase
from mpf.platforms import openpixel

class TestPlatform(MpfTestCase):

    def getConfigFile(self):
        return 'test_platform.yaml'

    def getMachinePath(self):
        return 'tests/machine_files/platform/'

    def get_platform(self):
        return False

    def setUp(self):
        openpixel.OPCSerialSender = MagicMock()
        super().setUp()

    def test_platform_from_device(self):
        # tests that a platform can be added by a device, even if it's not
        # specified in the hardware section

        self.assertEqual(self.machine.leds.led1.platform,
                         self.machine.hardware_platforms['smart_virtual'])

        self.assertEqual(self.machine.leds.led2.platform,
                         self.machine.hardware_platforms['fadecandy'])
<commit_msg>mock fadecandy<commit_after>from unittest.mock import MagicMock

from mpf.tests.MpfTestCase import MpfTestCase
from mpf.platforms import openpixel
from mpf.tests.loop import MockSocket

class TestPlatform(MpfTestCase):

    def getConfigFile(self):
        return 'test_platform.yaml'

    def getMachinePath(self):
        return 'tests/machine_files/platform/'

    def get_platform(self):
        return False

    def setUp(self):
        openpixel.OPCSerialSender = MagicMock()
        super().setUp()

    def _mock_loop(self):
        self._mock_socket = MockSocket()
        self.loop.mock_socket("localhost", 7890, self._mock_socket)

    def test_platform_from_device(self):
        # tests that a platform can be added by a device, even if it's not
        # specified in the hardware section

        self.assertEqual(self.machine.leds.led1.platform,
                         self.machine.hardware_platforms['smart_virtual'])

        self.assertEqual(self.machine.leds.led2.platform,
                         self.machine.hardware_platforms['fadecandy'])
<|endoftext|>


<commit_before>import logging
import pprint
import urllib2
import couchdb
import json
import urlparse
import ConfigParser
import os
from pylons import config
from lr.lib import ModelParser, SpecValidationException,helpers as h
from lr.lib.couch_change_monitor import BaseChangeHandler
from lr.model import ResourceDataModel
from couchdb import ResourceConflict
log = logging.getLogger(__name__)

# this doesn't need to be done... should be handled by pylons.config
# scriptPath = os.path.dirname(os.path.abspath(__file__))
# _PYLONS_CONFIG =  os.path.join(scriptPath, '..', '..', '..', 'development.ini')
# _config = ConfigParser.ConfigParser()
# _config.read(_PYLONS_CONFIG)

_RESOURCE_DISTRIBUTABLE_TYPE = "resource_data_distributable"
_RESOURCE_TYPE = "resource_data"
_DOC_TYPE = "doc_type"
_DOC = "doc"

class IncomingCopyHandler(BaseChangeHandler):
	def __init__(self):
		self._serverUrl = config["couchdb.url"]
		self._targetName = config["couchdb.db.resourcedata"]

		s = couchdb.Server(self._serverUrl)
		self._db = s[self._targetName]

	def _canHandle(self, change, database):
		if ((_DOC in change) and (change[_DOC].get(_DOC_TYPE) ==_RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
			return True
		return False

	def _handle(self, change, database):

		try:
			newDoc = change[_DOC]
			newDoc['node_timestamp'] = h.nowToISO8601Zformat()
			if newDoc[_DOC_TYPE] == _RESOURCE_DISTRIBUTABLE_TYPE:
				newDoc['_id'] = newDoc['doc_ID']
				newDoc[_DOC_TYPE] = _RESOURCE_TYPE
			rd = ResourceDataModel(newDoc)
			rd.save()
			del database[newDoc['_id']]
		except SpecValidationException:
			log.error(str(newDoc) + " Fails Validation" )
		except ResourceConflict:
			pass #ignore conflicts
		except Exception as ex:
			log.error(ex)
<commit_msg>updated to delete for spec validation errors or conflicts<commit_after>import logging
import pprint
import urllib2
import couchdb
import json
import urlparse
import ConfigParser
import os
from pylons import config
from lr.lib import ModelParser, SpecValidationException,helpers as h
from lr.lib.couch_change_monitor import BaseChangeHandler
from lr.model import ResourceDataModel
from couchdb import ResourceConflict
log = logging.getLogger(__name__)

# this doesn't need to be done... should be handled by pylons.config
# scriptPath = os.path.dirname(os.path.abspath(__file__))
# _PYLONS_CONFIG =  os.path.join(scriptPath, '..', '..', '..', 'development.ini')
# _config = ConfigParser.ConfigParser()
# _config.read(_PYLONS_CONFIG)

_RESOURCE_DISTRIBUTABLE_TYPE = "resource_data_distributable"
_RESOURCE_TYPE = "resource_data"
_DOC_TYPE = "doc_type"
_DOC = "doc"

class IncomingCopyHandler(BaseChangeHandler):
	def __init__(self):
		self._serverUrl = config["couchdb.url"]
		self._targetName = config["couchdb.db.resourcedata"]

		s = couchdb.Server(self._serverUrl)
		self._db = s[self._targetName]

	def _canHandle(self, change, database):
		if ((_DOC in change) and (change[_DOC].get(_DOC_TYPE) ==_RESOURCE_DISTRIBUTABLE_TYPE or change[_DOC].get(_DOC_TYPE) == _RESOURCE_TYPE)):
			return True
		return False

	def _handle(self, change, database):
		should_delete = True
		try:
			log.debug('got here')
			newDoc = change[_DOC]
			newDoc['node_timestamp'] = h.nowToISO8601Zformat()
			rd = ResourceDataModel(newDoc)
			rd.save()
		except SpecValidationException:
			log.error(str(newDoc) + " Fails Validation" )
		except ResourceConflict:
			pass #ignore conflicts
		except Exception as ex:
			should_delete = False # don't delete something unexpected happend
			log.error(ex)
		if should_delete:
			try:
				del database[change[_DOC]['node_ID']]
			except Exception as ex:
				log.error(ex)<|endoftext|>