<commit_before>from __future__ import division
import sys
from optparse import OptionParser
import numpy
import matplotlib
from math import sqrt, floor, ceil

matplotlib.rcParams['backend'] = 'PDF'
from matplotlib import pyplot

def mc_bootstrap(data, n_sets):
    synth = numpy.empty((n_sets,) + data.shape, data.dtype)
    N = data.shape[0]
    for i in xrange(0, n_sets):
        indices = numpy.random.randint(N,size=(N,))
        synth[i] = data[indices]
    return synth

def mc_bootstrap_ci(estimator, data, alpha, n_sets=1000):
    alpha = min(alpha, abs(1-alpha))
    half_alpha = alpha / 2.0

    fhat = estimator(data)
    f_synth = numpy.empty((n_sets,), data.dtype)
    synth = mc_bootstrap(data, n_sets)
    for i in xrange(0, n_sets):
        f_synth[i] = estimator(synth[i])
    f_synth.sort()
    lb = f_synth[int(floor(n_sets*half_alpha))]
    ub = f_synth[int(ceil(n_sets*(1-half_alpha)))]

    return (fhat, lb, ub)

def blockavg(data,
             output_filename = None,
             plot_filename = None,
             title = 'Block Average Error Analysis',
             xlabel = 'Number of blocking transformations',
             ylabel = r'$\hat\sigma_{\left<x\right>}\ \pm \sigma_{\hat\sigma}$'):

    stds = []
    working_data = data.copy()
    while len(working_data) > 2:
        npoints = len(working_data)
        working_mean = working_data.mean()

        varest = ((working_data-working_mean)**2).sum() / (npoints*(npoints-1))
        stdest = sqrt(varest)
        stdstd = stdest/sqrt(2*(npoints-1))

        stds.append((stdest,stdstd))

        if len(working_data) % 2 == 1:
            working_data[-2] = (working_data[-2] + working_data[-1]) / 2
            working_data = working_data[:-1]
        working_data = (working_data[::2] + working_data[1::2]) / 2

    stds = numpy.array(stds)
    if output_filename:
        numpy.savetxt(output_filename, stds)

    if plot_filename:
        pyplot.errorbar(range(0,len(stds)), stds[:,0], yerr=stds[:,1],
                        zorder=10)
        pyplot.axis([0,len(stds),None,None])
        if title:
            pyplot.title(title)
        pyplot.xlabel(xlabel)
        pyplot.ylabel(ylabel)
        pyplot.savefig(plot_filename)
        pyplot.clf()

    for itrans in xrange(len(stds)-1, 1, -1):
        lb = stds[itrans,0] - stds[itrans,1]
        ub = stds[itrans,0] + stds[itrans,1]
        if not (lb <= stds[itrans-1,0] < ub):
            break
    else:
        print "block average did not converge"
        itrans = -1

    return (itrans, stds[itrans,0], stds[itrans,1], stds)

parser = OptionParser(usage='%prog [OPTIONS] CURRENT_FILE')
parser.add_option('-b', '--start', '--begin', dest='start', type='int',
                  help='begin analysis at START (default: 0)',
                  default=0)
parser.add_option('-e', '--stop', '--end', dest='stop', type='int',
                  help='end analysis at STOP (inclusive; default: all data)')
parser.add_option('-s', '--step', dest='step', type='int',
                  help='output analysis for every STEP current values '
                      +'(default: 100)',
                  default=100)
parser.add_option('--skip-zero', dest='skip_zero', type='int',
                  help='skip SKIP_ZERO times the number of zero current '
                      +'values (default: 2)',
                  default=2)
parser.add_option('--skip-points', dest='skip_points', type='int',
                  help='skip SKIP_POINTS (default: 0)',
                  default=0)
parser.add_option('--skip-fraction', dest='skip_fraction', type='float',
                  help='skip SKIP_FRACTION of the data (default 0.0)',
                  default=0.0)
parser.add_option('--blockavg-set-output', dest='blockavg_set_output',
                  help='record each set of block averaging results in '
                      +'BLOCKAVG_SET_OUTPUT; may contain a %d substitution '
                      +'for N_tau (default: current_blockavg_%d.txt)',
                  default='current_blockavg_%d.txt')
parser.add_option('--blockavg-set-plot', dest='blockavg_set_plot',
                  help='plot each set of block averaging results to '
                      +'BLOCKAVG_SET_PLOT; '
                      +'may contain a %d substitution for N_tau '
                      +'(default: current_blockavg_%d.pdf)',
                  default='current_blockavg_%d.pdf')
parser.add_option('--mc-alpha', dest='mc_alpha', type='float',
                  help='significance level for Monte Carlo bootstrap',
                  default=0.05)
parser.add_option('--mc-nsynth', dest='mc_nsynth', type='int',
                  help='number of synthetic data sets for MC bootstrap '
                      +'(default: 2000)',
                  default=2000)
parser.add_option('--blockavg-output', dest='blockavg_output',
                  help='store block average analysis results in '
                      +'BLOCKAVG_OUTPUT (default: current_taudep_ba.txt)',
                  default='current_taudep_ba.txt')
parser.add_option('--blockavg-plot', dest='blockavg_plot',
                  help='plot block average analysis results to '
                      +'BLOCKAVG_PLOT (default: current_taudep_ba.pdf)',
                  default='current_taudep_ba.pdf')
parser.add_option('--mc-output', dest='mc_output',
                  help='store MC bootstrap analysis results in MC_OUTPUT '
                      +'(default: current_taudep_mc.txt)',
                  default='current_taudep_mc.txt')
parser.add_option('--mc-plot', dest='mc_plot',
                  help='plot MC bootstrap analysis results to MC_PLOT '
                      +'(default: current_taudep_mc.pdf',
                  default='current_taudep_mc.pdf')

(opts, args) = parser.parse_args()

if not args:
    sys.stderr.write('a file containing current values is required\n')
    parser.print_help(sys.stderr)
    sys.exit(1)
else:
    print "loading current values from '%s'" % args[0]

current = numpy.loadtxt(args[0])
numzero = len(current[current[:,2] == 0.0])
numtrunc = max([int(opts.skip_zero) * numzero,
                int(opts.skip_points),
                int(opts.skip_fraction * current.shape[0])])

print "omitting first %d points" % numtrunc

td_current_ba = []
td_current_mc = []
start = opts.start
if opts.stop:
    stop = opts.stop
else:
    stop = len(current)+1
step = opts.step
for n_tau in xrange(start, stop, step):
    if n_tau <= numtrunc: continue

    if opts.blockavg_set_output:
        try:
            bavg_output = opts.blockavg_set_output % n_tau
        except TypeError:
            bavg_output = '%s_%d' % (opts.blockavg_set_output, n_tau)
    else:
        bavg_output = None

    if opts.blockavg_set_plot:
        try:
            bavg_plot = opts.blockavg_set_plot % n_tau
        except TypeError:
            bavg_plot = '%s_%d' % (opts.blockavg_set_plot, n_tau)
    else:
        bavg_plot = None

    print "\nn_tau = %d" % n_tau
    avg_current = current[numtrunc:n_tau,2].mean()
    bavg_data = blockavg(current[numtrunc:n_tau, 2],
                         bavg_output, bavg_plot,
                         title = r'Block Average Error Analysis ($N_\tau=%d$)'%n_tau)
    (ntrans, sem, sem_error, stds) = bavg_data
    print "  block average results:"
    print "    correlation time:    %d tau" % ntrans
    print "    block average S.E.M: %g" % sem
    print "    rate +/- 2 S.E.M.:   %g +/- %g" % (avg_current, 2*sem)

    td_current_ba.append((n_tau, avg_current, 2*sem))

    (mc_avg, mc_lb, mc_ub) = mc_bootstrap_ci(numpy.mean,
                                             current[numtrunc:n_tau, 2],
                                             opts.mc_alpha,
                                             opts.mc_nsynth)
    print "  MC bootstrap results:"
    print "    95%% CI lower bound:  %g" % mc_lb
    print "    average:             %g" % mc_avg
    print "    95%% CI upper bound:  %g" % mc_ub
    print "    95%% CI width:        %g" % (mc_ub - mc_lb)

    td_current_mc.append((n_tau, mc_avg, mc_lb, mc_ub))

if opts.blockavg_output:
    numpy.savetxt(opts.blockavg_output, numpy.array(td_current_ba))

if opts.mc_output:
    numpy.savetxt(opts.mc_output, numpy.array(td_current_mc))

if opts.blockavg_plot:
    pyplot.clf()
    pyplot.errorbar([tdc[0] for tdc in td_current_ba],
                    [tdc[1] for tdc in td_current_ba],
                    [tdc[2]*2 for tdc in td_current_ba])
    pyplot.ylabel(r'$\left<k\right>\pm2\sigma(\left<k\right>)$')
    pyplot.xlabel(r'$N_\tau$')
    pyplot.title('Current Analysis (Block Averaging)')
    pyplot.savefig(opts.blockavg_plot)

if opts.mc_plot:
    pyplot.clf()
    td_current_bounds = numpy.empty((len(td_current_mc), 2), numpy.float64)
    for i in xrange(0, len(td_current_mc)):
        td_current_bounds[i,0] = td_current_mc[i][2]
        td_current_bounds[i,1] = td_current_mc[i][3]
    pyplot.errorbar([tdc[0] for tdc in td_current_mc],
                    [tdc[1] for tdc in td_current_mc],
                    td_current_bounds.T)
    pyplot.ylabel(r'$\left<k\right>$ and 95% CI')
    pyplot.xlabel(r'$N_\tau$')
    pyplot.title('Current Analysis (Monte Carlo Bootstrap)')
    pyplot.savefig(opts.mc_plot)

<commit_msg>current_taudep.py moved to a new \"wemdtools\" repo<commit_after><|endoftext|>


<commit_before>####################################################################################################
#
# These are the reference implementations for the node functions of the standard node types.
# Node net engine implementations performing node calculations in Python are free to use
# these implementations directly.
# Non-python engine implementations, or performance-oriented python implementations,
# can use these implementations as definitions (or for equivalence tests) and implement them
# differently.
#
####################################################################################################

def register(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    node.get_gate('gen').gate_function(activation)

def sensor(netapi, node=None, datasource=None, **params):
    if datasource in netapi.world.get_available_datasources(netapi.uid):
        datasource_value = netapi.world.get_datasource(netapi.uid, datasource)
    else:
        datasource_value = netapi.get_modulator(datasource)
    node.activation = datasource_value
    node.get_gate('gen').gate_function(datasource_value)

def actor(netapi, node=None, datatarget=None, **params):
    if not netapi.world:
        return
    activation_to_set = node.get_slot("gen").activation
    if datatarget in netapi.world.get_available_datatargets(netapi.uid):
        netapi.world.add_to_datatarget(netapi.uid, datatarget, activation_to_set)
        feedback = netapi.world.get_datatarget_feedback(netapi.uid, datatarget)
    else:
        netapi.set_modulator(datatarget, activation_to_set)
        feedback = 1
    if feedback is not None:
        node.get_gate('gen').gate_function(feedback)

def concept(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    for gate_type in node.get_gate_types():
        node.get_gate(gate_type).gate_function(activation)

def script(netapi, node=None, **params):
    """ Script nodes are state machines that use the node activation for determining their behavior.
        They form hierarchical scripts that are started via sub-activating their top-node. If the sur-activation
        is turned off, the script stops executing.
        At the bottom, the scripts must connect to activation sources (that yield positive sur activation), otherwise
        they will fail.
        The links have the following semantics:
            - sub determines sub-actions. Inactive sub-actions will turn "ready" if their parent action turns
              "requesting", by sending activation via sub.
            - por determines successors. If an action is neither inactive nor confirmed, it will inhibit its successors
              from requesting. Successors must wait for their predecessors to stop sending por-activation. The first
              action in a por-chain receives no such inhibition, it may turn from "ready" to "requesting". After
              an action has become "requesting", it turns "pending" and continues to request its sub-actions.
              All other "ready" actions will switch to "waiting" until their predecessors stop inhibiting them.
            - ret determines predecessors. Successors inhibit their predecessors from telling their parents when they
              confirm. (Once an action without sucessor confirms, we are done.)
            - sur has two functions. A high level of sur activation confirms the parent action. A low level of sur
              tells the parent action that at least one of its sub-actions has not given up yet. If no more sur
              activation is received by a pending parent action, it will fail.
            The distinction between "ready" and "waiting", and "requesting and "pending" is necessary to bridge the
            time until neighboring nodes have had time to make themselves heard.

        Currently, the node states correspond to the following activation levels:
        < 0     failed (will stick until requesting ends)
        < 0.01  inactive (will change to prepared when requested)
        < 0.3   preparing (inhibits neighbors  and changes to suppressed)
        < 0.5   suppressed (inhibits neighbors and goes to requesting when no longer inhibited)
        < 0.7   requesting (starts requesting and changes to pending)
        < 1     pending (keeps requesting, will either change to confirmed or failed)
        >=1     confirmed (will stick until requesting ends)

    """

    if node.get_slot("sub").activation < 0.01:  # node is not requested and is turned off
        node.activation = 0.0
        node.get_gate("por").gate_function(0.0)
        node.get_gate("ret").gate_function(0.0)
        node.get_gate("sub").gate_function(0.0)
        node.get_gate("sur").gate_function(0.0)
        return

    node.activation = (
        node.activation if node.activation < -0.01 else  # failed -> failed
        0.2 if node.activation < 0.01 else  # (we already tested that sub is positive): inactive -> preparing
        0.4 if node.activation < 0.5 and node.get_slot("por").activation < 0 else  # preparing -> supressed
        0.6 if node.activation < 0.5 else  # preparing/supressed -> requesting
        0.8 if node.activation < 0.7 else  # requesting -> pending
        1.0 if node.get_slot("sur").activation >= 1 else  # pending -> confirmed
        -1. if node.get_slot("sur").activation <= 0 else  # pending -> failed
        node.activation
    )

    # always inhibit successor, except when confirmed
    node.get_gate("por").gate_function(-1.0 if node.activation < 1 else 1.0)
    # inhibit confirmation of predecessor, and tell it to stop once successor is requested
    node.get_gate("ret").gate_function(-1.0 if 0.1 < node.activation < 1 else 1.0)
    # request children when becoming requesting
    node.get_gate("sub").gate_function(1.0 if 0.5 < node.activation else 0)
    # keep parent from failing while pending or processing, confirm parent when confirmed
    node.get_gate("sur").gate_function(
        0 if node.activation < 0.01 or node.get_slot("ret").activation > 0 else
        0.01 if node.activation < 1 else
        0.01 if node.get_slot("ret").activation < 0 else
        1)

def pipe(netapi, node=None, sheaf="default", **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0
    cat = 0.0
    exp = 0.0

    countdown = int(node.get_state("countdown") or 0)
    expectation = float(node.get_parameter("expectation") or 1.0)
    if node.get_slot("sub").activation <= 0 or (not node.get_slot("por").empty and node.get_slot("por").activation <= 0):
        countdown = int(node.get_parameter("wait") or 1)
    else:
        countdown -= 1

    gen += node.get_slot("gen").get_activation(sheaf) * node.get_slot("sub").get_activation(sheaf)
    if abs(gen) < 0.1: gen = 0                                          # cut off gen loop at lower threshold

    if node.get_slot("por").get_activation(sheaf) == 0 and not node.get_slot("por").empty:
        gen = 0

    if gen == 0:
        gen += node.get_slot("sur").get_activation(sheaf)
        gen += node.get_slot("exp").get_activation(sheaf)

    # commented: trigger and pipes should be able to escape the [-1;1] cage on gen
    # if gen > 1: gen = 1
    # if gen < -1: gen = -1

    sub += max(node.get_slot("sur").get_activation(sheaf), 0)
    sub += node.get_slot("sub").get_activation(sheaf)
    sub += node.get_slot("cat").get_activation(sheaf)
    sub *= max(node.get_slot("por").get_activation(sheaf), 0) if not node.get_slot("por").empty else 1
    sub *= 0 if node.get_slot("gen").get_activation(sheaf) != 0 else 1
    if sub > 0: sub = 1
    if sub < 0: sub = -1

    sur += node.get_slot("sur").get_activation(sheaf)
    if sur == 0: sur += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if abs(node.get_slot("gen").get_activation(sheaf) * node.get_slot("sub").get_activation(sheaf)) > 0.2:               # cut off sur-reports from gen looping before the loop fades away
        sur += 1 if node.get_slot("gen").get_activation(sheaf) > 0 else -1
    sur += node.get_slot("exp").get_activation(sheaf)

    if node.get_slot("ret").get_activation(sheaf) < 0:
        sur = 0
    if node.get_slot("por").empty and not node.get_slot("ret").empty and sur > 0:
        sur = 0                                                             # first nodes in scripts can never report sur
    if node.get_slot("por").get_activation(sheaf) <= 0 and not node.get_slot("por").empty:
        sur = 0

    if sur > 0 and sur < expectation:
        sur = 0

    if countdown <= 0 and sur < expectation:                                # timeout, fail
        sur = -1

    if sur >= expectation:                                                  # success, reset countdown counter
        countdown = int(node.get_parameter("wait") or 1)

    if node.get_slot('por').empty and node.get_slot('ret').empty:           # both empty
        classifierelements = 0
        if len(node.get_gate("sur").get_links()) == 1:
            surnode = node.get_gate("sur").get_links()[0].target_node
            if surnode.type == "Pipe":
                classifierelements = len(surnode.get_slot("sur").get_links())
        sur /= classifierelements if classifierelements > 1 else 1          # classifier case

    if sur > 1:
        sur = 1
    if sur < -1:
        sur = -1

    por += node.get_slot("sur").get_activation(sheaf)
    por += (0 if node.get_slot("gen").get_activation(sheaf) < 0.1 else 1) * \
           (1+node.get_slot("por").get_activation(sheaf))
    por *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1  # only por if por
    por *= node.get_slot("sub").get_activation(sheaf)                                           # only por if sub
    por += node.get_slot("por").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if por > 0: por = 1

    ret += node.get_slot("ret").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if node.get_slot("por").get_activation(sheaf) >= 0:
        ret -= node.get_slot("sub").get_activation(sheaf)
    if ret > 1:
        ret = 1

    cat = sub
    if cat == 0: cat += node.get_slot("cat").get_activation(sheaf)
    if cat < 0: cat = 0

    exp += node.get_slot("sur").get_activation(sheaf)
    exp += node.get_slot("exp").get_activation(sheaf)
    if exp == 0: exp += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if exp > 1: exp = 1

    if node.get_slot('sub').get_activation(sheaf) > 0:
        if sur > 0:
            netapi.change_modulator('base_number_of_expected_events', 1)
        elif sur < 0:
            severity = len(node.get_gate("sub").get_links()) + len(node.get_gate("cat").get_links())
            netapi.change_modulator('base_number_of_unexpected_events', severity)

    node.set_state("countdown", countdown)

    # set gates
    node.set_sheaf_activation(gen, sheaf)
    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("por").gate_function(por, sheaf)
    node.get_gate("ret").gate_function(ret, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)
    node.get_gate("exp").gate_function(exp, sheaf)
    node.get_gate("cat").gate_function(cat, sheaf)
    #if cat > 0 and node.get_slot("sub").get_activation(sheaf) > 0:     # cats will be checked in their own sheaf
    #    node.get_gate("cat").open_sheaf(cat, sheaf)
    #    node.get_gate("cat").gate_function(0, sheaf)
    #else:
    #    node.get_gate("cat").gate_function(cat, sheaf)

def trigger(netapi, node=None, sheaf="default", **params):
    """
    Trigger nodes, when activated on their sub slot, create a single burst of activation and wait for
    activation on their sur slot that equals or exceeds a configured value.
    If the value does not show up after a configured number of timeout steps, trigger nodes fail,
    else, they succeed.
    """

    gen = 0
    sub = 0
    sur = 0

    if abs(node.get_slot('gen').activation) != 0:
        sur = node.get_slot('gen').activation
    else:
        waitfrom = node.get_state("waitfrom") or -1

        condition = node.get_parameter("condition") or "="
        try:
            timeout = int(node.get_parameter("timeout"))         # todo: use emo_resolution
        except TypeError or ValueError:
            timeout = 10
        try:
            response = float(node.get_parameter("response"))
        except TypeError or ValueError:
            response = 1

        if node.get_slot('sub').activation > 0:
            currentstep = netapi.step

            success = (condition == "=" and node.get_slot('sur').activation == response) or \
                      (condition == ">" and node.get_slot('sur').activation > response)

            if waitfrom > 0:                       # waiting for results
                if success:
                    sur = 1
                    node.set_state("waitfrom", currentstep)
                elif currentstep > waitfrom + timeout or node.get_slot('sur').activation < 0:
                    sur = -1
            else:                                   # perform burst
                sub = 1
                node.set_state("waitfrom", currentstep)
                if success:
                    sur = 1

        else:

            if waitfrom > 0:
                node.set_state("waitfrom", -1)

            if node.get_slot('sur').activation:
                success = (condition == "=" and node.get_slot('sur').activation == response) or \
                          (condition == ">" and node.get_slot('sur').activation > response)
                if success:
                    sur = 1

    if sur < 0:
        # if trigger is failing, gen should also propagate failing
        gen = sur
    else:
        # otherwise, gen should be allowed to pass on the original sur-activation
        # and not be confined to {0;1}
        gen = node.get_slot('sur').activation + node.get_slot('gen').activation

    node.activation = gen

    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)

def activator(netapi, node, **params):
    node.activation = node.get_slot("gen").activation
    netapi.get_nodespace(node.parent_nodespace).set_activator_value(node.get_parameter('type'), node.activation)
<commit_msg>Fixing alternatives for dict pipes<commit_after>####################################################################################################
#
# These are the reference implementations for the node functions of the standard node types.
# Node net engine implementations performing node calculations in Python are free to use
# these implementations directly.
# Non-python engine implementations, or performance-oriented python implementations,
# can use these implementations as definitions (or for equivalence tests) and implement them
# differently.
#
####################################################################################################

def register(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    node.get_gate('gen').gate_function(activation)

def sensor(netapi, node=None, datasource=None, **params):
    if datasource in netapi.world.get_available_datasources(netapi.uid):
        datasource_value = netapi.world.get_datasource(netapi.uid, datasource)
    else:
        datasource_value = netapi.get_modulator(datasource)
    node.activation = datasource_value
    node.get_gate('gen').gate_function(datasource_value)

def actor(netapi, node=None, datatarget=None, **params):
    if not netapi.world:
        return
    activation_to_set = node.get_slot("gen").activation
    if datatarget in netapi.world.get_available_datatargets(netapi.uid):
        netapi.world.add_to_datatarget(netapi.uid, datatarget, activation_to_set)
        feedback = netapi.world.get_datatarget_feedback(netapi.uid, datatarget)
    else:
        netapi.set_modulator(datatarget, activation_to_set)
        feedback = 1
    if feedback is not None:
        node.get_gate('gen').gate_function(feedback)

def concept(netapi, node=None, **params):
    activation = node.get_slot('gen').activation
    node.activation = activation
    for gate_type in node.get_gate_types():
        node.get_gate(gate_type).gate_function(activation)

def script(netapi, node=None, **params):
    """ Script nodes are state machines that use the node activation for determining their behavior.
        They form hierarchical scripts that are started via sub-activating their top-node. If the sur-activation
        is turned off, the script stops executing.
        At the bottom, the scripts must connect to activation sources (that yield positive sur activation), otherwise
        they will fail.
        The links have the following semantics:
            - sub determines sub-actions. Inactive sub-actions will turn "ready" if their parent action turns
              "requesting", by sending activation via sub.
            - por determines successors. If an action is neither inactive nor confirmed, it will inhibit its successors
              from requesting. Successors must wait for their predecessors to stop sending por-activation. The first
              action in a por-chain receives no such inhibition, it may turn from "ready" to "requesting". After
              an action has become "requesting", it turns "pending" and continues to request its sub-actions.
              All other "ready" actions will switch to "waiting" until their predecessors stop inhibiting them.
            - ret determines predecessors. Successors inhibit their predecessors from telling their parents when they
              confirm. (Once an action without sucessor confirms, we are done.)
            - sur has two functions. A high level of sur activation confirms the parent action. A low level of sur
              tells the parent action that at least one of its sub-actions has not given up yet. If no more sur
              activation is received by a pending parent action, it will fail.
            The distinction between "ready" and "waiting", and "requesting and "pending" is necessary to bridge the
            time until neighboring nodes have had time to make themselves heard.

        Currently, the node states correspond to the following activation levels:
        < 0     failed (will stick until requesting ends)
        < 0.01  inactive (will change to prepared when requested)
        < 0.3   preparing (inhibits neighbors  and changes to suppressed)
        < 0.5   suppressed (inhibits neighbors and goes to requesting when no longer inhibited)
        < 0.7   requesting (starts requesting and changes to pending)
        < 1     pending (keeps requesting, will either change to confirmed or failed)
        >=1     confirmed (will stick until requesting ends)

    """

    if node.get_slot("sub").activation < 0.01:  # node is not requested and is turned off
        node.activation = 0.0
        node.get_gate("por").gate_function(0.0)
        node.get_gate("ret").gate_function(0.0)
        node.get_gate("sub").gate_function(0.0)
        node.get_gate("sur").gate_function(0.0)
        return

    node.activation = (
        node.activation if node.activation < -0.01 else  # failed -> failed
        0.2 if node.activation < 0.01 else  # (we already tested that sub is positive): inactive -> preparing
        0.4 if node.activation < 0.5 and node.get_slot("por").activation < 0 else  # preparing -> supressed
        0.6 if node.activation < 0.5 else  # preparing/supressed -> requesting
        0.8 if node.activation < 0.7 else  # requesting -> pending
        1.0 if node.get_slot("sur").activation >= 1 else  # pending -> confirmed
        -1. if node.get_slot("sur").activation <= 0 else  # pending -> failed
        node.activation
    )

    # always inhibit successor, except when confirmed
    node.get_gate("por").gate_function(-1.0 if node.activation < 1 else 1.0)
    # inhibit confirmation of predecessor, and tell it to stop once successor is requested
    node.get_gate("ret").gate_function(-1.0 if 0.1 < node.activation < 1 else 1.0)
    # request children when becoming requesting
    node.get_gate("sub").gate_function(1.0 if 0.5 < node.activation else 0)
    # keep parent from failing while pending or processing, confirm parent when confirmed
    node.get_gate("sur").gate_function(
        0 if node.activation < 0.01 or node.get_slot("ret").activation > 0 else
        0.01 if node.activation < 1 else
        0.01 if node.get_slot("ret").activation < 0 else
        1)

def pipe(netapi, node=None, sheaf="default", **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0
    cat = 0.0
    exp = 0.0

    countdown = int(node.get_state("countdown") or 0)
    expectation = float(node.get_parameter("expectation") or 1.0)
    if node.get_slot("sub").activation <= 0 or (not node.get_slot("por").empty and node.get_slot("por").activation <= 0):
        countdown = int(node.get_parameter("wait") or 1)
    else:
        countdown -= 1

    gen += node.get_slot("gen").get_activation(sheaf) * node.get_slot("sub").get_activation(sheaf)
    if abs(gen) < 0.1: gen = 0                                          # cut off gen loop at lower threshold

    if node.get_slot("por").get_activation(sheaf) == 0 and not node.get_slot("por").empty:
        gen = 0

    if gen == 0:
        gen += node.get_slot("sur").get_activation(sheaf)
        gen += node.get_slot("exp").get_activation(sheaf)

    # commented: trigger and pipes should be able to escape the [-1;1] cage on gen
    # if gen > 1: gen = 1
    # if gen < -1: gen = -1

    sub += max(node.get_slot("sur").get_activation(sheaf), 0)
    sub += node.get_slot("sub").get_activation(sheaf)
    sub += node.get_slot("cat").get_activation(sheaf)
    sub *= max(node.get_slot("por").get_activation(sheaf), 0) if not node.get_slot("por").empty else 1
    sub *= 0 if node.get_slot("gen").get_activation(sheaf) != 0 else 1
    if sub > 0: sub = 1
    if sub < 0: sub = -1

    sur += node.get_slot("sur").get_activation(sheaf)
    if sur == 0: sur += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if abs(node.get_slot("gen").get_activation(sheaf) * node.get_slot("sub").get_activation(sheaf)) > 0.2:               # cut off sur-reports from gen looping before the loop fades away
        sur += 1 if node.get_slot("gen").get_activation(sheaf) > 0 else -1
    sur += node.get_slot("exp").get_activation(sheaf)

    if sur > 0 and sur < expectation:                                       # don't report anything below expectation
        sur = 0

    if countdown <= 0 and sur < expectation:                                # timeout, fail
        sur = -1

    # silencing for pipes in in por-ret chains
    if not node.get_slot("ret").empty:                                      # we're not-last in a chain
        if not(node.get_slot("ret").get_activation(sheaf) == 0 and node.get_slot("sub").get_activation(sheaf) > 0):
            sur = 0
    if not node.get_slot("por").empty and node.get_slot("por").get_activation(sheaf) <= 0:
        sur = 0

    if sur >= expectation:                                                  # success, reset countdown counter
        countdown = int(node.get_parameter("wait") or 1)

    if node.get_slot('por').empty and node.get_slot('ret').empty:           # both empty
        classifierelements = 0
        if len(node.get_gate("sur").get_links()) == 1:
            surnode = node.get_gate("sur").get_links()[0].target_node
            if surnode.type == "Pipe":
                classifierelements = len(surnode.get_slot("sur").get_links())
        sur /= classifierelements if classifierelements > 1 else 1          # classifier case

    if sur > 1:
        sur = 1
    if sur < -1:
        sur = -1

    por += node.get_slot("sur").get_activation(sheaf)
    por += (0 if node.get_slot("gen").get_activation(sheaf) < 0.1 else 1) * \
           (1+node.get_slot("por").get_activation(sheaf))
    por *= node.get_slot("por").get_activation(sheaf) if not node.get_slot("por").empty else 1  # only por if por
    por *= node.get_slot("sub").get_activation(sheaf)                                           # only por if sub
    por += node.get_slot("por").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if por > 0: por = 1

    ret += node.get_slot("ret").get_activation(sheaf) if node.get_slot("sub").get_activation(sheaf) == 0 and node.get_slot("sur").get_activation(sheaf) == 0 else 0
    if node.get_slot("por").get_activation(sheaf) >= 0:
        ret -= node.get_slot("sub").get_activation(sheaf)
    if ret > 1:
        ret = 1

    cat = sub
    if cat == 0: cat += node.get_slot("cat").get_activation(sheaf)
    if cat < 0: cat = 0

    exp += node.get_slot("sur").get_activation(sheaf)
    exp += node.get_slot("exp").get_activation(sheaf)
    if exp == 0: exp += node.get_slot("sur").get_activation("default")      # no activation in our sheaf, maybe from sensors?
    if exp > 1: exp = 1

    if node.get_slot('sub').get_activation(sheaf) > 0:
        if sur > 0:
            netapi.change_modulator('base_number_of_expected_events', 1)
        elif sur < 0:
            severity = len(node.get_gate("sub").get_links()) + len(node.get_gate("cat").get_links())
            netapi.change_modulator('base_number_of_unexpected_events', severity)

    node.set_state("countdown", countdown)

    # set gates
    node.set_sheaf_activation(gen, sheaf)
    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("por").gate_function(por, sheaf)
    node.get_gate("ret").gate_function(ret, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)
    node.get_gate("exp").gate_function(exp, sheaf)
    node.get_gate("cat").gate_function(cat, sheaf)
    #if cat > 0 and node.get_slot("sub").get_activation(sheaf) > 0:     # cats will be checked in their own sheaf
    #    node.get_gate("cat").open_sheaf(cat, sheaf)
    #    node.get_gate("cat").gate_function(0, sheaf)
    #else:
    #    node.get_gate("cat").gate_function(cat, sheaf)

def trigger(netapi, node=None, sheaf="default", **params):
    """
    Trigger nodes, when activated on their sub slot, create a single burst of activation and wait for
    activation on their sur slot that equals or exceeds a configured value.
    If the value does not show up after a configured number of timeout steps, trigger nodes fail,
    else, they succeed.
    """

    gen = 0
    sub = 0
    sur = 0

    if abs(node.get_slot('gen').activation) != 0:
        sur = node.get_slot('gen').activation
    else:
        waitfrom = node.get_state("waitfrom") or -1

        condition = node.get_parameter("condition") or "="
        try:
            timeout = int(node.get_parameter("timeout"))         # todo: use emo_resolution
        except TypeError or ValueError:
            timeout = 10
        try:
            response = float(node.get_parameter("response"))
        except TypeError or ValueError:
            response = 1

        if node.get_slot('sub').activation > 0:
            currentstep = netapi.step

            success = (condition == "=" and node.get_slot('sur').activation == response) or \
                      (condition == ">" and node.get_slot('sur').activation > response)

            if waitfrom > 0:                       # waiting for results
                if success:
                    sur = 1
                    node.set_state("waitfrom", currentstep)
                elif currentstep > waitfrom + timeout or node.get_slot('sur').activation < 0:
                    sur = -1
            else:                                   # perform burst
                sub = 1
                node.set_state("waitfrom", currentstep)
                if success:
                    sur = 1

        else:

            if waitfrom > 0:
                node.set_state("waitfrom", -1)

            if node.get_slot('sur').activation:
                success = (condition == "=" and node.get_slot('sur').activation == response) or \
                          (condition == ">" and node.get_slot('sur').activation > response)
                if success:
                    sur = 1

    if sur < 0:
        # if trigger is failing, gen should also propagate failing
        gen = sur
    else:
        # otherwise, gen should be allowed to pass on the original sur-activation
        # and not be confined to {0;1}
        gen = node.get_slot('sur').activation + node.get_slot('gen').activation

    node.activation = gen

    node.get_gate("gen").gate_function(gen, sheaf)
    node.get_gate("sub").gate_function(sub, sheaf)
    node.get_gate("sur").gate_function(sur, sheaf)

def activator(netapi, node, **params):
    node.activation = node.get_slot("gen").activation
    netapi.get_nodespace(node.parent_nodespace).set_activator_value(node.get_parameter('type'), node.activation)
<|endoftext|>


<commit_before>
def register(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def sensor(nodenet, node=None, datasource=None, **params):
    node.gates["gen"].gate_function(nodenet.world.get_datasource(nodenet.uid, datasource))

def actor(nodenet, node=None, datatarget=None, **params):
    node.nodenet.world.set_datatarget(nodenet.uid, datatarget, node.get_slot("gen").activation)
    node.activation = node.get_slot("gen").activation

def concept(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def pipe(nodenet, node=None, **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0

    gen += node.get_slot("sur").activation
    if gen < 0: gen = 0
    if gen > 1: gen = 1

    sub += node.get_slot("gen").activation * 3
    sub += node.get_slot("sub").activation
    sub += node.get_slot("por").activation
    if sub == 0: sub = -1

    sur += node.get_slot("sur").activation * \
           (1+node.get_slot("sub").activation) * \
           (1+node.get_slot("ret").activation) * \
           (1+node.get_slot("por").activation)
    if sur < 0: sur = 0

    por += node.get_slot("sur").activation
    if por == 0: por = -1

    ret += node.get_slot("ret").activation
    if ret == 0: ret = -1

    node.get_gate("gen").gate_function(gen)
    node.get_gate("por").gate_function(por)
    node.get_gate("ret").gate_function(ret)
    node.get_gate("sub").gate_function(sub)
    node.get_gate("sur").gate_function(sur)

def label(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def event(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def activator(nodenet, node, **params):
    node.activation = node.get_slot("gen").activation
    nodenet.nodespaces[node.parent_nodespace].activators[node.parameters["type"]] = node.activation
<commit_msg>Limiting sub to prevent overpowering the  inhibitions and subbing motors to early<commit_after>
def register(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def sensor(nodenet, node=None, datasource=None, **params):
    node.gates["gen"].gate_function(nodenet.world.get_datasource(nodenet.uid, datasource))

def actor(nodenet, node=None, datatarget=None, **params):
    node.nodenet.world.set_datatarget(nodenet.uid, datatarget, node.get_slot("gen").activation)
    node.activation = node.get_slot("gen").activation

def concept(nodenet, node=None, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def pipe(nodenet, node=None, **params):
    gen = 0.0
    por = 0.0
    ret = 0.0
    sub = 0.0
    sur = 0.0

    gen += node.get_slot("sur").activation
    if gen < 0: gen = 0
    if gen > 1: gen = 1

    sub += node.get_slot("gen").activation * 3
    sub += node.get_slot("sub").activation
    sub += node.get_slot("por").activation
    if sub == 0: sub = -1
    if sub > 1: sub = 1

    sur += node.get_slot("sur").activation * \
           (1+node.get_slot("sub").activation) * \
           (1+node.get_slot("ret").activation) * \
           (1+node.get_slot("por").activation)
    if sur < 0: sur = 0

    por += node.get_slot("sur").activation
    if por == 0: por = -1

    ret += node.get_slot("ret").activation
    if ret == 0: ret = -1

    node.get_gate("gen").gate_function(gen)
    node.get_gate("por").gate_function(por)
    node.get_gate("ret").gate_function(ret)
    node.get_gate("sub").gate_function(sub)
    node.get_gate("sur").gate_function(sur)

def label(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def event(nodenet, node, **params):
    for type, gate in node.gates.items():
        gate.gate_function(node.get_slot("gen").activation)

def activator(nodenet, node, **params):
    node.activation = node.get_slot("gen").activation
    nodenet.nodespaces[node.parent_nodespace].activators[node.parameters["type"]] = node.activation
<|endoftext|>


<commit_before>import logging
import sys, os
import re
import os.path
import hashlib
import ConfigParser
from ConfigParser import SafeConfigParser

class DataPrep():
    def __init__(self, dataPath=None, dataRegex=None, shardNum=None, targetPath=None, configFile=None):
        self.dataPath=""
        self.dataRegex=""
        self.shardNum=0
        self.targetPath=""
        print "DATAPREP [DEBUG]: Preparing data for hdfs"
        if configFile:
            self.loadFromConfig(configFile)

        if dataPath:
            if not os.path.isdir(dataPath):
                raise ValueError("DATAPREP [ERROR]: source directory do not exist")
            self.dataPath = dataPath
        if shardNum:
            if (not isinstance(shardNum, int)) and int(shardNum)<=0 :
                raise ValueError("DATAPREP [ERROR]: shard number needs to be a positive integer")
            self.shardNum = shardNum
        if dataRegex:
		    self.dataRegex = dataRegex
        if targetPath:
		    self.targetPath = targetPath

        if self.dataPath=="":
            raise ValueError("DATAPREP [ERROR]: dataPath not specified")
        if self.dataRegex=="":
            raise ValueError("DATAPREP [ERROR]: dataRegex not specified")
        if self.shardNum==0:
            raise ValueError("DATAPREP [ERROR]: shardNumber not specified")
        if self.targetPath=="":
            raise ValueError("DATAPREP [ERROR]: targetPath not specified")

        print "DATAPREP [INFO]: Using data from path:" + self.dataPath

        sourcePath = self.dataPrep(self.dataPath, self.dataRegex, self.shardNum)
        self.dataUpload(sourcePath, self.targetPath)
        return

    def loadFromConfig(self, configFile):
        if not os.path.exists(configFile):
            raise ValueError("DATAPREP [ERROR]: config file does not exist")
        if os.path.isdir(configFile):
            raise ValueError("DATAPREP [ERROR]: config file leads to directory instead of file")

        print("DARAPREP [INFO]: Reading configurations from file: %s" % (configFile))
        cf = SafeConfigParser(os.environ)
        cf.read(configFile)

        self.dataRegex    =    cf.get("data",   "train")
        self.dataPath     =    cf.get("data",   "data_path")
        self.targetPath   =    cf.get("data",   "prep_path")
        self.shardNum     = cf.getint("option", "shards")
        return

    def sentCount(self, dataPath, dataRegex):
        '''
        :dataPath: the dir storing the training data
        :format: the format of the training data
        :return: the totalnumber of sentences
        '''
        count = 0
        sectionPattern = re.compile(dataRegex)

        for dirName, subdirList, fileList in os.walk(dataPath):
            for fileName in fileList:
                if sectionPattern.match(str(fileName)) != None:
                    filePath = "%s/%s" % ( str(dirName), str(fileName) )
                    with open(filePath,"r") as theFile:
                        for line in theFile:
                            if line == '\n':
                                count += 1
        return count

    def dataPrep(self, dataPath, dataRegex, shardNum, targetPath="../data/prep/"):
        '''
        :param dataPath: the input directory storing training data_pool
        :param shardNum: the number of partisions of data_pool
        :param targetPath: the output directory storing the sharded data_pool
        '''
        print "DATAPREP [DEBUG]: Partitioning Data locally"
        if not os.path.exists(targetPath):
            os.makedirs(targetPath)

        sectionPattern = re.compile(dataRegex)
        fileList = []

        for dirName, subdirList, fileList in os.walk(dataPath):
            for fileName in fileList:
                if sectionPattern.match(str(fileName)) != None:
                    filePath = "%s/%s" % ( str(dirName), str(fileName) )
                    print "DATAPREP [DEBUG]: Pendng file " + filePath
                    fileList.append(filePath)

        input_string = ''.join(fileList) + str(shardNum)
        folder = hashlib.md5(input_string).hexdigest()[:7]
        output_path = targetPath + folder + '/'

        if not os.path.exists(output_path):
            os.makedirs(output_path)

            fid = shardNum-1

            output_file = output_path + str(fid)
            fout = open(output_file,"w")

            count = 0

            n = self.sentCount(dataPath,dataRegex)/shardNum # number of sentences per shard

            for filePath in fileList:
                print "DATAPREP [DEBUG]: Opening file "+ filePath
                fin = open(filePath, "r")

                for line in fin:
                    if count == n and fid is not 0:
                        fid -= 1
                        fout.close()
                        output_file = output_path + str(fid)
                        fout = open(output_file,"w")
                        count = 0

                    fout.write(line)

                    if line == '\n':
                        count += 1

            fout.close()
            print "DATAPREP [DEBUG]: Partition complete"
        return output_path

    def dataUpload(self, sourcePath, targetPath="./data/prep/"):
        print "DATAPREP [DEBUG]: Uploading data to HDFS"
        print "DATAPREP [DEBUG]: Creating target directory " + targetPath
        os.system("hdfs dfs -mkdir -p " + targetPath)
        os.system("hdfs dfs -put -f %s %s"%(sourcePath,targetPath))
        print "DATAPREP [DEBUG]: Upload complete"
        return

if __name__ == "__main__":
    #dataPath   = sys.argv[1]
    #dataRegex  = sys.argv[2]
    #shardNum   = int(sys.argv[3])
    DataPrep(targetPath="data/meow/", shardNum=8, configFile=sys.argv[1])
<commit_msg>bug fix, data_prep.py ready<commit_after># -*- coding: utf-8 -*-

#
# Global Linear Model Parser
# Simon Fraser University
# NLP Lab
#
# Author: Vivian Kou, Sean La, Jetic Gu
# (Please add on your name if you have authored this file)
#
# Origin: src/learn/partition.py
#

#
# The DataPrep class aims to serve one purpose and one purpose only,
# to upload the data to hdfs when running the glm_parser on spark yarn
# cluster mode.
#
import logging
import sys, os
import re
import os.path
import hashlib
import ConfigParser
from ConfigParser import SafeConfigParser

class DataPrep():
    def __init__(self, dataPath=None, dataRegex=None, shardNum=None, targetPath=None, configFile=None, debug=False):
        self.dataPath=""
        self.dataRegex=""
        self.shardNum=0
        self.targetPath=""
        self.debug=debug
        if self.debug: print "DATAPREP [DEBUG]: Preparing data for hdfs"

        # Load config file
        if configFile:
            self.loadFromConfig(configFile)

        # Load params
        if dataPath:
            if not os.path.isdir(dataPath):
                raise ValueError("DATAPREP [ERROR]: source directory do not exist")
            self.dataPath = dataPath
        if shardNum:
            if (not isinstance(shardNum, int)) and int(shardNum)<=0 :
                raise ValueError("DATAPREP [ERROR]: shard number needs to be a positive integer")
            self.shardNum = shardNum
        if dataRegex:
		    self.dataRegex = dataRegex
        if targetPath:
		    self.targetPath = targetPath

        # Check param validity
        if self.dataPath=="":
            raise ValueError("DATAPREP [ERROR]: dataPath not specified")
        if self.dataRegex=="":
            raise ValueError("DATAPREP [ERROR]: dataRegex not specified")
        if self.shardNum==0:
            raise ValueError("DATAPREP [ERROR]: shardNumber not specified")
        if self.targetPath=="":
            raise ValueError("DATAPREP [ERROR]: targetPath not specified")

        print "DATAPREP [INFO]: Using data from path:" + self.dataPath

        sourcePath = self.dataPrep(self.dataPath, self.dataRegex, self.shardNum)
        self.dataUpload(sourcePath, self.targetPath)
        return

    def loadFromConfig(self, configFile):
        if not os.path.exists(configFile):
            raise ValueError("DATAPREP [ERROR]: config file does not exist")
        if os.path.isdir(configFile):
            raise ValueError("DATAPREP [ERROR]: config file leads to directory instead of file")

        print("DARAPREP [INFO]: Reading configurations from file: %s" % (configFile))
        cf = SafeConfigParser(os.environ)
        cf.read(configFile)

        self.dataRegex    =    cf.get("data",   "train")
        self.dataPath     =    cf.get("data",   "data_path")
        self.targetPath   =    cf.get("data",   "prep_path")
        self.shardNum     = cf.getint("option", "shards")
        return

    def sentCount(self, dataPath, dataRegex):
        '''
        :dataPath: the dir storing the training data
        :format: the format of the training data
        :return: the totalnumber of sentences
        '''
        count = 0
        sectionPattern = re.compile(dataRegex)

        for dirName, subdirList, fileList in os.walk(dataPath):
            for fileName in fileList:
                if sectionPattern.match(str(fileName)) != None:
                    filePath = "%s/%s" % ( str(dirName), str(fileName) )
                    with open(filePath,"r") as theFile:
                        for line in theFile:
                            if line == '\n':
                                count += 1
        return count

    def dataPrep(self, dataPath, dataRegex, shardNum, targetPath="../data/prep/"):
        '''
        :param dataPath: the input directory storing training data_pool
        :param shardNum: the number of partisions of data_pool
        :param targetPath: the output directory storing the sharded data_pool
        '''
        if self.debug: print "DATAPREP [DEBUG]: Partitioning Data locally"
        if not os.path.exists(targetPath):
            os.makedirs(targetPath)

        sectionPattern = re.compile(dataRegex)
        aFileList = []

        for dirName, subdirList, fileList in os.walk(dataPath):
            for fileName in fileList:
                if sectionPattern.match(str(fileName)) != None:
                    filePath = "%s/%s" % ( str(dirName), str(fileName) )
                    if self.debug: print "DATAPREP [DEBUG]: Pendng file " + filePath
                    aFileList.append(filePath)

        input_string = ''.join(aFileList) + str(shardNum)
        folder = hashlib.md5(input_string).hexdigest()[:7]
        output_path = targetPath + folder + '/'

        if not os.path.exists(output_path):
            os.makedirs(output_path)

            fid = shardNum-1

            output_file = output_path + str(fid)
            fout = open(output_file,"w")

            count = 0

            n = self.sentCount(dataPath,dataRegex)/shardNum # number of sentences per shard

            for filePath in aFileList:
                if self.debug: print "DATAPREP [DEBUG]: Opening file "+ filePath
                fin = open(filePath, "r")

                for line in fin:
                    if count == n and fid is not 0:
                        fid -= 1
                        fout.close()
                        output_file = output_path + str(fid)
                        fout = open(output_file,"w")
                        count = 0

                    fout.write(line)

                    if line == '\n':
                        count += 1

            fout.close()
            if self.debug: print "DATAPREP [DEBUG]: Partition complete"
        return output_path

    def dataUpload(self, sourcePath, targetPath="./data/prep/"):
        '''
        This function uploads the folder sourcePath to targetPath on hadoop.
        '''
        if self.debug: print "DATAPREP [DEBUG]: Uploading data to HDFS"
        if self.debug: print "DATAPREP [DEBUG]: Creating target directory " + targetPath
        os.system("hdfs dfs -mkdir -p " + targetPath)
        os.system("hdfs dfs -put -f %s %s"%(sourcePath,targetPath))
        if self.debug: print "DATAPREP [DEBUG]: Upload complete"
        return

# Uncomment the following code for tests

if __name__ == "__main__":
    #dataPath   = sys.argv[1]
    #dataRegex  = sys.argv[2]
    #shardNum   = int(sys.argv[3])
    DataPrep(targetPath="data/meow/", shardNum=8, configFile=sys.argv[1])
<|endoftext|>


<commit_before>from django.core.urlresolvers import reverse
from django.http.response import HttpResponseRedirect, Http404, HttpResponse
from django.shortcuts import render, get_list_or_404
from django.utils.translation import ugettext as _

from books.forms import BookForm
from common.models import BookType

def index(request):
    book_list = BookType.objects.all()
    args = {'book_list': book_list}
    if 'success_msg' in request.session:
        args['success_msg'] = {
            'book_added': _("The book was added successfully."),
            'book_edited': _("The book was edited successfully."),
            'book_removed': _("The book was removed successfully."),
            'books_removed': _("The books were removed successfully."),
        }[request.session['success_msg']]
        del request.session['success_msg']
    return render(request, 'books/index.html', args)

def add_book(request):
    if request.method == 'POST':
        form = BookForm(request.POST)
        if form.is_valid():
            form.save()
            request.session['success_msg'] = 'book_added'
            return HttpResponseRedirect(reverse('index'))
    else:
        form = BookForm()
    return render(request, 'books/add.html', {'form': form})

def edit_book(request, book_id):
    book = BookType.objects.get(id=book_id)
    if request.method == 'POST':
        form = BookForm(request.POST, instance=book)
        if form.is_valid():
            form.save()
            request.session['success_msg'] = 'book_edited'
            return HttpResponseRedirect(reverse('index'))
    else:
        form = BookForm(instance=book)
    return render(request, 'books/edit.html', {'form': form})

def remove_book(request, book_ids):
    book_list = get_list_or_404(BookType, id__in=book_ids.split(','))
    if request.method == 'POST':
        request.session['success_msg'] = 'book_removed' if len(book_list) == 1 else 'books_removed'
        book_list.delete()
        return HttpResponseRedirect(reverse('index'))
    else:
        return render(request, 'books/remove.html', {'book_list': book_list})

def bulk_actions(request, action_name):
    book_list = []
    if action_name == 'remove' and request.method == 'POST':
        for item in request.POST.lists():
            print(item[1])
            if item[1][0] == 'on':
                book_list.append(item[0][7:])
        return HttpResponseRedirect(reverse('books.views.remove_book', args=[",".join(book_list)]))
    else:
        raise Http404<commit_msg>Fix an error during removing multiple books<commit_after>from django.core.urlresolvers import reverse
from django.http.response import HttpResponseRedirect, Http404, HttpResponse
from django.shortcuts import render, get_list_or_404
from django.utils.translation import ugettext as _

from books.forms import BookForm
from common.models import BookType

def index(request):
    book_list = BookType.objects.all()
    args = {'book_list': book_list}
    if 'success_msg' in request.session:
        args['success_msg'] = {
            'book_added': _("The book was added successfully."),
            'book_edited': _("The book was edited successfully."),
            'book_removed': _("The book was removed successfully."),
            'books_removed': _("The books were removed successfully."),
        }[request.session['success_msg']]
        del request.session['success_msg']
    return render(request, 'books/index.html', args)

def add_book(request):
    if request.method == 'POST':
        form = BookForm(request.POST)
        if form.is_valid():
            form.save()
            request.session['success_msg'] = 'book_added'
            return HttpResponseRedirect(reverse('index'))
    else:
        form = BookForm()
    return render(request, 'books/add.html', {'form': form})

def edit_book(request, book_id):
    book = BookType.objects.get(id=book_id)
    if request.method == 'POST':
        form = BookForm(request.POST, instance=book)
        if form.is_valid():
            form.save()
            request.session['success_msg'] = 'book_edited'
            return HttpResponseRedirect(reverse('index'))
    else:
        form = BookForm(instance=book)
    return render(request, 'books/edit.html', {'form': form})

def remove_book(request, book_ids):
    book_list = BookType.objects.filter(id__in=book_ids.split(','))
    if not book_list:
        raise Http404
    if request.method == 'POST':
        request.session['success_msg'] = 'book_removed' if len(book_list) == 1 else 'books_removed'
        book_list.delete()
        return HttpResponseRedirect(reverse('index'))
    else:
        return render(request, 'books/remove.html', {'book_list': book_list})

def bulk_actions(request, action_name):
    book_list = []
    if action_name == 'remove' and request.method == 'POST':
        for item in request.POST.lists():
            print(item[1])
            if item[1][0] == 'on':
                book_list.append(item[0][7:])
        return HttpResponseRedirect(reverse('books.views.remove_book', args=[",".join(book_list)]))
    else:
        raise Http404<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

VERSION = (0, 10, 0)
__version__ = '.'.join(map(str, VERSION))
<commit_msg>django-content-editor 0.10.1<commit_after>from __future__ import absolute_import, unicode_literals

VERSION = (0, 10, 1)
__version__ = '.'.join(map(str, VERSION))
<|endoftext|>


<commit_before>import argparse
import os
import sys

from collections import defaultdict
from operator import itemgetter

from exceptions import *
from key import KeyFactory
from track import *

def num_candidates(key_counts, starting_key):
    key = KeyFactory.new(starting_key)
    same_key = key_counts[starting_key] - 1
    higher = key_counts.get(key.up_fifth, 0)
    lower = key_counts.get(key.down_fifth, 0)
    relative = key_counts.get(key.relative, 0)
    return same_key + higher + lower

def candidates(key_counts):
    candidate_counts = {}
    for name in key_counts.keys():
        candidate_counts[name] = num_candidates(key_counts, name)
    return candidate_counts

def fewest_candidates(key_counts):
    return sorted(candidates(key_counts).iteritems(), key=itemgetter(1))

def main():
    parser = argparse.ArgumentParser(
        description=(
            'Analyze all of the MP3s in a collection and report which '
            'keys are underrepresented; that is, which keys have the '
            'least options to mix out of.'))
    parser.add_argument('base_path')
    parser.add_argument('--bpm', type=int)
    parser.add_argument('--range', type=int)
    parser.add_argument('--verbose', action='store_true',
                        help='list all appropriate tracks')
    args = parser.parse_args()

    keys_found = defaultdict(int)

    for dirpath, dirnames, filenames in os.walk(args.base_path):
        for filename in filenames:
            if os.path.splitext(filename)[1].lower() != '.mp3':
                continue
            full_path = os.path.join(dirpath, filename)
            try:
                track = Track(full_path)
            except IOError:
                continue

            try:
                key = KeyFactory.new(track.key)
            except KeyNotFound:
                print "key missing from metadata for {}".format(full_path)
                continue

            if args.bpm and args.range:
                try:
                    bpm = track.bpm
                    if not track.tempo_compatible(
                            args.bpm, args.range):
                        continue
                except BpmNotFound:
                    print "no BPM: {}".format(full_path)
                    continue
            keys_found[str(key)] += 1
            if args.verbose:
                print str(track)

    track_count = sum(keys_found.values())
    print "\n==== SUMMARY ===="
    print "* {} tracks found to evaluate.".format(track_count)
    if track_count == 0:
        sys.exit(1)

    mixing_options = fewest_candidates(keys_found)
    fewest = mixing_options[0]
    most = mixing_options[-1]

    if args.bpm and args.range:
        print "* Tempos are between {}bpm and {}bpm.".format(
                args.bpm - args.range, args.bpm + args.range)
    print "* A {:5} track can be mixed into only   {:3} other tracks.".format(
        fewest[0], fewest[1])
    print "* A {:5} track can be mixed into any of {:3} other tracks.".format(
        most[0], most[1])
    print "\n"
    print "\n".join(["{}: {}".format(key[0], key[1])
            for key in mixing_options])

if __name__ == '__main__':
    main()

<commit_msg>clarify report output<commit_after>import argparse
import os
import sys

from collections import defaultdict
from operator import itemgetter

from exceptions import *
from key import KeyFactory
from track import *

def num_candidates(key_counts, starting_key):
    key = KeyFactory.new(starting_key)
    same_key = key_counts[starting_key] - 1
    higher = key_counts.get(key.up_fifth, 0)
    lower = key_counts.get(key.down_fifth, 0)
    relative = key_counts.get(key.relative, 0)
    return same_key + higher + lower

def candidates(key_counts):
    candidate_counts = {}
    for name in key_counts.keys():
        candidate_counts[name] = num_candidates(key_counts, name)
    return candidate_counts

def fewest_candidates(key_counts):
    return sorted(candidates(key_counts).iteritems(), key=itemgetter(1))

def main():
    parser = argparse.ArgumentParser(
        description=(
            'Analyze all of the MP3s in a collection and report which '
            'keys are underrepresented; that is, which keys have the '
            'least options to mix out of.'))
    parser.add_argument('base_path')
    parser.add_argument('--bpm', type=int)
    parser.add_argument('--range', type=int)
    parser.add_argument('--verbose', action='store_true',
                        help='list all appropriate tracks')
    args = parser.parse_args()

    keys_found = defaultdict(int)

    for dirpath, dirnames, filenames in os.walk(args.base_path):
        for filename in filenames:
            if os.path.splitext(filename)[1].lower() != '.mp3':
                continue
            full_path = os.path.join(dirpath, filename)
            try:
                track = Track(full_path)
            except IOError:
                continue

            try:
                key = KeyFactory.new(track.key)
            except KeyNotFound:
                print "key missing from metadata for {}".format(full_path)
                continue

            if args.bpm and args.range:
                try:
                    bpm = track.bpm
                    if not track.tempo_compatible(
                            args.bpm, args.range):
                        continue
                except BpmNotFound:
                    print "no BPM: {}".format(full_path)
                    continue
            keys_found[str(key)] += 1
            if args.verbose:
                print str(track)

    track_count = sum(keys_found.values())
    print "\n==== SUMMARY ===="
    print "* {} tracks found to evaluate.".format(track_count)
    if track_count == 0:
        sys.exit(1)

    mixing_options = fewest_candidates(keys_found)
    fewest = mixing_options[0]
    most = mixing_options[-1]

    if args.bpm and args.range:
        print "* Tempos are between {} bpm and {} bpm.".format(
                args.bpm - args.range, args.bpm + args.range)
    print "* A {:5} track can be mixed into only   {:3} other tracks.".format(
        fewest[0], fewest[1])
    print "* A {:5} track can be mixed into any of {:3} other tracks.".format(
        most[0], most[1])
    print "\n"
    print "\n".join(["{}: {}".format(key[0], key[1])
            for key in mixing_options])

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import argparse, dns.query, dns.resolver, dns.rdatatype, redis, socket, struct
from dns.exception import DNSException
from time import time
from sys import stderr

parser = argparse.ArgumentParser(description='Reverse DNS Scraper')
parser.add_argument('range', metavar='octet', type=int, nargs='+',
                   help='Specify first octet range')
parser.add_argument('--threading', action='store_true', help='Use threads instead of processes')
parser.add_argument('--debug', default=False, action='store_true', help='Print More Errors and Launch interactive console on exception or forced exit')
parser.add_argument('--octet', type=int, action='store', default=256)
parser.add_argument('--concurrent', type=int, action='store', default=500)
parser.add_argument('--timeout', type=float, action='store', default=.5)

arguments = parser.parse_args()

try:
    ip_start, ip_end = arguments.range
    octet_range = arguments.octet
    concurrent = arguments.concurrent
    debug = arguments.debug
    print "IP Range: %i - %i" % (ip_start, ip_end)
    print "Octet Range: %i" % octet_range
    print "Concurrent: %i" % concurrent
    print "Timeout: %f" % arguments.timeout
    ip_end += 1
except Exception as e:
    print >> stderr, e
    exit(1)

try:
    import redis
    r_server = redis.Redis('localhost')
except:
    print >> stderr, 'Could not connect to Redis'

default = dns.resolver.get_default_resolver()
default.timeout = arguments.timeout
default_ns = default.nameservers[0]

if arguments.threading:
    print 'Using Threading'
    from threading import Thread as Split
    from Queue import Queue
    from array import array as Array
else:
    print 'Using Multiprocessing'
    from multiprocessing import Process as Split
    from multiprocessing import JoinableQueue as Queue
    from multiprocessing import Array
    from multiprocessing import Manager
q = Queue()

def main():
    start = time()
    if arguments.threading:
        arr = Array('d', (0,)*concurrent)
        dictionary = dict()
    else:
        arr = Array('d', concurrent, lock=False)
        dictionary = Manager().dict()
    for i in range(concurrent):
        t=Split(target=doWork, args=(arr, i, dictionary))
        t.daemon=True
        t.start()
    ips = ("%i" % octet for octet in range(ip_start,ip_end))
    for ip in ips:
        q.put((ip, 1, default_ns))
    q.join()
    end = max(arr)
    print "Total Time: %f seconds" % (end - start)

def doWork(arr, id, dictionary):
    while True:
        try:
            ip, level, ns = q.get(timeout=1)
        except:
            continue

        addr, auth, add = lookup(ip, ns, level, arr, id)
        if not auth and not add:
            pass
        else:
            next_ns = processRecords(auth, add, level, dictionary)
            if level < 3 and next_ns:
                ips = ("%s.%i" % (prefix, octet) for octet in range(0,octet_range))
                for ip in ips:
                    q.put((ip, level+1, next_ns))
        q.task_done()

def lookupHost(host, level):
    if host:
        for i in range(5-level):
            try:
                return default.query(host).rrset[0].address
            except dns.exception.Timeout:
                if debug: print >> stderr, host, 'Timeout'
            except dns.resolver.NXDOMAIN:
                if debug: print >> stderr, host, 'NXDOMAIN'
            except dns.resolver.NoAnswer:
                if debug: print >> stderr, host, 'NoAnswer'
            except dns.resolver.NoNameservers:
                if debug: print >> stderr, host, 'NoNameservers'
            except Exception:
                print >> stderr, host, 'Unknown Error'
    return None

def lookup(ip, ns, level, arr, id):
    addr = ip2reverse(ip)
    query = dns.message.make_query(addr, dns.rdatatype.PTR)
    for i in range(5-level):
        try:
            response = dns.query.udp(query, ns, timeout=arguments.timeout)
            arr[id] = time()
            rcode = response.rcode()
            if rcode == dns.rcode.NOERROR:
                return addr, response.authority, response.additional
            else:
                return addr, None, None
        except dns.exception.Timeout:
            if debug: print >> stderr, 'Timeout, Count: %i, Level: %i' % (i, level)
        except dns.query.BadResponse:
            if debug: print >> stderr, 'Bad Response, Count: %i, Level: %i' % (i, level)
        except dns.query.UnexpectedSource:
            if debug: print >> stderr, 'Unexpected Source, Count: %i, Level: %i' % (i, level)
        except Exception:
            print >> stderr, 'Unknown Error, Count: %i, Level: %i' % (i, level)
    return addr, None, None

#############
# Utilities #
#############
def ip2int(addr):
    return struct.unpack("!L", socket.inet_aton(addr))[0]

def int2ip(addr):
    return socket.inet_ntoa(struct.pack("!L", addr))

def ip2reverse(ip):
    ip = ip.split('.')
    ip.reverse()
    return '%s.in-addr.arpa' % '.'.join(ip)

def processRecords(auth, add, level, dictionary):
    records = {}
    next_ns = None
    for rrset in auth:
        for rec in rrset:
            if rec.rdtype is dns.rdatatype.NS:
                host_name = str(rec).lower()
            elif rec.rdtype is dns.rdatatype.SOA:
                host_name = str(rec.mname).lower()
            else:
                continue

            if host_name in dictionary:
                next_ns = dictionary[host_name]
            else:
                next_ns = lookupHost(host_name, level)
                dictionary[host_name] = next_ns
                records[host_name] = next_ns
    for rrset in add:
        host_name = rrset.name.to_text().lower()
        for rec in rrset:
            if rec.rdtype is dns.rdatatype.A:
                ip = rec.to_text()
                dictionary[host_name] = ip
                records[host_name] = ip
    if records:
        try:
            insertDB(records)
        except Exception as e:
            print "DB Error", e
            print records

    return next_ns

def insertDB(records):
    pipe = r_server.pipeline()
    for name, ip in records.items():
        if ip:
            pipe.sadd(name, ip)
        else:
            pipe.sadd(name, '')
    if not pipe.execute():
        raise Exception

try:
    main()
except KeyboardInterrupt:
    if arguments.debug:
        from IPython import embed
        embed()
<commit_msg>fixed typo<commit_after>import argparse, dns.query, dns.resolver, dns.rdatatype, redis, socket, struct
from dns.exception import DNSException
from time import time
from sys import stderr

parser = argparse.ArgumentParser(description='Reverse DNS Scraper')
parser.add_argument('range', metavar='octet', type=int, nargs='+',
                   help='Specify first octet range')
parser.add_argument('--threading', action='store_true', help='Use threads instead of processes')
parser.add_argument('--debug', default=False, action='store_true', help='Print More Errors and Launch interactive console on exception or forced exit')
parser.add_argument('--octet', type=int, action='store', default=256)
parser.add_argument('--concurrent', type=int, action='store', default=500)
parser.add_argument('--timeout', type=float, action='store', default=.5)

arguments = parser.parse_args()

try:
    ip_start, ip_end = arguments.range
    octet_range = arguments.octet
    concurrent = arguments.concurrent
    debug = arguments.debug
    print "IP Range: %i - %i" % (ip_start, ip_end)
    print "Octet Range: %i" % octet_range
    print "Concurrent: %i" % concurrent
    print "Timeout: %f" % arguments.timeout
    ip_end += 1
except Exception as e:
    print >> stderr, e
    exit(1)

try:
    import redis
    r_server = redis.Redis('localhost')
except:
    print >> stderr, 'Could not connect to Redis'

default = dns.resolver.get_default_resolver()
default.timeout = arguments.timeout
default_ns = default.nameservers[0]

if arguments.threading:
    print 'Using Threading'
    from threading import Thread as Split
    from Queue import Queue
    from array import array as Array
else:
    print 'Using Multiprocessing'
    from multiprocessing import Process as Split
    from multiprocessing import JoinableQueue as Queue
    from multiprocessing import Array
    from multiprocessing import Manager
q = Queue()

def main():
    start = time()
    if arguments.threading:
        arr = Array('d', (0,)*concurrent)
        dictionary = dict()
    else:
        arr = Array('d', concurrent, lock=False)
        dictionary = Manager().dict()
    for i in range(concurrent):
        t=Split(target=doWork, args=(arr, i, dictionary))
        t.daemon=True
        t.start()
    ips = ("%i" % octet for octet in range(ip_start,ip_end))
    for ip in ips:
        q.put((ip, 1, default_ns))
    q.join()
    end = max(arr)
    print "Total Time: %f seconds" % (end - start)

def doWork(arr, id, dictionary):
    while True:
        try:
            ip, level, ns = q.get(timeout=1)
        except:
            continue

        addr, auth, add = lookup(ip, ns, level, arr, id)
        if not auth and not add:
            pass
        else:
            next_ns = processRecords(auth, add, level, dictionary)
            if level < 3 and next_ns:
                ips = ("%s.%i" % (ip, octet) for octet in range(0,octet_range))
                for ip in ips:
                    q.put((ip, level+1, next_ns))
        q.task_done()

def lookupHost(host, level):
    if host:
        for i in range(5-level):
            try:
                return default.query(host).rrset[0].address
            except dns.exception.Timeout:
                if debug: print >> stderr, host, 'Timeout'
            except dns.resolver.NXDOMAIN:
                if debug: print >> stderr, host, 'NXDOMAIN'
            except dns.resolver.NoAnswer:
                if debug: print >> stderr, host, 'NoAnswer'
            except dns.resolver.NoNameservers:
                if debug: print >> stderr, host, 'NoNameservers'
            except Exception:
                print >> stderr, host, 'Unknown Error'
    return None

def lookup(ip, ns, level, arr, id):
    addr = ip2reverse(ip)
    query = dns.message.make_query(addr, dns.rdatatype.PTR)
    for i in range(5-level):
        try:
            response = dns.query.udp(query, ns, timeout=arguments.timeout)
            arr[id] = time()
            rcode = response.rcode()
            if rcode == dns.rcode.NOERROR:
                return addr, response.authority, response.additional
            else:
                return addr, None, None
        except dns.exception.Timeout:
            if debug: print >> stderr, 'Timeout, Count: %i, Level: %i' % (i, level)
        except dns.query.BadResponse:
            if debug: print >> stderr, 'Bad Response, Count: %i, Level: %i' % (i, level)
        except dns.query.UnexpectedSource:
            if debug: print >> stderr, 'Unexpected Source, Count: %i, Level: %i' % (i, level)
        except Exception:
            print >> stderr, 'Unknown Error, Count: %i, Level: %i' % (i, level)
    return addr, None, None

#############
# Utilities #
#############
def ip2int(addr):
    return struct.unpack("!L", socket.inet_aton(addr))[0]

def int2ip(addr):
    return socket.inet_ntoa(struct.pack("!L", addr))

def ip2reverse(ip):
    ip = ip.split('.')
    ip.reverse()
    return '%s.in-addr.arpa' % '.'.join(ip)

def processRecords(auth, add, level, dictionary):
    records = {}
    next_ns = None
    for rrset in auth:
        for rec in rrset:
            if rec.rdtype is dns.rdatatype.NS:
                host_name = str(rec).lower()
            elif rec.rdtype is dns.rdatatype.SOA:
                host_name = str(rec.mname).lower()
            else:
                continue

            if host_name in dictionary:
                next_ns = dictionary[host_name]
            else:
                next_ns = lookupHost(host_name, level)
                dictionary[host_name] = next_ns
                records[host_name] = next_ns
    for rrset in add:
        host_name = rrset.name.to_text().lower()
        for rec in rrset:
            if rec.rdtype is dns.rdatatype.A:
                ip = rec.to_text()
                dictionary[host_name] = ip
                records[host_name] = ip
    if records:
        try:
            insertDB(records)
        except Exception as e:
            print "DB Error", e
            print records

    return next_ns

def insertDB(records):
    pipe = r_server.pipeline()
    for name, ip in records.items():
        if ip:
            pipe.sadd(name, ip)
        else:
            pipe.sadd(name, '')
    if not pipe.execute():
        raise Exception

try:
    main()
except KeyboardInterrupt:
    if arguments.debug:
        from IPython import embed
        embed()
<|endoftext|>