<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import unittest2 as unittest

from sensorimotor.one_d_world import OneDWorld
from sensorimotor.one_d_universe import OneDUniverse
from sensorimotor.random_one_d_agent import RandomOneDAgent

from sensorimotor.test.abstract_sensorimotor_test import (
  AbstractSensorimotorTest)

class SensorimotorTemporalMemoryTest(AbstractSensorimotorTest):

  VERBOSITY = 1
  DEFAULT_TM_PARAMS = {
    "columnDimensions": [100],
    "cellsPerColumn": 8,
    "initialPermanence": 0.5,
    "connectedPermanence": 0.6,
    "minThreshold": 15,
    "activationThreshold": 15,
    "maxNewSynapseCount": 50,
    "permanenceIncrement": 0.1,
    "permanenceDecrement": 0.02
  }

  def testSingleWorldOneBitPerPattern(self):
    """
    Test Sensorimotor Temporal Memory learning in a single world.
    Patterns (sensor and motor) are represented with one active bit per pattern.
    """
    self._init({"columnDimensions": [4],
                "minThreshold": 2,
                "activationThreshold": 2})

    universe = OneDUniverse(debugSensor=True, debugMotor=True,
                            nSensor=4, wSensor=1,
                            nMotor=3, wMotor=1)
    world = OneDWorld(universe, [0, 1, 2, 3], 2)
    agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-1, 2)))

    sequence = self._generateSensorimotorSequences(100, [agent])
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(20, [agent])
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testSingleWorldBasic(self):
    """
    Test Sensorimotor Temporal Memory learning in a single world.
    Patterns are represented as complete SDRs. No patterns are repeated.
    Prediction should be perfect.
    """
    self._init()

    universe = OneDUniverse(debugSensor=True, debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)
    world = OneDWorld(universe, [0, 1, 2, 3], 2)
    agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))

    sequence = self._generateSensorimotorSequences(100, [agent])
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, [agent])
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testMultipleWorldsBasic(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. No patterns are repeated.
    Prediction should be perfect.
    """
    self._init()

    universe = OneDUniverse(debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    numWorlds = 5
    sequenceLength = 4

    for i in xrange(numWorlds):
      start = i * sequenceLength
      patterns = range(start, start + sequenceLength)
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))
      agents.append(agent)

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testMultipleWorldsSharedPatterns(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. Patterns are shared between
    worlds.
    All active columns should have been predicted.
    """
    self._init()

    universe = OneDUniverse(debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    numWorlds = 5

    for i in xrange(numWorlds):
      patterns = range(4)
      self._random.shuffle(patterns)
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))
      agents.append(agent)

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self.assertTrue(0 < stats.predictedInactiveColumns.average < 10)

  @unittest.skip("TODO: Determine why this test fails")
  def testMultipleWorldsSharedPatternsNoSharedSubsequences(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. Patterns are shared between
    worlds. Worlds have no shared subsequences.
    All active columns should have been predicted.
    Patterns in different worlds should have different cell representations
    """
    self._init()

    universe = OneDUniverse(debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    patterns = range(4)
    for _ in xrange(2):
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))
      agents.append(agent)
      patterns = list(patterns)  # copy
      patterns.reverse()

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self.assertTrue(0 < stats.predictedInactiveColumns.average < 5)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)
    self._assertOneSequencePerPredictedActiveCell(stats)

  # ==============================
  # Helper functions
  # ==============================

  def _assertAllActiveWerePredicted(self, stats, universe):
    self.assertEqual(stats.unpredictedActiveColumns.sum, 0)

    self.assertEqual(stats.predictedActiveColumns.min, universe.wSensor)
    self.assertEqual(stats.predictedActiveColumns.max, universe.wSensor)

  def _assertAllInactiveWereUnpredicted(self, stats):
    self.assertEqual(stats.predictedInactiveColumns.sum, 0)

  def _assertAllActiveWereUnpredicted(self, stats, universe):
    self.assertEqual(stats.predictedActiveColumns.sum, 0)

    self.assertEqual(stats.unpredictedActiveColumns.min, universe.wSensor)
    self.assertEqual(stats.unpredictedActiveColumns.max, universe.wSensor)

  def _assertSequencesOnePredictedActiveCellPerColumn(self, stats):
    self.assertEqual(stats.sequencesPredictedActiveCellsPerColumn.min, 1)
    self.assertEqual(stats.sequencesPredictedActiveCellsPerColumn.max, 1)

  def _assertOneSequencePerPredictedActiveCell(self, stats):
    self.assertEqual(stats.sequencesPredictedActiveCellsShared.min, 1)
    self.assertEqual(stats.sequencesPredictedActiveCellsShared.max, 1)

if __name__ == "__main__":
  unittest.main()

<commit_msg>Fix test to make it pass<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import unittest2 as unittest

from sensorimotor.one_d_world import OneDWorld
from sensorimotor.one_d_universe import OneDUniverse
from sensorimotor.random_one_d_agent import RandomOneDAgent

from sensorimotor.test.abstract_sensorimotor_test import (
  AbstractSensorimotorTest)

class SensorimotorTemporalMemoryTest(AbstractSensorimotorTest):

  VERBOSITY = 1
  DEFAULT_TM_PARAMS = {
    "columnDimensions": [100],
    "cellsPerColumn": 8,
    "initialPermanence": 0.5,
    "connectedPermanence": 0.6,
    "minThreshold": 15,
    "activationThreshold": 15,
    "maxNewSynapseCount": 50,
    "permanenceIncrement": 0.1,
    "permanenceDecrement": 0.02
  }

  def testSingleWorldOneBitPerPattern(self):
    """
    Test Sensorimotor Temporal Memory learning in a single world.
    Patterns (sensor and motor) are represented with one active bit per pattern.
    """
    self._init({"columnDimensions": [4],
                "minThreshold": 2,
                "activationThreshold": 2})

    universe = OneDUniverse(debugSensor=True, debugMotor=True,
                            nSensor=4, wSensor=1,
                            nMotor=3, wMotor=1)
    world = OneDWorld(universe, [0, 1, 2, 3], 2)
    agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-1, 2)))

    sequence = self._generateSensorimotorSequences(100, [agent])
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(20, [agent])
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testSingleWorldBasic(self):
    """
    Test Sensorimotor Temporal Memory learning in a single world.
    Patterns are represented as complete SDRs. No patterns are repeated.
    Prediction should be perfect.
    """
    self._init()

    universe = OneDUniverse(debugSensor=True, debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)
    world = OneDWorld(universe, [0, 1, 2, 3], 2)
    agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))

    sequence = self._generateSensorimotorSequences(100, [agent])
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, [agent])
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testMultipleWorldsBasic(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. No patterns are repeated.
    Prediction should be perfect.
    """
    self._init()

    universe = OneDUniverse(debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    numWorlds = 5
    sequenceLength = 4

    for i in xrange(numWorlds):
      start = i * sequenceLength
      patterns = range(start, start + sequenceLength)
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))
      agents.append(agent)

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self._assertAllInactiveWereUnpredicted(stats)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)

  def testMultipleWorldsSharedPatterns(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. Patterns are shared between
    worlds.
    All active columns should have been predicted.
    """
    self._init()

    universe = OneDUniverse(debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    numWorlds = 5

    for i in xrange(numWorlds):
      patterns = range(4)
      self._random.shuffle(patterns)
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set(xrange(-3, 4)))
      agents.append(agent)

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self.assertTrue(0 < stats.predictedInactiveColumns.average < 10)

  def testMultipleWorldsSharedPatternsNoSharedSubsequences(self):
    """
    Test Sensorimotor Temporal Memory learning in multiple separate worlds.
    Patterns are represented as complete SDRs. Patterns are shared between
    worlds. Worlds have no shared subsequences.
    All active columns should have been predicted.
    Patterns in different worlds should have different cell representations
    """
    self._init()

    universe = OneDUniverse(debugSensor=True, debugMotor=True,
                            nSensor=100, wSensor=10,
                            nMotor=70, wMotor=10)

    agents = []
    patterns = range(4)
    for _ in xrange(2):
      world = OneDWorld(universe, patterns, 2)
      agent = RandomOneDAgent(world, possibleMotorValues=set([-3, -2, -1,
                                                              1,  2, 3]))
      agents.append(agent)
      patterns = list(patterns)  # copy
      patterns.reverse()

    sequence = self._generateSensorimotorSequences(150, agents)
    self._feedTM(sequence)

    sequence = self._generateSensorimotorSequences(100, agents)
    stats = self._testTM(sequence)

    self._assertAllActiveWerePredicted(stats, universe)
    self.assertTrue(0 < stats.predictedInactiveColumns.average < 5)
    self._assertSequencesOnePredictedActiveCellPerColumn(stats)
    self._assertOneSequencePerPredictedActiveCell(stats)

  # ==============================
  # Helper functions
  # ==============================

  def _assertAllActiveWerePredicted(self, stats, universe):
    self.assertEqual(stats.unpredictedActiveColumns.sum, 0)

    self.assertEqual(stats.predictedActiveColumns.min, universe.wSensor)
    self.assertEqual(stats.predictedActiveColumns.max, universe.wSensor)

  def _assertAllInactiveWereUnpredicted(self, stats):
    self.assertEqual(stats.predictedInactiveColumns.sum, 0)

  def _assertAllActiveWereUnpredicted(self, stats, universe):
    self.assertEqual(stats.predictedActiveColumns.sum, 0)

    self.assertEqual(stats.unpredictedActiveColumns.min, universe.wSensor)
    self.assertEqual(stats.unpredictedActiveColumns.max, universe.wSensor)

  def _assertSequencesOnePredictedActiveCellPerColumn(self, stats):
    self.assertEqual(stats.sequencesPredictedActiveCellsPerColumn.min, 1)
    self.assertEqual(stats.sequencesPredictedActiveCellsPerColumn.max, 1)

  def _assertOneSequencePerPredictedActiveCell(self, stats):
    self.assertEqual(stats.sequencesPredictedActiveCellsShared.min, 1)
    self.assertEqual(stats.sequencesPredictedActiveCellsShared.max, 1)

if __name__ == "__main__":
  unittest.main()

<|endoftext|>


<commit_before>"""Class to parse specifications for Dockerfile."""
# Author: Jakub Kaczmarzyk <jakubk@mit.edu>
from __future__ import absolute_import, division, print_function

from neurodocker import SUPPORTED_SOFTWARE
from neurodocker.utils import load_json

class SpecsParser(object):
    """Class to parse specifications for Dockerfile.

    Parameters
    ----------
    filepath : str
        Path to JSON file containing environment specifications.
    specs : dict
        Dictionary of environment specifications.
    """
    VALID_TOP_LEVEL_KEYS = ['base', 'pkg_manager']
    VALID_TOP_LEVEL_KEYS.extend(SUPPORTED_SOFTWARE.keys())

    def __init__(self, filepath=None, specs=None):
        if filepath is not None and specs is not None:
            raise ValueError("Specify either `filepath` or `specs`, not both.")
        elif filepath is not None:
            self.specs = load_json(filepath)
        elif specs is not None:
            self.specs = specs

        self.parse()

    def __str__(self):
        return str(self.specs)

    def parse(self):
        self._validate_keys()
        try:
            self._parse_conda_pip()
        except KeyError:
            pass

    def _validate_keys(self):
        """Raise KeyError if invalid top-level key(s)."""
        if 'base' not in self.specs.keys():
            raise KeyError("A base image must be specified in the key 'base'.")
        if 'pkg_manager' not in self.specs.keys():
            raise KeyError("The Linux package manager must be specified in the "
                           "key 'pkg_manager'.")
        invalid = set(self.specs) - set(self.VALID_TOP_LEVEL_KEYS)
        if invalid:
            invalid = ", ".join(invalid)
            valid = ", ".join(self.VALID_TOP_LEVEL_KEYS)
            raise KeyError("Unexpected top-level key(s) in input: {0}. Valid "
                           "keys are {1}."
                           "".format(invalid, valid))

    def _parse_conda_pip(self):
        """Parse packages to install with `conda` and/or `pip`."""
        for key, val in self.specs['miniconda'].items():
            if key == "python_version":
                continue
            if isinstance(val, (list, tuple)):
                self.specs['miniconda'][key] = ' '.join(val)
            if isinstance(val, str):
                self.specs['miniconda'][key] = val
<commit_msg>allow SpecsParser to accept dict or filepath<commit_after>"""Class to parse specifications for Dockerfile."""
# Author: Jakub Kaczmarzyk <jakubk@mit.edu>
from __future__ import absolute_import, division, print_function

from neurodocker import SUPPORTED_SOFTWARE
from neurodocker.utils import load_json

class SpecsParser(object):
    """Class to parse specifications for Dockerfile.

    Parameters
    ----------
    dict_or_filepath : dict, str
        The dictionary of specifications, or the name of the JSON file with the
        specifications.
    """
    VALID_TOP_LEVEL_KEYS = ['base', 'pkg_manager', 'check_urls']
    VALID_TOP_LEVEL_KEYS.extend(SUPPORTED_SOFTWARE.keys())

    def __init__(self, dict_or_filepath):
        try:
            self.specs = load_json(dict_or_filepath)
        except TypeError:
            self.specs = dict_or_filepath

        self.parse()

    def __str__(self):
        return str(self.specs)

    def parse(self):
        self._validate_keys()
        try:
            self._parse_conda_pip()
        except KeyError:
            pass

    def _validate_keys(self):
        """Raise KeyError if invalid top-level key(s)."""
        if 'base' not in self.specs.keys():
            raise KeyError("A base image must be specified in the key 'base'.")
        if 'pkg_manager' not in self.specs.keys():
            raise KeyError("The Linux package manager must be specified in the "
                           "key 'pkg_manager'.")

        invalid = set(self.specs) - set(self.VALID_TOP_LEVEL_KEYS)
        if invalid:
            invalid = ", ".join(invalid)
            valid = ", ".join(self.VALID_TOP_LEVEL_KEYS)
            raise KeyError("Unexpected top-level key(s) in input: {}. Valid "
                           "keys are {}.".format(invalid, valid))

    def _parse_conda_pip(self):
        """Parse packages to install with `conda` and/or `pip`."""
        for key, val in self.specs['miniconda'].items():
            if key == "python_version":
                continue
            if isinstance(val, (list, tuple)):
                self.specs['miniconda'][key] = ' '.join(val)
            if isinstance(val, str):
                self.specs['miniconda'][key] = val
<|endoftext|>


<commit_before>import itertools
import hashlib
import os
import stat
from neurogenesis.base import SimulationRun
from neurogenesis.util import Logger

class DynamicLine(): # TODO better name?
    def __init__(self):
        self.head_part = ""
        self.dynamic_part = ""
        self.tail_part = ""
        self.current_print_representation = ""
        self.parameter_representation = {}

    def get_current_value_tuple(self):
        return (self.head_part, self.current_print_representation, self.tail_part)
    def __str__(self):
        return self.head_part + self.current_print_representation + self.tail_part
    def __repr__(self):
        return self.__str__()

def demux_and_write_simulation(args):
    ini_file = args.inifile
    out_dir = args.outdir
    config = args.configName

    lines = []
    dynamic_lines = []
    simulation_runs = {}

    config_line = "# This is config %s\n" % (config)
    lines.append(config_line)
    with open(ini_file) as input_file:
        for row in input_file:
            if '{' in row.split('#')[0]: # ignore comments (T)
                line = create_dynamic_line(row)
                dynamic_lines.append(line)
                lines.append(line)
            else:
                lines.append(row)

    all_dynamic_lines = [line.dynamic_part for line in dynamic_lines]
    for perm in itertools.product(*all_dynamic_lines):
        run = SimulationRun()
        for idx, val in enumerate(perm):
            dynamic_lines[idx].current_print_representation = val
            run.parameters.append((dynamic_lines[idx].head_part.split()[0], val.strip()))
        hash = create_file_hash(lines)
        target_file = "run.sh"
        write_sim_data(args, lines, hash, target_file)
        run.hash = hash
        [run.config.append(line.get_current_value_tuple()) for line in dynamic_lines]
        run.path = out_dir + hash + "/"
        run.executable_path = out_dir + hash + "/" + target_file
        run.config_name= config
        simulation_runs[hash] = run
    Logger.info("Generated %s simulation configs." % (len(simulation_runs)))
    return simulation_runs

def write_sim_data(args, lines, hash, target_file):

    full_folder_path = check_and_create_folder(args.outdir, hash)
    write_ini(full_folder_path, lines)
    create_bash_script(args, full_folder_path,target_file)
    write_additional_files(args, full_folder_path)

def create_file_hash(lines):
    hash = hashlib.md5()
    [hash.update(str(line).encode('utf-8')) for line in lines]
    return hash.hexdigest()

def write_additional_files(args, folder_path):
    files = args.additionalFiles
    for file in files:
        base_name = os.path.basename(file)
        new_file_path = folder_path + '/'+ base_name
        f = check_and_create_file(new_file_path)
        with open(file) as input_file:
            for row in input_file:
                f.write(row)
            f.close()
        input_file.close()

def write_ini(folder_path, file):
    full_path = folder_path + "/omnetpp.ini"
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    for line in file:
        f.write(str(line))
    f.close()

def check_and_create_folder(base_path, folder_name):
    full_path = base_path + folder_name
    if not os.path.exists(full_path):
        os.makedirs(full_path)
    return full_path

def check_and_create_file(full_path):
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    return f

def create_bash_script(args, target_folder, target_file):

    omnet_exec = args.omnetdir
    inet_dir = args.inetdir
    config_name = args.configName

    script = """
    #!/bin/bash
    DIR=%s
    TARGET=%s
    CONFIG=%s
    cd $DIR
    %s -u Cmdenv -l $DIR/INET -c $CONFIG -n $DIR/inet:$DIR/../tutorials:$DIR/../examples:$DIR/../examples:$TARGET/ $TARGET/omnetpp.ini > /dev/null
    rc=$?
    if [ $rc -gt 0 ]; then
        echo There is something wrong! omnet exited with non 0 exit code!
        exit $rc
    fi
    """ % (inet_dir[:-1], target_folder, config_name, omnet_exec)
    full_path = target_folder + "/" + target_file
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    f.write(script)
    f.close()
    file_handle = os.stat(full_path)
    os.chmod(full_path, file_handle.st_mode | stat.S_IEXEC)

def create_dynamic_line(line):
    dline = DynamicLine()
    head_tokens  = line.split('{')
    if head_tokens[0][-1] == "$":
        dline.head_part = head_tokens[0] + '{'
        tail_tokens = head_tokens[1].split('}')

        if( '=' in head_tokens[1]): #assignment
            assignemt = head_tokens[1].split('=')
            dline.head_part += assignemt[0] + " = "# puts the variable name at the beginning
            dynamic_parts = assignemt[1].split(",")
            dynamic_parts[-1] = dynamic_parts[-1].split('}')[0]
            dline.dynamic_part = dynamic_parts
            dline.tail_part = '}' + tail_tokens[1]
        else:
            dline.head_part = line
            dline.dynamic_part = []
            dline.tail_part = ""
    else: #legacy config syntax
        dline.head_part = head_tokens[0]
        tail_tokens = head_tokens[1].split('}')
        dline.dynamic_part = tail_tokens[0].split(',')
        dline.tail_part = tail_tokens[1]
    return dline

def parse_tokens(tokens):
    more_tokens = []
    for token in tokens:
        if "=" in token: # assignment
            assignment_tokens = token.split("=")
            more_tokens.append(Token(0, TokenTypes.SYMBOL), )

def dynamic_line_range(line):
    #has '..', optional step parameter
    pass
<commit_msg>fixed generator condition<commit_after>import itertools
import hashlib
import os
import stat
from neurogenesis.base import SimulationRun
from neurogenesis.util import Logger

class DynamicLine(): # TODO better name?
    def __init__(self):
        self.head_part = ""
        self.dynamic_part = ""
        self.tail_part = ""
        self.current_print_representation = ""
        self.parameter_representation = {}

    def get_current_value_tuple(self):
        return (self.head_part, self.current_print_representation, self.tail_part)
    def __str__(self):
        return self.head_part + self.current_print_representation + self.tail_part
    def __repr__(self):
        return self.__str__()

def demux_and_write_simulation(args):
    ini_file = args.inifile
    out_dir = args.outdir
    config = args.configName

    lines = []
    dynamic_lines = []
    simulation_runs = {}

    config_line = "# This is config %s\n" % (config)
    lines.append(config_line)
    with open(ini_file) as input_file:
        for row in input_file:
            if '{' in row.split('#')[0]: # ignore comments (T)
                line = create_dynamic_line(row)
                dynamic_lines.append(line)
                lines.append(line)
            else:
                lines.append(row)

    all_dynamic_lines = [ line.dynamic_part for line in dynamic_lines if len(line.dynamic_part) > 0]
    for perm in itertools.product(*all_dynamic_lines):
        run = SimulationRun()
        for idx, val in enumerate(perm):
            dynamic_lines[idx].current_print_representation = val
            run.parameters.append((dynamic_lines[idx].head_part.split()[0], val.strip()))
        hash = create_file_hash(lines)
        target_file = "run.sh"
        write_sim_data(args, lines, hash, target_file)
        run.hash = hash
        [run.config.append(line.get_current_value_tuple()) for line in dynamic_lines]
        run.path = out_dir + hash + "/"
        run.executable_path = out_dir + hash + "/" + target_file
        run.config_name= config
        simulation_runs[hash] = run
    Logger.info("Generated %s simulation configs." % (len(simulation_runs)))
    return simulation_runs

def write_sim_data(args, lines, hash, target_file):

    full_folder_path = check_and_create_folder(args.outdir, hash)
    write_ini(full_folder_path, lines)
    create_bash_script(args, full_folder_path,target_file)
    write_additional_files(args, full_folder_path)

def create_file_hash(lines):
    hash = hashlib.md5()
    [hash.update(str(line).encode('utf-8')) for line in lines]
    return hash.hexdigest()

def write_additional_files(args, folder_path):
    files = args.additionalFiles
    for file in files:
        base_name = os.path.basename(file)
        new_file_path = folder_path + '/'+ base_name
        f = check_and_create_file(new_file_path)
        with open(file) as input_file:
            for row in input_file:
                f.write(row)
            f.close()
        input_file.close()

def write_ini(folder_path, file):
    full_path = folder_path + "/omnetpp.ini"
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    for line in file:
        f.write(str(line))
    f.close()

def check_and_create_folder(base_path, folder_name):
    full_path = base_path + folder_name
    if not os.path.exists(full_path):
        os.makedirs(full_path)
    return full_path

def check_and_create_file(full_path):
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    return f

def create_bash_script(args, target_folder, target_file):

    omnet_exec = args.omnetdir
    inet_dir = args.inetdir
    config_name = args.configName

    script = """
    #!/bin/bash
    DIR=%s
    TARGET=%s
    CONFIG=%s
    cd $DIR
    %s -u Cmdenv -l $DIR/INET -c $CONFIG -n $DIR/inet:$DIR/../tutorials:$DIR/../examples:$DIR/../examples:$TARGET/ $TARGET/omnetpp.ini > /dev/null
    rc=$?
    if [ $rc -gt 0 ]; then
        echo There is something wrong! omnet exited with non 0 exit code!
        exit $rc
    fi
    """ % (inet_dir[:-1], target_folder, config_name, omnet_exec)
    full_path = target_folder + "/" + target_file
    if os.path.exists(full_path):
        os.remove(full_path)
    f = open (full_path, "a")
    f.write(script)
    f.close()
    file_handle = os.stat(full_path)
    os.chmod(full_path, file_handle.st_mode | stat.S_IEXEC)

def create_dynamic_line(line):
    dline = DynamicLine()
    head_tokens  = line.split('{')
    if head_tokens[0][-1] == "$":
        dline.head_part = head_tokens[0] + '{'
        tail_tokens = head_tokens[1].split('}')

        if( '=' in head_tokens[1]): #assignment
            assignemt = head_tokens[1].split('=')
            dline.head_part += assignemt[0] + " = "# puts the variable name at the beginning
            dynamic_parts = assignemt[1].split(",")
            dynamic_parts[-1] = dynamic_parts[-1].split('}')[0]
            dline.dynamic_part = dynamic_parts
            dline.tail_part = '}' + tail_tokens[1]
        else:
            dline.head_part = line
            dline.dynamic_part = []
            dline.tail_part = ""
    else: #legacy config syntax
        dline.head_part = head_tokens[0]
        tail_tokens = head_tokens[1].split('}')
        dline.dynamic_part = tail_tokens[0].split(',')
        dline.tail_part = tail_tokens[1]
    return dline

def parse_tokens(tokens):
    more_tokens = []
    for token in tokens:
        if "=" in token: # assignment
            assignment_tokens = token.split("=")
            more_tokens.append(Token(0, TokenTypes.SYMBOL), )

def dynamic_line_range(line):
    #has '..', optional step parameter
    pass
<|endoftext|>


<commit_before>from untwisted.network import SuperSocket, die
from untwisted.client import Client, CONNECT, CONNECT_ERR
from untwisted import core
import errno

def on_connect(con, addr, port):
    print('Connected to %s:%s !' % (addr, port))

def on_connect_err(con, err, addr, port):
    print("Failed to connect to %s:%s, errcode " % (addr, port), errno.errorcode[err])

def create_connection(addr, port):
    con = SuperSocket()
    Client(con)
    con.add_map(CONNECT, on_connect, addr, port)
    con.add_map(CONNECT_ERR, on_connect_err, addr, port)
    con.add_map(CONNECT, lambda con: die())
    con.add_map(CONNECT_ERR, lambda con, err: die())

    con.connect_ex((addr, port))

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--addr', help='Address')
    parser.add_argument('-p', '--port', type=int, help='Port')
    args = parser.parse_args()

    create_connection(args.addr, args.port)
    core.gear.mainloop()

<commit_msg>Fixing.<commit_after>from untwisted.network import SuperSocket
from untwisted.client import Client, CONNECT, CONNECT_ERR
from untwisted.core import die
from untwisted import core
import errno

def on_connect(con, addr, port):
    print('Connected to %s:%s !' % (addr, port))

def on_connect_err(con, err, addr, port):
    print("Failed to connect to %s:%s, errcode " % (addr, port), errno.errorcode[err])

def create_connection(addr, port):
    con = SuperSocket()
    Client(con)
    con.add_map(CONNECT, on_connect, addr, port)
    con.add_map(CONNECT_ERR, on_connect_err, addr, port)
    con.add_map(CONNECT, lambda con: die())
    con.add_map(CONNECT_ERR, lambda con, err: die())

    con.connect_ex((addr, port))
    print('connecting')

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--addr', help='Address')
    parser.add_argument('-p', '--port', type=int, help='Port')
    args = parser.parse_args()

    create_connection(args.addr, args.port)
    core.gear.mainloop()

<|endoftext|>


<commit_before>"""
Django settings for chef_buddy_django project.
For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Simplified static file serving.
# https://warehouse.python.org/project/whitenoise/

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ['SECRET_KEY']

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'chef_buddy',
    'rest_framework',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'chef_buddy_django.urls'

WSGI_APPLICATION = 'chef_buddy_django.wsgi.application'

# if not os.environ.get("HOME") == '/Users/johnmitsch':
#     import dj_database_url
#     DATABASES['default'] =  dj_database_url.config()
#     # Enable Connection Pooling
#     DATABASES['default']['ENGINE'] = 'django_postgrespool'

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_ROOT = 'staticfiles'
STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static'),
)

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
    )
}<commit_msg>final changes to new scoring system are now complete<commit_after>"""
Django settings for chef_buddy_django project.
For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Simplified static file serving.
# https://warehouse.python.org/project/whitenoise/

STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ['SECRET_KEY']

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

AFTER_RESPONSE_RUN_ASYNC = False

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_extensions',
    'chef_buddy',
    'rest_framework',
    'after_response'
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'chef_buddy_django.urls'

WSGI_APPLICATION = 'chef_buddy_django.wsgi.application'

# if not os.environ.get("HOME") == '/Users/johnmitsch':
#     import dj_database_url
#     DATABASES['default'] =  dj_database_url.config()
#     # Enable Connection Pooling
#     DATABASES['default']['ENGINE'] = 'django_postgrespool'

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_ROOT = 'staticfiles'
STATIC_URL = '/static/'

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, 'static'),
)

REST_FRAMEWORK = {
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
    )
}
<|endoftext|>


<commit_before>"""
API Client Library for Salesforce.com (SFDC)
"""
from django.conf import settings

import simple_salesforce as sfapi
from django_statsd.clients import statsd
from simple_salesforce.api import DEFAULT_API_VERSION

from news.backends.common import get_timer_decorator
from news.newsletters import newsletter_map, newsletter_inv_map

time_request = get_timer_decorator('news.backends.sfdc')
LAST_NAME_DEFAULT_VALUE = '_'
FIELD_MAP = {
    'id': 'Id',
    'email': 'Email',
    'first_name': 'FirstName',
    'last_name': 'LastName',
    'format': 'Email_Format__c',
    'country': 'MailingCountryCode',
    'lang': 'Email_Language__c',
    'token': 'Token__c',
    'optin': 'Double_Opt_In__c',
    'optout': 'HasOptedOutOfEmail',
    'source_url': 'Signup_Source_URL__c',
    'created_date': 'CreatedDate',
    'last_modified_date': 'LastModifiedDate',
    'reason': 'Unsubscribe_Reason__c',
    'fsa_school': 'FSA_School__c',
    'fsa_grad_year': 'FSA_Grad_Year__c',
    'fsa_major': 'FSA_Major__c',
    'fsa_city': 'MailingCity',
    'fsa_current_status': 'FSA_Current_Status__c',
    'fsa_allow_share': 'FSA_Allow_Info_Shared__c',
}
INV_FIELD_MAP = {v: k for k, v in FIELD_MAP.items()}
FIELD_DEFAULTS = {
    'format': 'H',
    'country': '',
    'lang': '',
}
FIELD_PROCESSORS = {
    'country': lambda x: x.lower(),
    'lang': lambda x: x.lower(),
}
FIELD_MAX_LENGTHS = {
    'FirstName': 40,
    'LastName': 80,
    'Browser_Locale__c': 10,
    'Signup_Source_URL__c': 255,
    'Unsubscribe_Reason__c': 1000,
    'FSA_School__c': 100,
    'FSA_Grad_Year__c': 4,
    'FSA_Major__c': 100,
    'MailingCity': 100,
}

def to_vendor(data):
    """
    Take data received by basket and convert it to be sent to SFDC

    @param data: dict data received
    @return:
    """
    contact = {
        # True for every contact moving through basket
        'Subscriber__c': True,
    }
    for k, v in data.iteritems():
        if k in FIELD_MAP:
            contact[FIELD_MAP[k]] = v

    news_map = newsletter_map()
    newsletters = data.get('newsletters', None)
    if newsletters:
        if isinstance(newsletters, dict):
            # we got newsletter slugs with boolean values
            for k, v in newsletters.items():
                try:
                    contact[news_map[k]] = v
                except KeyError:
                    pass
        else:
            # we got a list of slugs for subscriptions
            for nl in newsletters:
                try:
                    contact[news_map[nl]] = True
                except KeyError:
                    pass

    # truncate long data
    for field, length in FIELD_MAX_LENGTHS.items():
        if field in contact and len(contact[field]) > length:
            statsd.incr('news.backends.sfdc.data_truncated')
            contact[field] = contact[field][:length]

    return contact

def from_vendor(contact):
    """
    Take contact data retrieved from SFDC and convert it for ease of use

    @param contact: contact data from SFDC
    @return:
    """
    news_map = newsletter_inv_map()
    data = {}
    newsletters = []
    for fn, fv in contact.iteritems():
        if fn in INV_FIELD_MAP:
            data_name = INV_FIELD_MAP[fn]
            if data_name in FIELD_DEFAULTS:
                fv = fv or FIELD_DEFAULTS[data_name]
            if data_name in FIELD_PROCESSORS:
                data[data_name] = FIELD_PROCESSORS[data_name](fv)
            else:
                data[data_name] = fv
        elif fn in news_map and fv:
            newsletters.append(news_map[fn])

    data['newsletters'] = newsletters
    return data

def get_sf_session():
    return sfapi.SalesforceLogin(**settings.SFDC_SETTINGS)

class RefreshingSFType(sfapi.SFType):
    def _call_salesforce(self, method, url, **kwargs):
        try:
            resp = super(RefreshingSFType, self)._call_salesforce(method, url, **kwargs)
        except sfapi.SalesforceExpiredSession:
            self.session_id, _ = get_sf_session()
            resp = super(RefreshingSFType, self)._call_salesforce(method, url, **kwargs)
            statsd.incr('news.backends.sfdc.session_expired')

        if 'sforce-limit-info' in resp.headers:
            try:
                usage, limit = resp.headers['sforce-limit-info'].split('=')[1].split('/')
            except Exception:
                usage = limit = None

            if usage:
                percentage = float(usage) / float(limit) * 100
                statsd.gauge('news.backends.sfdc.percent_daily_api_used', percentage, rate=0.5)

        return resp

class SFDC(object):
    contact = None

    def __init__(self):
        if not settings.SFDC_SETTINGS.get('username'):
            return

        session_id, sf_instance = get_sf_session()
        self.contact = RefreshingSFType('Contact', session_id, sf_instance,
                                        DEFAULT_API_VERSION)

    @time_request
    def get(self, token=None, email=None):
        """
        Get a contact record.

        @param token: external ID
        @param email: email address
        @return: dict
        """
        assert token or email, 'token or email is required'
        id_field = FIELD_MAP['token' if token else 'email']
        contact = self.contact.get_by_custom_id(id_field, token or email)
        return from_vendor(contact)

    @time_request
    def add(self, data):
        """
        Create a contact record.

        @param data: user data to add as a new contact.
        @return: None
        """
        data.setdefault('last_name', LAST_NAME_DEFAULT_VALUE)
        self.contact.create(to_vendor(data))

    @time_request
    def update(self, record, data):
        """
        Update data in an existing record.

        @param record: current contact record
        @param data: dict of user data
        @return: None
        """
        # need a copy because we'll modify it
        data = data.copy()
        if 'id' in record:
            contact_id = record['id']
        elif 'token' in record or 'email' in record:
            fn = 'token' if 'token' in record else 'email'
            contact_id = '{}/{}'.format(FIELD_MAP[fn], record[fn])
            # can't send the ID field in the data
            data.pop(fn, None)
        else:
            raise KeyError('id, token, or email required')

        self.contact.update(contact_id, to_vendor(data))

    @time_request
    def delete(self, record):
        """
        Delete a contact record.

        @param record: current contact record
        @return: None
        """
        self.contact.delete(record['id'])

sfdc = SFDC()
<commit_msg>Change last name default to a comma<commit_after>"""
API Client Library for Salesforce.com (SFDC)
"""
from django.conf import settings

import simple_salesforce as sfapi
from django_statsd.clients import statsd
from simple_salesforce.api import DEFAULT_API_VERSION

from news.backends.common import get_timer_decorator
from news.newsletters import newsletter_map, newsletter_inv_map

time_request = get_timer_decorator('news.backends.sfdc')
LAST_NAME_DEFAULT_VALUE = ','
FIELD_MAP = {
    'id': 'Id',
    'email': 'Email',
    'first_name': 'FirstName',
    'last_name': 'LastName',
    'format': 'Email_Format__c',
    'country': 'MailingCountryCode',
    'lang': 'Email_Language__c',
    'token': 'Token__c',
    'optin': 'Double_Opt_In__c',
    'optout': 'HasOptedOutOfEmail',
    'source_url': 'Signup_Source_URL__c',
    'created_date': 'CreatedDate',
    'last_modified_date': 'LastModifiedDate',
    'reason': 'Unsubscribe_Reason__c',
    'fsa_school': 'FSA_School__c',
    'fsa_grad_year': 'FSA_Grad_Year__c',
    'fsa_major': 'FSA_Major__c',
    'fsa_city': 'MailingCity',
    'fsa_current_status': 'FSA_Current_Status__c',
    'fsa_allow_share': 'FSA_Allow_Info_Shared__c',
}
INV_FIELD_MAP = {v: k for k, v in FIELD_MAP.items()}
FIELD_DEFAULTS = {
    'format': 'H',
    'country': '',
    'lang': '',
}
FIELD_PROCESSORS = {
    'country': lambda x: x.lower(),
    'lang': lambda x: x.lower(),
}
FIELD_MAX_LENGTHS = {
    'FirstName': 40,
    'LastName': 80,
    'Browser_Locale__c': 10,
    'Signup_Source_URL__c': 255,
    'Unsubscribe_Reason__c': 1000,
    'FSA_School__c': 100,
    'FSA_Grad_Year__c': 4,
    'FSA_Major__c': 100,
    'MailingCity': 100,
}

def to_vendor(data):
    """
    Take data received by basket and convert it to be sent to SFDC

    @param data: dict data received
    @return:
    """
    contact = {
        # True for every contact moving through basket
        'Subscriber__c': True,
    }
    for k, v in data.iteritems():
        if k in FIELD_MAP:
            contact[FIELD_MAP[k]] = v

    news_map = newsletter_map()
    newsletters = data.get('newsletters', None)
    if newsletters:
        if isinstance(newsletters, dict):
            # we got newsletter slugs with boolean values
            for k, v in newsletters.items():
                try:
                    contact[news_map[k]] = v
                except KeyError:
                    pass
        else:
            # we got a list of slugs for subscriptions
            for nl in newsletters:
                try:
                    contact[news_map[nl]] = True
                except KeyError:
                    pass

    # truncate long data
    for field, length in FIELD_MAX_LENGTHS.items():
        if field in contact and len(contact[field]) > length:
            statsd.incr('news.backends.sfdc.data_truncated')
            contact[field] = contact[field][:length]

    return contact

def from_vendor(contact):
    """
    Take contact data retrieved from SFDC and convert it for ease of use

    @param contact: contact data from SFDC
    @return:
    """
    news_map = newsletter_inv_map()
    data = {}
    newsletters = []
    for fn, fv in contact.iteritems():
        if fn in INV_FIELD_MAP:
            data_name = INV_FIELD_MAP[fn]
            if data_name in FIELD_DEFAULTS:
                fv = fv or FIELD_DEFAULTS[data_name]
            if data_name in FIELD_PROCESSORS:
                data[data_name] = FIELD_PROCESSORS[data_name](fv)
            else:
                data[data_name] = fv
        elif fn in news_map and fv:
            newsletters.append(news_map[fn])

    data['newsletters'] = newsletters
    return data

def get_sf_session():
    return sfapi.SalesforceLogin(**settings.SFDC_SETTINGS)

class RefreshingSFType(sfapi.SFType):
    def _call_salesforce(self, method, url, **kwargs):
        try:
            resp = super(RefreshingSFType, self)._call_salesforce(method, url, **kwargs)
        except sfapi.SalesforceExpiredSession:
            self.session_id, _ = get_sf_session()
            resp = super(RefreshingSFType, self)._call_salesforce(method, url, **kwargs)
            statsd.incr('news.backends.sfdc.session_expired')

        if 'sforce-limit-info' in resp.headers:
            try:
                usage, limit = resp.headers['sforce-limit-info'].split('=')[1].split('/')
            except Exception:
                usage = limit = None

            if usage:
                percentage = float(usage) / float(limit) * 100
                statsd.gauge('news.backends.sfdc.percent_daily_api_used', percentage, rate=0.5)

        return resp

class SFDC(object):
    contact = None

    def __init__(self):
        if not settings.SFDC_SETTINGS.get('username'):
            return

        session_id, sf_instance = get_sf_session()
        self.contact = RefreshingSFType('Contact', session_id, sf_instance,
                                        DEFAULT_API_VERSION)

    @time_request
    def get(self, token=None, email=None):
        """
        Get a contact record.

        @param token: external ID
        @param email: email address
        @return: dict
        """
        assert token or email, 'token or email is required'
        id_field = FIELD_MAP['token' if token else 'email']
        contact = self.contact.get_by_custom_id(id_field, token or email)
        return from_vendor(contact)

    @time_request
    def add(self, data):
        """
        Create a contact record.

        @param data: user data to add as a new contact.
        @return: None
        """
        data.setdefault('last_name', LAST_NAME_DEFAULT_VALUE)
        self.contact.create(to_vendor(data))

    @time_request
    def update(self, record, data):
        """
        Update data in an existing record.

        @param record: current contact record
        @param data: dict of user data
        @return: None
        """
        # need a copy because we'll modify it
        data = data.copy()
        if 'id' in record:
            contact_id = record['id']
        elif 'token' in record or 'email' in record:
            fn = 'token' if 'token' in record else 'email'
            contact_id = '{}/{}'.format(FIELD_MAP[fn], record[fn])
            # can't send the ID field in the data
            data.pop(fn, None)
        else:
            raise KeyError('id, token, or email required')

        self.contact.update(contact_id, to_vendor(data))

    @time_request
    def delete(self, record):
        """
        Delete a contact record.

        @param record: current contact record
        @return: None
        """
        self.contact.delete(record['id'])

sfdc = SFDC()
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *

from news_sitemaps import registry

urlpatterns = patterns('news_sitemaps.views',
    url(r'^index\.xml$',
        'index',
        {'sitemaps': registry},
        name='news_sitemaps_index'),

    url(r'^(?P<section>.+)\.xml',
        'news_sitemap',
        {'sitemaps': registry},
        name='news_sitemaps_sitemap'),
)
<commit_msg>URLs: End regex with $<commit_after>from django.conf.urls.defaults import *

from news_sitemaps import registry

urlpatterns = patterns('news_sitemaps.views',
    url(r'^index\.xml$',
        'index',
        {'sitemaps': registry},
        name='news_sitemaps_index'),

    url(r'^(?P<section>.+)\.xml$',
        'news_sitemap',
        {'sitemaps': registry},
        name='news_sitemaps_sitemap'),
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Holds the code for cleaning out unwanted tags from the lxml
dom xpath.
"""
from .utils import ReplaceSequence

class DocumentCleaner(object):

    def __init__(self, config):
        self.config = config
        self.parser = self.config.get_parser()

        self.remove_nodes_re = (
        "^side$|combx|retweet|mediaarticlerelated|menucontainer|"
        "navbar|storytopbar-bucket|utility-bar|inline-share-tools"
        "|comment|PopularQuestions|contact|foot|footer|Footer|footnote"
        "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt"
        "|links|meta$|shoutbox|sponsor"
        "|tags|socialnetworking|socialNetworking|cnnStryHghLght"
        "|cnn_stryspcvbx|^inset$|pagetools|post-attributes"
        "|welcome_form|contentTools2|the_answers"
        "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings"
        "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
        "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
        "|legende|ajoutVideo|timestamp|js_replies"
        )
        self.regexp_namespace = "http://exslt.org/regular-expressions"
        self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_classes_re = "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re
        self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)"
        self.caption_re = "^caption$"
        self.google_re = " google "
        self.entries_re = "^[^entry-]more.*$"
        self.facebook_re = "[^-]facebook"
        self.facebook_braodcasting_re = "facebook-broadcasting"
        self.twitter_re = "[^-]twitter"
        self.tablines_replacements = ReplaceSequence()\
                                            .create("\n", "\n\n")\
                                            .append("\t")\
                                            .append("^\\s+$")

    def clean(self, article):
        """
        """
        doc_to_clean = article.doc
        doc_to_clean = self.clean_body_classes(doc_to_clean)
        doc_to_clean = self.clean_article_tags(doc_to_clean)
        doc_to_clean = self.clean_em_tags(doc_to_clean)
        doc_to_clean = self.remove_drop_caps(doc_to_clean)
        doc_to_clean = self.remove_scripts_styles(doc_to_clean)
        doc_to_clean = self.clean_bad_tags(doc_to_clean)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.caption_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_braodcasting_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re)
        doc_to_clean = self.clean_para_spans(doc_to_clean)
        doc_to_clean = self.div_to_para(doc_to_clean, 'div')
        doc_to_clean = self.div_to_para(doc_to_clean, 'span')
        return doc_to_clean

    def clean_body_classes(self, doc):
        # we don't need body classes
        # in case it matches an unwanted class all the document
        # will be empty
        elements = self.parser.getElementsByTag(doc, tag="body")
        if elements:
            self.parser.delAttribute(elements[0], attr="class")
        return doc

    def clean_article_tags(self, doc):
        articles = self.parser.getElementsByTag(doc, tag='article')
        for article in articles:
            for attr in ['id', 'name', 'class']:
                self.parser.delAttribute(article, attr=attr)
        return doc

    def clean_em_tags(self, doc):
        ems = self.parser.getElementsByTag(doc, tag='em')
        for node in ems:
            images = self.parser.getElementsByTag(node, tag='img')
            if len(images) == 0:
                self.parser.drop_tag(node)
        return doc

    def remove_drop_caps(self, doc):
        items = self.parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]")
        for item in items:
            self.parser.drop_tag(item)

        return doc

    def remove_scripts_styles(self, doc):
        # remove scripts
        scripts = self.parser.getElementsByTag(doc, tag='script')
        for item in scripts:
            self.parser.remove(item)

        # remove styles
        styles = self.parser.getElementsByTag(doc, tag='style')
        for item in styles:
            self.parser.remove(item)

        # remove comments
        comments = self.parser.getComments(doc)
        for item in comments:
            self.parser.remove(item)

        return doc

    def clean_bad_tags(self, doc):
        # ids
        naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re)
        for node in naughty_list:
            self.parser.remove(node)

        # class
        naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re)
        for node in naughty_classes:
            self.parser.remove(node)

        # name
        naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re)
        for node in naughty_names:
            self.parser.remove(node)

        return doc

    def remove_nodes_regex(self, doc, pattern):
        for selector in ['id', 'class']:
            reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern)
            naughty_list = self.parser.xpath_re(doc, reg)
            for node in naughty_list:
                self.parser.remove(node)
        return doc

    def clean_para_spans(self, doc):
        spans = self.parser.css_select(doc, 'p > span')
        for item in spans:
            self.parser.drop_tag(item)
        return doc

    def get_flushed_buffer(self, replacement_text, doc):
        return self.parser.textToPara(replacement_text)

    def get_replacement_nodes(self, doc, div):
        replacement_text = []
        nodes_to_return = []
        nodes_to_remove = []
        childs = self.parser.childNodesWithText(div)

        for kid in childs:
            # node is a p
            # and already have some replacement text
            if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0:
                newNode = self.get_flushed_buffer(''.join(replacement_text), doc)
                nodes_to_return.append(newNode)
                replacement_text = []
                nodes_to_return.append(kid)
            # node is a text node
            elif self.parser.isTextNode(kid):
                kid_text_node = kid
                kid_text = self.parser.getText(kid)
                replace_text = self.tablines_replacements.replaceAll(kid_text)
                if(len(replace_text)) > 1:
                    previous_sibling_node = self.parser.previousSibling(kid_text_node)
                    while previous_sibling_node is not None \
                        and self.parser.getTag(previous_sibling_node) == "a" \
                        and self.parser.getAttribute(previous_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(previous_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(previous_sibling_node)
                        self.parser.setAttribute(previous_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        prev = self.parser.previousSibling(previous_sibling_node)
                        previous_sibling_node = prev if prev is not None else None
                    # append replace_text
                    replacement_text.append(replace_text)
                    #
                    next_sibling_node = self.parser.nextSibling(kid_text_node)
                    while next_sibling_node is not None \
                        and self.parser.getTag(next_sibling_node) == "a" \
                        and self.parser.getAttribute(next_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(next_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(next_sibling_node)
                        self.parser.setAttribute(next_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        next = self.parser.nextSibling(next_sibling_node)
                        previous_sibling_node = next if next is not None else None

            # otherwise
            else:
                nodes_to_return.append(kid)

        # flush out anything still remaining
        if(len(replacement_text) > 0):
            new_node = self.get_flushed_buffer(''.join(replacement_text), doc)
            nodes_to_return.append(new_node)
            replacement_text = []

        for n in nodes_to_remove:
            self.parser.remove(n)

        return nodes_to_return

    def replace_with_para(self, doc, div):
        self.parser.replaceTag(div, 'p')

    def div_to_para(self, doc, dom_type):
        bad_divs = 0
        else_divs = 0
        divs = self.parser.getElementsByTag(doc, tag=dom_type)
        tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', 'pre', 'table', 'ul']

        for div in divs:
            items = self.parser.getElementsByTags(div, tags)
            if div is not None and len(items) == 0:
                self.replace_with_para(doc, div)
                bad_divs += 1
            elif div is not None:
                replaceNodes = self.get_replacement_nodes(doc, div)
                div.clear()

                for c, n in enumerate(replaceNodes):
                    div.insert(c, n)

                else_divs += 1

        return doc

class StandardDocumentCleaner(DocumentCleaner):
    pass
<commit_msg>fix remove all span tags below p tags<commit_after># -*- coding: utf-8 -*-
"""
Holds the code for cleaning out unwanted tags from the lxml
dom xpath.
"""
from .utils import ReplaceSequence

class DocumentCleaner(object):

    def __init__(self, config):
        self.config = config
        self.parser = self.config.get_parser()

        self.remove_nodes_re = (
        "^side$|combx|retweet|mediaarticlerelated|menucontainer|"
        "navbar|storytopbar-bucket|utility-bar|inline-share-tools"
        "|comment|PopularQuestions|contact|foot|footer|Footer|footnote"
        "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt"
        "|links|meta$|shoutbox|sponsor"
        "|tags|socialnetworking|socialNetworking|cnnStryHghLght"
        "|cnn_stryspcvbx|^inset$|pagetools|post-attributes"
        "|welcome_form|contentTools2|the_answers"
        "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings"
        "|date|^print$|popup|author-dropdown|tools|socialtools|byline"
        "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text"
        "|legende|ajoutVideo|timestamp|js_replies"
        )
        self.regexp_namespace = "http://exslt.org/regular-expressions"
        self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_classes_re = "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re
        self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re
        self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)"
        self.caption_re = "^caption$"
        self.google_re = " google "
        self.entries_re = "^[^entry-]more.*$"
        self.facebook_re = "[^-]facebook"
        self.facebook_braodcasting_re = "facebook-broadcasting"
        self.twitter_re = "[^-]twitter"
        self.tablines_replacements = ReplaceSequence()\
                                            .create("\n", "\n\n")\
                                            .append("\t")\
                                            .append("^\\s+$")

    def clean(self, article):
        """
        """
        doc_to_clean = article.doc
        doc_to_clean = self.clean_body_classes(doc_to_clean)
        doc_to_clean = self.clean_article_tags(doc_to_clean)
        doc_to_clean = self.clean_em_tags(doc_to_clean)
        doc_to_clean = self.remove_drop_caps(doc_to_clean)
        doc_to_clean = self.remove_scripts_styles(doc_to_clean)
        doc_to_clean = self.clean_bad_tags(doc_to_clean)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.caption_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_braodcasting_re)
        doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re)
        doc_to_clean = self.clean_para_spans(doc_to_clean)
        doc_to_clean = self.div_to_para(doc_to_clean, 'div')
        doc_to_clean = self.div_to_para(doc_to_clean, 'span')
        return doc_to_clean

    def clean_body_classes(self, doc):
        # we don't need body classes
        # in case it matches an unwanted class all the document
        # will be empty
        elements = self.parser.getElementsByTag(doc, tag="body")
        if elements:
            self.parser.delAttribute(elements[0], attr="class")
        return doc

    def clean_article_tags(self, doc):
        articles = self.parser.getElementsByTag(doc, tag='article')
        for article in articles:
            for attr in ['id', 'name', 'class']:
                self.parser.delAttribute(article, attr=attr)
        return doc

    def clean_em_tags(self, doc):
        ems = self.parser.getElementsByTag(doc, tag='em')
        for node in ems:
            images = self.parser.getElementsByTag(node, tag='img')
            if len(images) == 0:
                self.parser.drop_tag(node)
        return doc

    def remove_drop_caps(self, doc):
        items = self.parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]")
        for item in items:
            self.parser.drop_tag(item)

        return doc

    def remove_scripts_styles(self, doc):
        # remove scripts
        scripts = self.parser.getElementsByTag(doc, tag='script')
        for item in scripts:
            self.parser.remove(item)

        # remove styles
        styles = self.parser.getElementsByTag(doc, tag='style')
        for item in styles:
            self.parser.remove(item)

        # remove comments
        comments = self.parser.getComments(doc)
        for item in comments:
            self.parser.remove(item)

        return doc

    def clean_bad_tags(self, doc):
        # ids
        naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re)
        for node in naughty_list:
            self.parser.remove(node)

        # class
        naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re)
        for node in naughty_classes:
            self.parser.remove(node)

        # name
        naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re)
        for node in naughty_names:
            self.parser.remove(node)

        return doc

    def remove_nodes_regex(self, doc, pattern):
        for selector in ['id', 'class']:
            reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern)
            naughty_list = self.parser.xpath_re(doc, reg)
            for node in naughty_list:
                self.parser.remove(node)
        return doc

    def clean_para_spans(self, doc):
        spans = self.parser.css_select(doc, 'p span')
        for item in spans:
            self.parser.drop_tag(item)
        return doc

    def get_flushed_buffer(self, replacement_text, doc):
        return self.parser.textToPara(replacement_text)

    def get_replacement_nodes(self, doc, div):
        replacement_text = []
        nodes_to_return = []
        nodes_to_remove = []
        childs = self.parser.childNodesWithText(div)

        for kid in childs:
            # node is a p
            # and already have some replacement text
            if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0:
                newNode = self.get_flushed_buffer(''.join(replacement_text), doc)
                nodes_to_return.append(newNode)
                replacement_text = []
                nodes_to_return.append(kid)
            # node is a text node
            elif self.parser.isTextNode(kid):
                kid_text_node = kid
                kid_text = self.parser.getText(kid)
                replace_text = self.tablines_replacements.replaceAll(kid_text)
                if(len(replace_text)) > 1:
                    previous_sibling_node = self.parser.previousSibling(kid_text_node)
                    while previous_sibling_node is not None \
                        and self.parser.getTag(previous_sibling_node) == "a" \
                        and self.parser.getAttribute(previous_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(previous_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(previous_sibling_node)
                        self.parser.setAttribute(previous_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        prev = self.parser.previousSibling(previous_sibling_node)
                        previous_sibling_node = prev if prev is not None else None
                    # append replace_text
                    replacement_text.append(replace_text)
                    #
                    next_sibling_node = self.parser.nextSibling(kid_text_node)
                    while next_sibling_node is not None \
                        and self.parser.getTag(next_sibling_node) == "a" \
                        and self.parser.getAttribute(next_sibling_node, 'grv-usedalready') != 'yes':
                        outer = " " + self.parser.outerHtml(next_sibling_node) + " "
                        replacement_text.append(outer)
                        nodes_to_remove.append(next_sibling_node)
                        self.parser.setAttribute(next_sibling_node,
                                    attr='grv-usedalready', value='yes')
                        next = self.parser.nextSibling(next_sibling_node)
                        previous_sibling_node = next if next is not None else None

            # otherwise
            else:
                nodes_to_return.append(kid)

        # flush out anything still remaining
        if(len(replacement_text) > 0):
            new_node = self.get_flushed_buffer(''.join(replacement_text), doc)
            nodes_to_return.append(new_node)
            replacement_text = []

        for n in nodes_to_remove:
            self.parser.remove(n)

        return nodes_to_return

    def replace_with_para(self, doc, div):
        self.parser.replaceTag(div, 'p')

    def div_to_para(self, doc, dom_type):
        bad_divs = 0
        else_divs = 0
        divs = self.parser.getElementsByTag(doc, tag=dom_type)
        tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', 'pre', 'table', 'ul']

        for div in divs:
            items = self.parser.getElementsByTags(div, tags)
            if div is not None and len(items) == 0:
                self.replace_with_para(doc, div)
                bad_divs += 1
            elif div is not None:
                replaceNodes = self.get_replacement_nodes(doc, div)
                div.clear()

                for c, n in enumerate(replaceNodes):
                    div.insert(c, n)

                else_divs += 1

        return doc

class StandardDocumentCleaner(DocumentCleaner):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

from models import *
from apps.reporters.models import *
from apps.form.formslogic import FormsLogic

class NigeriaFormsLogic(FormsLogic):
    ''' This class will hold the nigeria-specific forms logic.
        I'm not sure whether this will be the right structure
        this was just for getting something hooked up '''

    _form_lookups = {"nets" : {
                                "class" : NetDistribution,
                                "netloc" : "location",
                                "distributed" : "distributed",
                                "expected" : "expected",
                                "actual" : "actual",
                                "discrepancy" : "discrepancy",
                                },
                     "net" : {    "class" : CardDistribution,
                                  "cardloc" : "location",
                                  "villages" : "settlements",
                                  "people" : "people",
                                  "coupons" : "distributed",
                                  }
                     }

    _foreign_key_lookups = {"Location" : "code" }

    def validate(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        if form_entry.form.type == "register":
            data = form_entry.to_dict()
            print "\n\n%r\n\n" % data

            # check that ALL FIELDS were provided
            required = ["location", "role", "password", "name"]
            missing = [t for t in required if data[t] is None]

            # in case we need help, build a valid reminder string
            help = ("%s register " % form_entry.domain.code.lower()) +\
                " ".join(["<%s>" % t for t in required])

            # missing fields! collate them, and
            # send back a friendly non-localized
            # error message, then abort
            if missing:
                mis_str = ", ".join(missing)
                return ["Missing fields: %s" % mis_str, help]

            try:
                # attempt to load the location and role objects
                # via their codes, which will raise if nothing
                # was found TODO: combine the exceptions
                data["location"] = Location.objects.get(code__iexact=data["location"])
                data["role"]     = Role.objects.get(code__iexact=data["role"])

            except Location.DoesNotExist:
                return ["Invalid location code: %s" %
                    data["location"], help]

            except Role.DoesNotExist:
                return ["Invalid role code: %s" %
                    data["role"], help]

            # parse the name via Reporter
            data["alias"], data["first_name"], data["last_name"] =\
                Reporter.parse_name(data.pop("name"))

            # check that the name/alias
            # hasn't already been registered
            reps = Reporter.objects.filter(alias=data["alias"])
            if len(reps):
                return ["Already been registed: %s" %
                    data["alias"], help]

            # all fields were present and correct, so copy them into the
            # form_entry, for "actions" to pick up again without re-fetching
            form_entry.rep_data = data

            # nothing went wrong. the data structure
            # is ready to spawn a Reporter object
            return None
        elif form_entry.form.type in self._form_lookups.keys() and not hasattr(message,"reporter"):
            return [ "You must register your phone before submitting data" ]

    def actions(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        if form_entry.form.type == "register":

            # spawn and save the reporter using the
            # data we collected in self.validate
            rep = Reporter.objects.create(**form_entry.rep_data)

            # we can assume that the new reporter will be using
            # this device again, so register a connection. this
            # automatically logs them in, so they can start
            # reporting straight away
            conn = PersistantConnection.from_message(message)
            conn.reporter = rep
            conn.save()

            # notify the user that everyting went okay
            # TODO: proper (localized?) messages here
            message.respond("Reporter %s (#%d/%d) added" % (rep.alias, rep.pk, conn.pk))

        elif self._form_lookups.has_key(form_entry.form.type):
            # this borrows a lot from supply.  I think we can make this a utility call
            to_use = self._form_lookups[form_entry.form.type]
            form_class = to_use["class"]
            instance = self._model_from_form(message, form_entry, form_class, to_use, self._foreign_key_lookups)
            instance.save()
<commit_msg>fix the validation that tests the existence of a reporter<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

from models import *
from apps.reporters.models import *
from apps.form.formslogic import FormsLogic

class NigeriaFormsLogic(FormsLogic):
    ''' This class will hold the nigeria-specific forms logic.
        I'm not sure whether this will be the right structure
        this was just for getting something hooked up '''

    _form_lookups = {"nets" : {
                                "class" : NetDistribution,
                                "netloc" : "location",
                                "distributed" : "distributed",
                                "expected" : "expected",
                                "actual" : "actual",
                                "discrepancy" : "discrepancy",
                                },
                     "net" : {    "class" : CardDistribution,
                                  "cardloc" : "location",
                                  "villages" : "settlements",
                                  "people" : "people",
                                  "coupons" : "distributed",
                                  }
                     }

    _foreign_key_lookups = {"Location" : "code" }

    def validate(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        if form_entry.form.type == "register":
            data = form_entry.to_dict()
            print "\n\n%r\n\n" % data

            # check that ALL FIELDS were provided
            required = ["location", "role", "password", "name"]
            missing = [t for t in required if data[t] is None]

            # in case we need help, build a valid reminder string
            help = ("%s register " % form_entry.domain.code.lower()) +\
                " ".join(["<%s>" % t for t in required])

            # missing fields! collate them, and
            # send back a friendly non-localized
            # error message, then abort
            if missing:
                mis_str = ", ".join(missing)
                return ["Missing fields: %s" % mis_str, help]

            try:
                # attempt to load the location and role objects
                # via their codes, which will raise if nothing
                # was found TODO: combine the exceptions
                data["location"] = Location.objects.get(code__iexact=data["location"])
                data["role"]     = Role.objects.get(code__iexact=data["role"])

            except Location.DoesNotExist:
                return ["Invalid location code: %s" %
                    data["location"], help]

            except Role.DoesNotExist:
                return ["Invalid role code: %s" %
                    data["role"], help]

            # parse the name via Reporter
            data["alias"], data["first_name"], data["last_name"] =\
                Reporter.parse_name(data.pop("name"))

            # check that the name/alias
            # hasn't already been registered
            reps = Reporter.objects.filter(alias=data["alias"])
            if len(reps):
                return ["Already been registed: %s" %
                    data["alias"], help]

            # all fields were present and correct, so copy them into the
            # form_entry, for "actions" to pick up again without re-fetching
            form_entry.rep_data = data

            # nothing went wrong. the data structure
            # is ready to spawn a Reporter object
            return None
        elif form_entry.form.type in self._form_lookups.keys() and (not hasattr(message,"reporter") or not message.reporter):

            return [ "You must register your phone before submitting data" ]

    def actions(self, *args, **kwargs):
        message = args[0]
        form_entry = args[1]
        if form_entry.form.type == "register":

            # spawn and save the reporter using the
            # data we collected in self.validate
            rep = Reporter.objects.create(**form_entry.rep_data)

            # we can assume that the new reporter will be using
            # this device again, so register a connection. this
            # automatically logs them in, so they can start
            # reporting straight away
            conn = PersistantConnection.from_message(message)
            conn.reporter = rep
            conn.save()

            # notify the user that everyting went okay
            # TODO: proper (localized?) messages here
            message.respond("Reporter %s (#%d/%d) added" % (rep.alias, rep.pk, conn.pk))

        elif self._form_lookups.has_key(form_entry.form.type):
            # this borrows a lot from supply.  I think we can make this a utility call
            to_use = self._form_lookups[form_entry.form.type]
            form_class = to_use["class"]
            instance = self._model_from_form(message, form_entry, form_class, to_use, self._foreign_key_lookups)
            instance.save()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import subprocess

from nimp.utilities.processes import *
from nimp.utilities.paths import *

def git_cat(path, repository, branch = 'master'):

    p = subprocess.Popen('git archive --remote=%s %s %s | tar -xOf -' % (repository, branch, path),
                         shell = True, stdout=subprocess.PIPE)
    return p.stdout.read()

<commit_msg>Add git_find() in addition to git_cat().<commit_after># -*- coding: utf-8 -*-

import os
import subprocess

from nimp.utilities.processes import *
from nimp.utilities.paths import *

def git_cat(path, repository, branch = 'master'):

    p = subprocess.Popen('git archive --remote=%s %s %s | tar -xOf -' % (repository, branch, path),
                         shell = True, stdout=subprocess.PIPE)
    return p.stdout.read()

# FIXME: this is rather inefficient since it transfers the whole tree instead of the filenames
def git_find(path, repository, branch = 'master'):
    p = subprocess.Popen('git archive --remote=%s %s %s | tar -tf -' % (repository, branch, path),
                         shell = True, stdout=subprocess.PIPE)
    return [l.strip().decode('utf-8') for l in p.stdout.readlines()]

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------------
import socket
import random
import string
import time
import contextlib
import shutil
import os

from nimp.utilities.build import *
from nimp.utilities.deployment import *
from nimp.utilities.file_mapper import *
from nimp.utilities.perforce import *
from nimp.utilities.system import *

#---------------------------------------------------------------------------
def ue4_build(env):
    if not env.ue4_build_configuration:
        log_error("Invalid empty value for configuration")
        return False

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if _ue4_generate_project() != 0:
        log_error("Error generating UE4 project files")
        return False

    # The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '12'
    try:
        for line in open(env.solution):
            if '# Visual Studio 2012' in line:
                vs_version = '11'
                break
    except:
        pass

    if not hasattr(env, 'no_prerequisites') or not env.no_prerequisites:

        # For some reason nothing copies this file on OS X
        if platform.system() == 'Darwin':
            robocopy('../Engine/Binaries/ThirdParty/Ionic/Ionic.Zip.Reduced.dll',
                     '../Engine/Binaries/DotNET/Ionic.Zip.Reduced.dll')

        # Build tools from the UE4 solution if necessary
        tools = []

        if env.platform == 'win64' and env.configuration == 'devel':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'UnrealFrontend',
                       'UnrealLightmass',
                       'UnrealFileServer',
                       'ShaderCompileWorker',
                       'SymbolDebugger',
                       'PS4DevKitUtil',
                       'PS4MapFileUtil',
                       'XboxOnePDBFileUtil', ]

        if env.platform == 'ps4':
            if 'PS4MapFileUtil' not in tools: tools += [ 'PS4MapFileUtil' ]

        if env.platform == 'xboxone':
            if 'XboxOnePDBFileUtil' not in tools: tools += [ 'XboxOnePDBFileUtil' ]

        for tool in tools:
            if not _ue4_build_project(env.solution, tool,
                                      'Win64', 'Development', vs_version, 'Build'):
                log_error("Could not build %s" % (tool, ))
                return False

        # Build tools from other solutions or with other flags
        if env.platform == 'win64' and env.configuration == 'devel':

            if not _ue4_build_project(env.solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                log_error("Could not build BootstrapPackagedGame")
                return False

            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not vsbuild('../Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build UnrealSwarm")
                return False

            if not vsbuild('../Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build SwarmInterface")
                return False

            if not vsbuild('../Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build NetworkProfiler")
                return False

            if not vsbuild('../Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln',
                           'x64', 'Development', None, '11', 'Build'):
                log_error("Could not build XboxOnePackageNameUtil")
                return False

    result = True
    # If we didn’t want only prerequisites, build UE4
    if not hasattr(env, 'only_prerequisites') or not env.only_prerequisites:

        # Build the main binaries
        result = _ue4_build_project(env.solution, env.game, env.ue4_build_platform,
                                    env.ue4_build_configuration, vs_version, 'Build')

    return result

#
# Generate UE4 project files
#

def _ue4_generate_project():
    if is_msys():
        return call_process('.', ['../GenerateProjectFiles.bat'])
    else:
        return call_process('..', ['/bin/sh', 'GenerateProjectFiles.sh'])

#
# Helper commands for configuration sanitising
#

def get_ue4_build_config(config, platform):
    d = { "debug"    : "Debug",
          "devel"    : "Development Editor" if platform in ["win64", "linux", "mac"] else "Development",
          "test"     : "Test",
          "shipping" : "Shipping", }
    if config not in d:
        log_warning('Unsupported UE4 build config “%s”' % (config))
        return None
    return d[config]

def get_ue4_build_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XboxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux",
          "mac"     : "Mac", }
    if platform not in d:
        log_warning('Unsupported UE4 build platform “%s”' % (platform))
        return None
    return d[platform]

def get_ue4_cook_platform(platform):
    d = { "ps4"     : "FIXME",
          "xboxone" : "FIXME",
          "win64"   : "FIXME",
          "win32"   : "FIXME",
          "linux"   : "FIXME", }
    if platform not in d:
        log_warning('Unsupported UE4 cook platform “%s”' % (platform))
        return None
    return d[platform]

#---------------------------------------------------------------------------
def _ue4_build_project(sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if is_msys():
        return vsbuild(sln_file, build_platform, configuration,
                       project, vs_version, target)

    elif platform.system() == 'Darwin':
        return call_process('..', ['/bin/sh', 'Engine/Build/BatchFiles/Mac/Build.sh',
                                   project, build_platform, configuration]) == 0

    else:
        project_name = project
        if configuration not in ['Development', 'Development Editor']:
            project_name += '-Linux-' + configuration
        elif configuration == 'Development Editor':
            project_name += 'Editor'
        return call_process('..', ['make', project_name]) == 0

#---------------------------------------------------------------------------
def ue4_commandlet(env, commandlet, *args):
    cmdline = [ "../Engine/Binaries/Win64/UE4Editor.exe",
                env.game,
                "-run=%s" % commandlet]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return call_process(".", cmdline) == 0

<commit_msg>Do not early exit if one of the UE4 tools failed to build.<commit_after># -*- coding: utf-8 -*-

#-------------------------------------------------------------------------------
import socket
import random
import string
import time
import contextlib
import shutil
import os

from nimp.utilities.build import *
from nimp.utilities.deployment import *
from nimp.utilities.file_mapper import *
from nimp.utilities.perforce import *
from nimp.utilities.system import *

#---------------------------------------------------------------------------
def ue4_build(env):
    if not env.ue4_build_configuration:
        log_error("Invalid empty value for configuration")
        return False

    if env.disable_unity:
        os.environ['UBT_bUseUnityBuild'] = 'false'

    if _ue4_generate_project() != 0:
        log_error("Error generating UE4 project files")
        return False

    # The Durango XDK does not support Visual Studio 2013 yet, so if UE4
    # detected it, it created VS 2012 project files and we have to use that
    # version to build the tools instead.
    vs_version = '12'
    try:
        for line in open(env.solution):
            if '# Visual Studio 2012' in line:
                vs_version = '11'
                break
    except:
        pass

    # We’ll try to build all tools even in case of failure
    result = True

    if not hasattr(env, 'no_prerequisites') or not env.no_prerequisites:

        # For some reason nothing copies this file on OS X
        if platform.system() == 'Darwin':
            robocopy('../Engine/Binaries/ThirdParty/Ionic/Ionic.Zip.Reduced.dll',
                     '../Engine/Binaries/DotNET/Ionic.Zip.Reduced.dll')

        # Build tools from the UE4 solution if necessary
        tools = []

        if env.platform == 'win64' and env.configuration == 'devel':
            tools += [ 'DotNETUtilities',
                       'AutomationTool',
                       'UnrealFrontend',
                       'UnrealLightmass',
                       'UnrealFileServer',
                       'ShaderCompileWorker',
                       'SymbolDebugger',
                       'PS4DevKitUtil',
                       'PS4MapFileUtil',
                       'XboxOnePDBFileUtil', ]

        if env.platform == 'ps4':
            if 'PS4MapFileUtil' not in tools: tools += [ 'PS4MapFileUtil' ]

        if env.platform == 'xboxone':
            if 'XboxOnePDBFileUtil' not in tools: tools += [ 'XboxOnePDBFileUtil' ]

        for tool in tools:
            if not _ue4_build_project(env.solution, tool,
                                      'Win64', 'Development', vs_version, 'Build'):
                log_error("Could not build %s" % (tool, ))
                result = False

        # Build tools from other solutions or with other flags
        if env.platform == 'win64' and env.configuration == 'devel':

            if not _ue4_build_project(env.solution, 'BootstrapPackagedGame',
                                      'Win64', 'Shipping', vs_version, 'Build'):
                log_error("Could not build BootstrapPackagedGame")
                result = False

            # This also builds AgentInterface.dll, needed by SwarmInterface.sln
            if not vsbuild('../Engine/Source/Programs/UnrealSwarm/UnrealSwarm.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build UnrealSwarm")
                result = False

            if not vsbuild('../Engine/Source/Editor/SwarmInterface/DotNET/SwarmInterface.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build SwarmInterface")
                result = False

            if not vsbuild('../Engine/Source/Programs/NetworkProfiler/NetworkProfiler.sln',
                           'Any CPU', 'Development', None, '11', 'Build'):
                log_error("Could not build NetworkProfiler")
                result = False

            if not vsbuild('../Engine/Source/Programs/XboxOne/XboxOnePackageNameUtil/XboxOnePackageNameUtil.sln',
                           'x64', 'Development', None, '11', 'Build'):
                log_error("Could not build XboxOnePackageNameUtil")
                result = False

    if not result:
        return result

    # If we didn’t want only prerequisites, build UE4
    if not hasattr(env, 'only_prerequisites') or not env.only_prerequisites:

        # Build the main binaries
        result = _ue4_build_project(env.solution, env.game, env.ue4_build_platform,
                                    env.ue4_build_configuration, vs_version, 'Build')

    return result

#
# Generate UE4 project files
#

def _ue4_generate_project():
    if is_msys():
        return call_process('.', ['../GenerateProjectFiles.bat'])
    else:
        return call_process('..', ['/bin/sh', 'GenerateProjectFiles.sh'])

#
# Helper commands for configuration sanitising
#

def get_ue4_build_config(config, platform):
    d = { "debug"    : "Debug",
          "devel"    : "Development Editor" if platform in ["win64", "linux", "mac"] else "Development",
          "test"     : "Test",
          "shipping" : "Shipping", }
    if config not in d:
        log_warning('Unsupported UE4 build config “%s”' % (config))
        return None
    return d[config]

def get_ue4_build_platform(platform):
    d = { "ps4"     : "PS4",
          "xboxone" : "XboxOne",
          "win64"   : "Win64",
          "win32"   : "Win32",
          "linux"   : "Linux",
          "mac"     : "Mac", }
    if platform not in d:
        log_warning('Unsupported UE4 build platform “%s”' % (platform))
        return None
    return d[platform]

def get_ue4_cook_platform(platform):
    d = { "ps4"     : "FIXME",
          "xboxone" : "FIXME",
          "win64"   : "FIXME",
          "win32"   : "FIXME",
          "linux"   : "FIXME", }
    if platform not in d:
        log_warning('Unsupported UE4 cook platform “%s”' % (platform))
        return None
    return d[platform]

#---------------------------------------------------------------------------
def _ue4_build_project(sln_file, project, build_platform,
                       configuration, vs_version, target = 'Rebuild'):

    if is_msys():
        return vsbuild(sln_file, build_platform, configuration,
                       project, vs_version, target)

    elif platform.system() == 'Darwin':
        return call_process('..', ['/bin/sh', 'Engine/Build/BatchFiles/Mac/Build.sh',
                                   project, build_platform, configuration]) == 0

    else:
        project_name = project
        if configuration not in ['Development', 'Development Editor']:
            project_name += '-Linux-' + configuration
        elif configuration == 'Development Editor':
            project_name += 'Editor'
        return call_process('..', ['make', project_name]) == 0

#---------------------------------------------------------------------------
def ue4_commandlet(env, commandlet, *args):
    cmdline = [ "../Engine/Binaries/Win64/UE4Editor.exe",
                env.game,
                "-run=%s" % commandlet]

    cmdline += list(args)
    cmdline += ['-nopause', '-buildmachine', '-forcelogflush', '-unattended', '-noscriptcheck']

    return call_process(".", cmdline) == 0

<|endoftext|>