<commit_before>#!/usr/bin/python

import sys
import os
from subprocess import call

class PartialProgramAnalyser:
    """ Partial program analyser """

    def __init__(self, in_file_name):
        self.in_file_name = in_file_name
        self.out_file_name = in_file_name + ".ctr"
        self.gen_file_name = in_file_name + ".gen.h"
        self.fixed_in_file_name = in_file_name[:-1] + "fixed.c"

    @staticmethod
    def write_log(msg):
        print "[Reconstructor] %s" % msg

    def gen_constraints(self):
        """ Invoke our constraint generator """
        self.write_log("Generating constraints")

        call(["./Gen", "%s" % self.in_file_name, "clean"])

    def solve_constraints(self):
        """ Invoke our constraint solver """
        self.write_log("Solving constraints")

        os.chdir("solver")
        # If there's an old generated file, delete it.
        if os.path.isfile("../%s" % self.gen_file_name):
            os.remove("../%s" % self.gen_file_name)
        call(["stack", "exec", "typedefsolver-exe", "--",
              "-i", "../%s" % self.out_file_name,
              "-o", "../%s" % self.gen_file_name])
        os.chdir("..")

    def fix_program(self):
        """ Create a new source (based on the original) that includes the solved stuff"""
        self.write_log("Creating new complete source")

        content = "/* Reconstructed from %s */\n" % self.in_file_name
        local_gen_file = self.gen_file_name.split('/')[-1:][0]
        content += '#include "%s"\n' % local_gen_file

        with open(self.in_file_name, "r") as f:
            content += f.read()

        with open(self.fixed_in_file_name, "w") as f:
            f.write(content)

    def analyse(self):
        self.gen_constraints()
        self.solve_constraints()
        self.fix_program()

if __name__ == "__main__":
    if (len(sys.argv)) != 2:
        print "Usage: ./Reconstruct.py relative/path/file.c"
        sys.exit(1)

    analyser = PartialProgramAnalyser(sys.argv[1])
    analyser.analyse()
<commit_msg>Correct solver's executable name<commit_after>#!/usr/bin/python

import sys
import os
from subprocess import call

class PartialProgramAnalyser:
    """ Partial program analyser """

    def __init__(self, in_file_name):
        self.in_file_name = in_file_name
        self.out_file_name = in_file_name + ".ctr"
        self.gen_file_name = in_file_name + ".gen.h"
        self.fixed_in_file_name = in_file_name[:-1] + "fixed.c"

    @staticmethod
    def write_log(msg):
        print "[Reconstructor] %s" % msg

    def gen_constraints(self):
        """ Invoke our constraint generator """
        self.write_log("Generating constraints")

        call(["./Gen", "%s" % self.in_file_name, "clean"])

    def solve_constraints(self):
        """ Invoke our constraint solver """
        self.write_log("Solving constraints")

        os.chdir("solver")
        # If there's an old generated file, delete it.
        if os.path.isfile("../%s" % self.gen_file_name):
            os.remove("../%s" % self.gen_file_name)
        call(["stack", "exec", "psychecsolver-exe", "--",
              "-i", "../%s" % self.out_file_name,
              "-o", "../%s" % self.gen_file_name])
        os.chdir("..")

    def fix_program(self):
        """ Create a new source (based on the original) that includes the solved stuff"""
        self.write_log("Creating new complete source")

        content = "/* Reconstructed from %s */\n" % self.in_file_name
        local_gen_file = self.gen_file_name.split('/')[-1:][0]
        content += '#include "%s"\n' % local_gen_file

        with open(self.in_file_name, "r") as f:
            content += f.read()

        with open(self.fixed_in_file_name, "w") as f:
            f.write(content)

    def analyse(self):
        self.gen_constraints()
        self.solve_constraints()
        self.fix_program()

if __name__ == "__main__":
    if (len(sys.argv)) != 2:
        print "Usage: ./Reconstruct.py relative/path/file.c"
        sys.exit(1)

    analyser = PartialProgramAnalyser(sys.argv[1])
    analyser.analyse()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
An example of a hierarchy of cortical regions in a Network.
"""

import copy
import csv
import json
import os

from nupic.algorithms.anomaly import computeRawAnomalyScore
from nupic.data.datasethelpers import findDataset
from nupic.data.file_record_stream import FileRecordStream
from nupic.engine import Network
from nupic.encoders import MultiEncoder

# Level of detail of console output. Int value from 0 (none)
# to 3 (super detailed)
_VERBOSITY = 0

# Seed used for random number generation
_SEED = 2045
_INPUT_FILE_PATH = "../prediction/data/extra/hotgym/rec-center-hourly.csv"
_OUTPUT_FILE_NAME = "hierarchy-demo-output.csv"

# Parameter dict for SPRegion
SP_PARAMS = {"spVerbosity": _VERBOSITY,
             "spatialImp": "cpp",
             "seed": _SEED,

             # determined and set during network creation
             "inputWidth": 0,

             # @see nupic.research.spatial_pooler.SpatialPooler for explanations
             "globalInhibition": 1,
             "columnCount": 2048,
             "numActiveColumnsPerInhArea": 40,
             "potentialPct": 0.8,
             "synPermConnected": 0.1,
             "synPermActiveInc": 0.0001,
             "synPermInactiveDec": 0.0005,
             "maxBoost": 1.0}

# Parameter dict for TPRegion
TP_PARAMS = {"verbosity": _VERBOSITY,
             "temporalImp": "cpp",
             "seed": _SEED,

             # @see nupic.research.temporal_memory.TemporalMemory
             # for explanations
             "columnCount": 2048,
             "cellsPerColumn": 12,
             "inputWidth": 2048,
             "newSynapseCount": 20,
             "maxSynapsesPerSegment": 32,
             "maxSegmentsPerCell": 128,
             "initialPerm": 0.21,
             "permanenceInc": 0.1,
             "permanenceDec": 0.1,
             "globalDecay": 0.0,
             "maxAge": 0,
             "minThreshold": 9,
             "activationThreshold": 12,
             "outputType": "normal",
             "pamLength": 3}

_RECORD_SENSOR = "sensorRegion"
_L1_SPATIAL_POOLER = "l1spatialPoolerRegion"
_L1_TEMPORAL_POOLER = "l1temporalPoolerRegion"
_L1_CLASSIFIER = "l1classifier"

_L2_SPATIAL_POOLER = "l2spatialPoolerRegion"
_L2_TEMPORAL_POOLER = "l2temporalPoolerRegion"

def createEncoder():
  """
  Creates and returns a #MultiEncoder including a ScalarEncoder for
  energy consumption and a DateEncoder for the time of the day.

  @see nupic/encoders/__init__.py for type to file-name mapping
  @see nupic/encoders for encoder source files
  """
  encoder = MultiEncoder()
  encoder.addMultipleEncoders({
      "consumption": {"fieldname": u"consumption",
                      "type": "ScalarEncoder",
                      "name": u"consumption",
                      "minval": 0.0,
                      "maxval": 100.0,
                      "clipInput": True,
                      "w": 21,
                      "n": 50},
      "timestamp_timeOfDay": {"fieldname": u"timestamp",
                              "type": "DateEncoder",
                              "name": u"timestamp_timeOfDay",
                              "timeOfDay": (21, 9.5)}
  })
  return encoder

def createRecordSensor(network, name, dataSource):
  """
  Creates a RecordSensor region that allows us to specify a file record
  stream as the input source.
  """

  # Specific type of region. Possible options can be found in /nupic/regions/
  regionType = "py.RecordSensor"

  # Creates a json from specified dictionary.
  regionParams = json.dumps({"verbosity": _VERBOSITY})
  network.addRegion(name, regionType, regionParams)

  # getSelf returns the actual region, instead of a region wrapper
  sensorRegion = network.regions[name].getSelf()

  # Specify how RecordSensor encodes input values
  sensorRegion.encoder = createEncoder()

  # Specify the dataSource as a file record stream instance
  sensorRegion.dataSource = dataSource
  return sensorRegion

def createSpatialPooler(network, name, inputWidth):
  # Create the spatial pooler region
  SP_PARAMS["inputWidth"] = inputWidth
  spatialPoolerRegion = network.addRegion(name, "py.SPRegion",
                                          json.dumps(SP_PARAMS))
  # Make sure learning is enabled
  spatialPoolerRegion.setParameter("learningMode", True)
  # We want temporal anomalies so disable anomalyMode in the SP. This mode is
  # used for computing anomalies in a non-temporal model.
  spatialPoolerRegion.setParameter("anomalyMode", False)
  return spatialPoolerRegion

def createTemporalPooler(network, name):
  temporalPoolerRegion = network.addRegion(name, "py.TPRegion",
                                           json.dumps(TP_PARAMS))
  # Enable topDownMode to get the predicted columns output
  temporalPoolerRegion.setParameter("topDownMode", True)
  # Make sure learning is enabled (this is the default)
  temporalPoolerRegion.setParameter("learningMode", True)
  # Enable inference mode so we get predictions
  temporalPoolerRegion.setParameter("inferenceMode", True)
  # Enable anomalyMode to compute the anomaly score. This actually doesn't work
  # now so doesn't matter. We instead compute the anomaly score based on
  # topDownOut (predicted columns) and SP bottomUpOut (active columns).
  temporalPoolerRegion.setParameter("anomalyMode", True)
  return temporalPoolerRegion

def createNetwork(dataSource):
  """Creates and returns a new Network with a sensor region reading data from
  'dataSource'.
  TODO describe hierarchy
  @param dataSource - A RecordStream containing the input data
  @returns a Network ready to run
  """
  network = Network()

  # Create and add a record sensor and a SP region
  sensor = createRecordSensor(network, name=_RECORD_SENSOR,
                              dataSource=dataSource)
  createSpatialPooler(network, name=_L1_SPATIAL_POOLER,
                      inputWidth=sensor.encoder.getWidth())

  # Link the SP region to the sensor input
  linkType = "UniformLink"
  linkParams = ""
  network.link(_RECORD_SENSOR, _L1_SPATIAL_POOLER, linkType, linkParams)

  # Create and add a TP region
  l1temporalPooler = createTemporalPooler(network, _L1_TEMPORAL_POOLER)

  # Link SP region to TP region in the feedforward direction
  network.link(_L1_SPATIAL_POOLER, _L1_TEMPORAL_POOLER, linkType, linkParams)

  # Add a classifier
  clParams = {  # Classifier learning/forgetting rate. Higher
                # values make it adapt faster and forget older patterns faster.
                'alpha': 0.005,

                # Comma separated list of the desired steps of
                # prediction that the classifier should learn
                'steps': '1',

                # Which implementation of the classifier to use.
                # See CLAClassifierFactory#create
                'implementation': 'cpp',

                # Diagnostic output verbosity control;
                # 0: silent; [1..6]: increasing levels of verbosity
                'clVerbosity': 0}
  network.addRegion(_L1_CLASSIFIER, "py.CLAClassifierRegion",
                    json.dumps(clParams))
  network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="categoryOut", destInput="categoryIn")
  network.link(_L1_TEMPORAL_POOLER, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="bottomUpOut", destInput="bottomUpIn")

  # Second Level
  # l2inputWidth = l1temporalPooler.getSelf().getOutputElementCount("bottomUpOut")
  # createSpatialPooler(network, name=_L2_SPATIAL_POOLER, inputWidth=l2inputWidth)
  # network.link(_L1_TEMPORAL_POOLER, _L2_SPATIAL_POOLER, linkType, linkParams)
  #
  # createTemporalPooler(network, _L2_TEMPORAL_POOLER)
  # network.link(_L2_SPATIAL_POOLER, _L2_TEMPORAL_POOLER, linkType, linkParams)
  return network

def runNetwork(network, numRecords, writer):
  """
  Runs specified Network writing the ensuing anomaly score to writer.

  @param network: The Network instance to be run
  @param writer: A csv.writer used to write to output file.
  """
  sensorRegion = network.regions[_RECORD_SENSOR]
  encoder = sensorRegion.getSelf().encoder
  l1SpRegion = network.regions[_L1_SPATIAL_POOLER]
  l1TpRegion = network.regions[_L1_TEMPORAL_POOLER]
  l1Classifier = network.regions[_L1_CLASSIFIER]

  # l2SpRegion = network.regions[_L2_SPATIAL_POOLER]
  # l2TpRegion = network.regions[_L2_TEMPORAL_POOLER]
  # TODO Print something out from L2?

  prevPredictedColumns = []
  for i in xrange(numRecords):
    # Run the network for a single iteration
    network.run(1)

    l1tpOutput = l1TpRegion.getOutputData("bottomUpOut").nonzero()[0]
    print type(l1tpOutput)
    consumption = float(sensorRegion.getOutputData("sourceOut")[0])
    bucketIndex = float(sensorRegion.getOutputData("categoryOut")[0])

    clDict = {"actValue": consumption, "bucketIdx": bucketIndex}

    # patternNZ:      list of the active indices from the output below
    # classification: dict of the classification information:
    #                   bucketIdx: index of the encoder bucket
    #                   actValue:  actual value going into the encoder
    #
    # retval:     dict containing inference results, one entry for each step in
    #             self.steps. The key is the number of steps, the value is an
    #             array containing the relative likelihood for each bucketIdx
    #             starting from bucketIdx 0.
    #
    #             for example:
    #               {1 : [0.1, 0.3, 0.2, 0.7]
    #                4 : [0.2, 0.4, 0.3, 0.5]}
    inferenceResults = l1Classifier.getSelf().customCompute(recordNum=i,
                                         patternNZ=l1tpOutput,
                                         classification=clDict)
    print "results: ", inferenceResults, "\n"

    # nonzero() returns the indices of the elements that are non-zero,
    # here the elements are the indices of the active columns
    activeColumns = l1SpRegion.getOutputData("bottomUpOut").nonzero()[0]

    # Calculate the anomaly score using the active columns
    # and previous predicted columns
    anomalyScore = computeRawAnomalyScore(activeColumns, prevPredictedColumns)

    # Write record number, consumption, and anomaly score
    writer.writerow((i, consumption, anomalyScore))

    # Store the predicted columns for the next timestep
    predictedColumns = l1TpRegion.getOutputData("topDownOut").nonzero()[0]
    prevPredictedColumns = copy.deepcopy(predictedColumns)

def runDemo():
  trainFile = findDataset(_INPUT_FILE_PATH)
  dataSource = FileRecordStream(streamID=trainFile)
  # numRecords = dataSource.getDataRowCount()
  numRecords = 100
  network = createNetwork(dataSource)
  outputPath = os.path.join(os.path.dirname(__file__), _OUTPUT_FILE_NAME)
  with open(outputPath, "w") as outputFile:
    writer = csv.writer(outputFile)
    print "Writing output to: %s" % outputPath
    runNetwork(network, numRecords, writer)

if __name__ == "__main__":
  runDemo()
<commit_msg>Added anomaly scoring for the second hierarchical level<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
An example of a hierarchy of cortical regions in a Network.
"""

import copy
import csv
import json
import os

from nupic.algorithms.anomaly import computeRawAnomalyScore
from nupic.data.datasethelpers import findDataset
from nupic.data.file_record_stream import FileRecordStream
from nupic.engine import Network
from nupic.encoders import MultiEncoder

# Level of detail of console output. Int value from 0 (none)
# to 3 (super detailed)
_VERBOSITY = 0

# Seed used for random number generation
_SEED = 2045
_INPUT_FILE_PATH = "../prediction/data/extra/hotgym/rec-center-hourly.csv"
_OUTPUT_FILE_NAME = "hierarchy-demo-output.csv"

# Parameter dict for SPRegion
SP_PARAMS = {"spVerbosity": _VERBOSITY,
             "spatialImp": "cpp",
             "seed": _SEED,

             # determined and set during network creation
             "inputWidth": 0,

             # @see nupic.research.spatial_pooler.SpatialPooler for explanations
             "globalInhibition": 1,
             "columnCount": 2048,
             "numActiveColumnsPerInhArea": 40,
             "potentialPct": 0.8,
             "synPermConnected": 0.1,
             "synPermActiveInc": 0.0001,
             "synPermInactiveDec": 0.0005,
             "maxBoost": 1.0}

# Parameter dict for TPRegion
TP_PARAMS = {"verbosity": _VERBOSITY,
             "temporalImp": "cpp",
             "seed": _SEED,

             # @see nupic.research.temporal_memory.TemporalMemory
             # for explanations
             "columnCount": 2048,
             "cellsPerColumn": 12,
             "inputWidth": 2048,
             "newSynapseCount": 20,
             "maxSynapsesPerSegment": 32,
             "maxSegmentsPerCell": 128,
             "initialPerm": 0.21,
             "permanenceInc": 0.1,
             "permanenceDec": 0.1,
             "globalDecay": 0.0,
             "maxAge": 0,
             "minThreshold": 9,
             "activationThreshold": 12,
             "outputType": "normal",
             "pamLength": 3}

_RECORD_SENSOR = "sensorRegion"
_L1_SPATIAL_POOLER = "l1spatialPoolerRegion"
_L1_TEMPORAL_POOLER = "l1temporalPoolerRegion"
_L1_CLASSIFIER = "l1classifier"

_L2_SPATIAL_POOLER = "l2spatialPoolerRegion"
_L2_TEMPORAL_POOLER = "l2temporalPoolerRegion"

def createEncoder():
  """
  Creates and returns a #MultiEncoder including a ScalarEncoder for
  energy consumption and a DateEncoder for the time of the day.

  @see nupic/encoders/__init__.py for type to file-name mapping
  @see nupic/encoders for encoder source files
  """
  encoder = MultiEncoder()
  encoder.addMultipleEncoders({
      "consumption": {"fieldname": u"consumption",
                      "type": "ScalarEncoder",
                      "name": u"consumption",
                      "minval": 0.0,
                      "maxval": 100.0,
                      "clipInput": True,
                      "w": 21,
                      "n": 50},
      "timestamp_timeOfDay": {"fieldname": u"timestamp",
                              "type": "DateEncoder",
                              "name": u"timestamp_timeOfDay",
                              "timeOfDay": (21, 9.5)}
  })
  return encoder

def createRecordSensor(network, name, dataSource):
  """
  Creates a RecordSensor region that allows us to specify a file record
  stream as the input source.
  """

  # Specific type of region. Possible options can be found in /nupic/regions/
  regionType = "py.RecordSensor"

  # Creates a json from specified dictionary.
  regionParams = json.dumps({"verbosity": _VERBOSITY})
  network.addRegion(name, regionType, regionParams)

  # getSelf returns the actual region, instead of a region wrapper
  sensorRegion = network.regions[name].getSelf()

  # Specify how RecordSensor encodes input values
  sensorRegion.encoder = createEncoder()

  # Specify the dataSource as a file record stream instance
  sensorRegion.dataSource = dataSource
  return sensorRegion

def createSpatialPooler(network, name, inputWidth):
  # Create the spatial pooler region
  SP_PARAMS["inputWidth"] = inputWidth
  spatialPoolerRegion = network.addRegion(name, "py.SPRegion",
                                          json.dumps(SP_PARAMS))
  # Make sure learning is enabled
  spatialPoolerRegion.setParameter("learningMode", True)
  # We want temporal anomalies so disable anomalyMode in the SP. This mode is
  # used for computing anomalies in a non-temporal model.
  spatialPoolerRegion.setParameter("anomalyMode", False)
  return spatialPoolerRegion

def createTemporalPooler(network, name):
  temporalPoolerRegion = network.addRegion(name, "py.TPRegion",
                                           json.dumps(TP_PARAMS))
  # Enable topDownMode to get the predicted columns output
  temporalPoolerRegion.setParameter("topDownMode", True)
  # Make sure learning is enabled (this is the default)
  temporalPoolerRegion.setParameter("learningMode", True)
  # Enable inference mode so we get predictions
  temporalPoolerRegion.setParameter("inferenceMode", True)
  # Enable anomalyMode to compute the anomaly score. This actually doesn't work
  # now so doesn't matter. We instead compute the anomaly score based on
  # topDownOut (predicted columns) and SP bottomUpOut (active columns).
  temporalPoolerRegion.setParameter("anomalyMode", True)
  return temporalPoolerRegion

def createNetwork(dataSource):
  """Creates and returns a new Network with a sensor region reading data from
  'dataSource'. There are two hierarchical levels, each with one SP and one TP.
  @param dataSource - A RecordStream containing the input data
  @returns a Network ready to run
  """
  network = Network()

  # Create and add a record sensor and a SP region
  sensor = createRecordSensor(network, name=_RECORD_SENSOR,
                              dataSource=dataSource)
  createSpatialPooler(network, name=_L1_SPATIAL_POOLER,
                      inputWidth=sensor.encoder.getWidth())

  # Link the SP region to the sensor input
  linkType = "UniformLink"
  linkParams = ""
  network.link(_RECORD_SENSOR, _L1_SPATIAL_POOLER, linkType, linkParams)

  # Create and add a TP region
  l1temporalPooler = createTemporalPooler(network, _L1_TEMPORAL_POOLER)

  # Link SP region to TP region in the feedforward direction
  network.link(_L1_SPATIAL_POOLER, _L1_TEMPORAL_POOLER, linkType, linkParams)

  # Add a classifier
  clParams = {  # Classifier learning/forgetting rate. Higher
                # values make it adapt faster and forget older patterns faster.
                'alpha': 0.005,

                # Comma separated list of the desired steps of
                # prediction that the classifier should learn
                'steps': '1',

                # Which implementation of the classifier to use.
                # See CLAClassifierFactory#create
                'implementation': 'cpp',

                # Diagnostic output verbosity control;
                # 0: silent; [1..6]: increasing levels of verbosity
                'clVerbosity': 0}
  network.addRegion(_L1_CLASSIFIER, "py.CLAClassifierRegion",
                    json.dumps(clParams))
  network.link(_RECORD_SENSOR, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="categoryOut", destInput="categoryIn")
  network.link(_L1_TEMPORAL_POOLER, _L1_CLASSIFIER, linkType, linkParams,
               srcOutput="bottomUpOut", destInput="bottomUpIn")

  # Second Level
  l2inputWidth = l1temporalPooler.getSelf().getOutputElementCount("bottomUpOut")
  createSpatialPooler(network, name=_L2_SPATIAL_POOLER, inputWidth=l2inputWidth)
  network.link(_L1_TEMPORAL_POOLER, _L2_SPATIAL_POOLER, linkType, linkParams)

  createTemporalPooler(network, _L2_TEMPORAL_POOLER)
  network.link(_L2_SPATIAL_POOLER, _L2_TEMPORAL_POOLER, linkType, linkParams)
  return network

def runNetwork(network, numRecords, writer):
  """
  Runs specified Network writing the ensuing anomaly score to writer.

  @param network: The Network instance to be run
  @param writer: A csv.writer used to write to output file.
  """
  sensorRegion = network.regions[_RECORD_SENSOR]
  l1SpRegion = network.regions[_L1_SPATIAL_POOLER]
  l1TpRegion = network.regions[_L1_TEMPORAL_POOLER]
  l1Classifier = network.regions[_L1_CLASSIFIER]

  l2SpRegion = network.regions[_L2_SPATIAL_POOLER]
  l2TpRegion = network.regions[_L2_TEMPORAL_POOLER]
  # TODO Print something out from L2?

  l1PreviousPredictedColumns = []
  l2PreviousPredictedColumns = []
  for i in xrange(numRecords):
    # Run the network for a single iteration
    network.run(1)

    l1tpOutput = l1TpRegion.getOutputData("bottomUpOut").nonzero()[0]
    # print type(l1tpOutput)
    consumption = float(sensorRegion.getOutputData("sourceOut")[0])
    bucketIndex = float(sensorRegion.getOutputData("categoryOut")[0])

    clDict = {"actValue": consumption, "bucketIdx": bucketIndex}

    # patternNZ:      list of the active indices from the output below
    # classification: dict of the classification information:
    #                   bucketIdx: index of the encoder bucket
    #                   actValue:  actual value going into the encoder
    #
    # retval:     dict containing inference results, one entry for each step in
    #             self.steps. The key is the number of steps, the value is an
    #             array containing the relative likelihood for each bucketIdx
    #             starting from bucketIdx 0.
    #
    #             for example:
    #               {1 : [0.1, 0.3, 0.2, 0.7]
    #                4 : [0.2, 0.4, 0.3, 0.5]}
    results = l1Classifier.getSelf().customCompute(recordNum=i,
                                                   patternNZ=l1tpOutput,
                                                   classification=clDict)
    # print "results: ", results, "\n"

    # nonzero() returns the indices of the elements that are non-zero,
    # here the elements are the indices of the active columns
    l1ActiveColumns = l1SpRegion.getOutputData("bottomUpOut").nonzero()[0]
    l2ActiveColumns = l2SpRegion.getOutputData("bottomUpOut").nonzero()[0]

    # Calculate the anomaly score using the active columns
    # and previous predicted columns
    l1AnomalyScore = computeRawAnomalyScore(l1ActiveColumns,
                                            l1PreviousPredictedColumns)
    l2AnomalyScore = computeRawAnomalyScore(l2ActiveColumns,
                                            l2PreviousPredictedColumns)

    # Write record number, consumption, and anomaly scores
    writer.writerow((i, consumption, l1AnomalyScore, l2AnomalyScore))

    # Store the predicted columns for the next timestep
    l1PredictedColumns = l1TpRegion.getOutputData("topDownOut").nonzero()[0]
    l1PreviousPredictedColumns = copy.deepcopy(l1PredictedColumns)
    #
    l2PredictedColumns = l2TpRegion.getOutputData("topDownOut").nonzero()[0]
    l2PreviousPredictedColumns = copy.deepcopy(l2PredictedColumns)

def runDemo():
  trainFile = findDataset(_INPUT_FILE_PATH)
  dataSource = FileRecordStream(streamID=trainFile)
  # numRecords = dataSource.getDataRowCount()
  numRecords = 100
  network = createNetwork(dataSource)
  outputPath = os.path.join(os.path.dirname(__file__), _OUTPUT_FILE_NAME)
  with open(outputPath, "w") as outputFile:
    writer = csv.writer(outputFile)
    print "Writing output to: %s" % outputPath
    runNetwork(network, numRecords, writer)

if __name__ == "__main__":
  runDemo()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import TestCase
from django.test.client import Client

class TemplateTagsTest(TestCase):
    urls = 'server.tests.urls'

    def test_csrf_token(self):
        client = Client(enforce_csrf_checks=True)
        response = client.get('/straight_methods/')
        self.assertContains(response, '<script>var x="')

    def test_load_all_urls(self):
        client = Client()
        response = client.get('/url_resolvers/')
        self.assertIn('"submethods:sub:app": "/sub_methods/sub/app/"', str(response.content))
        self.assertIn('"straightmethods": "/straight_methods/"', str(response.content))

    def test_load_self_urls(self):
        client = Client()
        response = client.get('/sub_methods/sub/app/', {'load': 'self_urls'})
        self.assertIn('"submethods:sub:app": "/sub_methods/sub/app/"', str(response.content))
        self.assertNotIn('"straightmethods": "/straight_methods/"', str(response.content))

    def test_load_root_urls(self):
        client = Client()
        response = client.get('/url_resolvers/', {'load': 'root_urls'})
        self.assertNotIn('"submethods:sub:app": "/sub_methods/sub/app/"', str(response.content))
        self.assertIn('"straightmethods": "/straight_methods/"', str(response.content))

    def test_angularon_tag(self):
        client = Client()
        response = client.get('/angular_tag/', {'switch': ''})
        self.assertContains(response, 'Hello World')
        response = client.get('/angular_tag/', {'switch': 'yes'})
        self.assertNotContains(response, 'Hello World')
        response = client.get('/angular_tag/', {'switch': '', 'tmpl_id': 'expand_object'})
        self.assertNotContains(response, 'expandme.foo')
        response = client.get('/angular_tag/', {'switch': 'yes', 'tmpl_id': 'expand_object'})
        self.assertContains(response, 'expandme.foo')
        response = client.get('/angular_tag/', {'switch': 'yes', 'tmpl_id': 'expand_array'})
        self.assertContains(response, 'expandme[1].foo')
        response = client.get('/angular_tag/', {'switch': '', 'tmpl_id': 'expand_array'})
        self.assertContains(response, 'one')
<commit_msg>remove templatetag tests for load_djng_urls<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import TestCase
from django.test.client import Client

class TemplateTagsTest(TestCase):
    urls = 'server.tests.urls'

    def test_csrf_token(self):
        client = Client(enforce_csrf_checks=True)
        response = client.get('/straight_methods/')
        self.assertContains(response, '<script>var x="')

    def test_angularon_tag(self):
        client = Client()
        response = client.get('/angular_tag/', {'switch': ''})
        self.assertContains(response, 'Hello World')
        response = client.get('/angular_tag/', {'switch': 'yes'})
        self.assertNotContains(response, 'Hello World')
        response = client.get('/angular_tag/', {'switch': '', 'tmpl_id': 'expand_object'})
        self.assertNotContains(response, 'expandme.foo')
        response = client.get('/angular_tag/', {'switch': 'yes', 'tmpl_id': 'expand_object'})
        self.assertContains(response, 'expandme.foo')
        response = client.get('/angular_tag/', {'switch': 'yes', 'tmpl_id': 'expand_array'})
        self.assertContains(response, 'expandme[1].foo')
        response = client.get('/angular_tag/', {'switch': '', 'tmpl_id': 'expand_array'})
        self.assertContains(response, 'one')
<|endoftext|>


<commit_before># Portuguese Language Test
from seleniumbase.translate.portuguese import CasoDeTeste

class MinhaClasseDeTeste(CasoDeTeste):
    def test_exemplo_1(self):
        self.abrir("https://pt.wikipedia.org/wiki/")
        self.verificar_texto("Wikipédia")
        self.verificar_elemento('[title="Língua portuguesa"]')
        self.js_digitar("#searchform input", "João Pessoa")
        self.clique("#searchform button")
        self.verificar_texto("João Pessoa", "#firstHeading")
        self.verificar_elemento('img[alt*="João Pessoa"]')
        self.js_digitar("#searchform input", "Florianópolis")
        self.clique("#searchform button")
        self.verificar_texto("Florianópolis", "h1#firstHeading")
        self.verificar_elemento('td:contains("Avenida Beira-Mar")')
        self.voltar()
        self.verificar_verdade("João" in self.obter_url_atual())
        self.js_digitar("#searchform input", "Teatro Amazonas")
        self.clique("#searchform button")
        self.verificar_texto("Teatro Amazonas", "#firstHeading")
        self.verificar_texto_do_link("Festival Amazonas de Ópera")
<commit_msg>Update the Portuguese example test<commit_after># Portuguese Language Test
from seleniumbase.translate.portuguese import CasoDeTeste

class MinhaClasseDeTeste(CasoDeTeste):
    def test_exemplo_1(self):
        self.abrir("https://pt.wikipedia.org/wiki/")
        self.verificar_texto("Wikipédia")
        self.verificar_elemento('[title="Língua portuguesa"]')
        self.digitar("#searchform input", "João Pessoa")
        self.clique("#searchform button")
        self.verificar_texto("João Pessoa", "#firstHeading")
        self.verificar_elemento('img[alt*="João Pessoa"]')
        self.digitar("#searchform input", "Florianópolis")
        self.clique("#searchform button")
        self.verificar_texto("Florianópolis", "h1#firstHeading")
        self.verificar_elemento('td:contains("Avenida Beira-Mar")')
        self.voltar()
        self.verificar_verdade("João" in self.obter_url_atual())
        self.digitar("#searchform input", "Teatro Amazonas")
        self.clique("#searchform button")
        self.verificar_texto("Teatro Amazonas", "#firstHeading")
        self.verificar_texto_do_link("Festival Amazonas de Ópera")
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Boston housing price regression dataset.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.keras._impl.keras.utils.data_utils import get_file

def load_data(path='boston_housing.npz', seed=113, test_split=0.2):
  """Loads the Boston Housing dataset.

  Arguments:
      path: path where to cache the dataset locally
          (relative to ~/.keras/datasets).
      seed: Random seed for shuffling the data
          before computing the test split.
      test_split: fraction of the data to reserve as test set.

  Returns:
      Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
  """
  assert 0 <= test_split < 1
  fh = 'f553886a1f8d56431e820c5b82552d9d95cfcb96d1e678153f8839538947dff5'
  path = get_file(
      path,
      origin='https://s3.amazonaws.com/keras-datasets/boston_housing.npz',
      file_hash=fh)
  f = np.load(path)
  x = f['x']
  y = f['y']
  f.close()

  np.random.seed(seed)
  indices = np.arrange(len(x))
  np.random.shuffle(indices)
  x = x[indices]
  y = y[indices]

  x_train = np.array(x[:int(len(x) * (1 - test_split))])
  y_train = np.array(y[:int(len(x) * (1 - test_split))])
  x_test = np.array(x[int(len(x) * (1 - test_split)):])
  y_test = np.array(y[int(len(x) * (1 - test_split)):])
  return (x_train, y_train), (x_test, y_test)
<commit_msg>Fix typo in boston housing dataset loader.<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Boston housing price regression dataset.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.keras._impl.keras.utils.data_utils import get_file

def load_data(path='boston_housing.npz', seed=113, test_split=0.2):
  """Loads the Boston Housing dataset.

  Arguments:
      path: path where to cache the dataset locally
          (relative to ~/.keras/datasets).
      seed: Random seed for shuffling the data
          before computing the test split.
      test_split: fraction of the data to reserve as test set.

  Returns:
      Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
  """
  assert 0 <= test_split < 1
  fh = 'f553886a1f8d56431e820c5b82552d9d95cfcb96d1e678153f8839538947dff5'
  path = get_file(
      path,
      origin='https://s3.amazonaws.com/keras-datasets/boston_housing.npz',
      file_hash=fh)
  f = np.load(path)
  x = f['x']
  y = f['y']
  f.close()

  np.random.seed(seed)
  indices = np.arange(len(x))
  np.random.shuffle(indices)
  x = x[indices]
  y = y[indices]

  x_train = np.array(x[:int(len(x) * (1 - test_split))])
  y_train = np.array(y[:int(len(x) * (1 - test_split))])
  x_test = np.array(x[int(len(x) * (1 - test_split)):])
  y_test = np.array(y[int(len(x) * (1 - test_split)):])
  return (x_train, y_train), (x_test, y_test)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# USAGE:
#   dynamic_url.py
#
# INPUT:
#   json, ds, hr
#
# INPUT ORDERING:
#   none
#
# OUTPUT:
#   json
#
# WORKFLOW:
#   1. accept 3 fields (json string, date string, hour string)
#   2. merge into single json object
#   3. determine page_key, append to json object
#   4. emit json object
#
# DESCRIPTION:
#   Simple decorator pattern. This scripts accepts json logs out of hive
#   with a date and hour partition. It determines the "dynamic url" of the
#   event, appeds the value and emits for any downstream source.

import sys
import hashlib
import httplib

import cjson

# functions #####################################################################

account_ids = {}

# functions #####################################################################

def sha1(str):
	h = hashlib.sha1()
	h.update(str)
	return h.hexdigest()

# body ##########################################################################

# pull config distributor url from argv[1]
config_distrib = sys.argv[1]
sys.stderr.write("Config Distributor: %s\n" % (config_distrib))

# rest/json request
conn = httplib.HTTPConnection(config_distrib)
conn.request("GET", "/Config/dcsid2account")
response = conn.getresponse()
if response.status != 200:
	sys.stderr.write("Bad http request status: %i %s\n" % (response.status, response.reason))
	exit(-1)

# parse dcsid -> account-id mappings
try:
	account_ids = cjson.decode(response.read())
except Exception, e:
	sys.stderr.write("Unable to parse json from http result\n")
	raise

# start sucking on the data
while True:
	try:
		line = sys.stdin.readline()
		if not line:
			break

		# get parameters (workflow #1)
		params = line.rstrip("\n").split("\t") # "json","ds","hr"
		tmp = cjson.decode(params[0])

		# convert WT.blah params to WT_blah
		obj = {
			"WT_hm_w": tmp["WT.hm_w"]
			, "WT_hm_h": tmp["WT.hm_h"]
			, "WT_hm_x": tmp["WT.hm_x"]
			, "WT_hm_y": tmp["WT.hm_y"]
			, "ds": params[1]
			, "hr": params[2]
		}

		# only continue if we know the account-id for the dcs-id
		if not(tmp["dcs-id"] in account_ids):
			continue

		# determine page hash (workflow #3)
		page_ident = tmp["cs-uri-stem"]
		if "WT.hm_url" in tmp:
			page_ident += "?" + tmp["WT.hm_url"]

		account_id = account_ids[tmp["dcs-id"]]
		obj["page_key"] = str(account_id) + ";" + tmp["cs-host"] + ";" + sha1(page_ident)

		# emit (workflow #4)
		sys.stdout.write("%s\n" % (cjson.encode(obj)))

	except Exception, e:
		sys.stderr.write("error: %s on line %s\n" % (e, line))

<commit_msg>emitting cs-uri-stem in reducer for heatmap prodding downstream<commit_after>#!/usr/bin/env python

# USAGE:
#   dynamic_url.py
#
# INPUT:
#   json, ds, hr
#
# INPUT ORDERING:
#   none
#
# OUTPUT:
#   json
#
# WORKFLOW:
#   1. accept 3 fields (json string, date string, hour string)
#   2. merge into single json object
#   3. determine page_key, append to json object
#   4. emit json object
#
# DESCRIPTION:
#   Simple decorator pattern. This scripts accepts json logs out of hive
#   with a date and hour partition. It determines the "dynamic url" of the
#   event, appeds the value and emits for any downstream source.

import sys
import hashlib
import httplib

import cjson

# functions #####################################################################

account_ids = {}

# functions #####################################################################

def sha1(str):
	h = hashlib.sha1()
	h.update(str)
	return h.hexdigest()

# body ##########################################################################

# pull config distributor url from argv[1]
config_distrib = sys.argv[1]
sys.stderr.write("Config Distributor: %s\n" % (config_distrib))

# rest/json request
conn = httplib.HTTPConnection(config_distrib)
conn.request("GET", "/Config/dcsid2account")
response = conn.getresponse()
if response.status != 200:
	sys.stderr.write("Bad http request status: %i %s\n" % (response.status, response.reason))
	exit(-1)

# parse dcsid -> account-id mappings
try:
	account_ids = cjson.decode(response.read())
except Exception, e:
	sys.stderr.write("Unable to parse json from http result\n")
	raise

# start sucking on the data
while True:
	try:
		line = sys.stdin.readline()
		if not line:
			break

		# get parameters (workflow #1)
		params = line.rstrip("\n").split("\t") # "json","ds","hr"
		tmp = cjson.decode(params[0])

		# convert WT.blah params to WT_blah
		obj = {
			"WT_hm_w": tmp["WT.hm_w"]
			, "WT_hm_h": tmp["WT.hm_h"]
			, "WT_hm_x": tmp["WT.hm_x"]
			, "WT_hm_y": tmp["WT.hm_y"]
			, "ds": params[1]
			, "hr": params[2]
			, "cs-uri-stem": tmp["cs-uri-stem"]
		}

		# only continue if we know the account-id for the dcs-id
		if not(tmp["dcs-id"] in account_ids):
			continue

		# determine page hash (workflow #3)
		page_ident = tmp["cs-uri-stem"]
		if "WT.hm_url" in tmp:
			page_ident += "?" + tmp["WT.hm_url"]

		account_id = account_ids[tmp["dcs-id"]]
		obj["page_key"] = str(account_id) + ";" + tmp["cs-host"] + ";" + sha1(page_ident)

		# emit (workflow #4)
		sys.stdout.write("%s\n" % (cjson.encode(obj)))

	except Exception, e:
		sys.stderr.write("error: %s on line %s\n" % (e, line))

<|endoftext|>


<commit_before>from UM.OutputDevice.OutputDevicePlugin import OutputDevicePlugin
from . import NetworkPrinterOutputDevice

from zeroconf import Zeroconf, ServiceBrowser, ServiceStateChange
from UM.Signal import Signal, signalemitter
from UM.Application import Application

##      This plugin handles the connection detection & creation of output device objects for the UM3 printer.
#       Zero-Conf is used to detect printers, which are saved in a dict.
#       If we discover a printer that has the same key as the active machine instance a connection is made.
@signalemitter
class NetworkPrinterOutputDevicePlugin(OutputDevicePlugin):
    def __init__(self):
        super().__init__()
        self._zero_conf = Zeroconf()
        self._browser = None
        self._printers = {}

        # Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
        self.addPrinterSignal.connect(self.addPrinter)
        Application.getInstance().globalContainerStackChanged.connect(self.reCheckConnections)

    addPrinterSignal = Signal()

    ##  Start looking for devices on network.
    def start(self):
        self._browser = ServiceBrowser(self._zero_conf, u'_ultimaker._tcp.local.', [self._onServiceChanged])

    ##  Stop looking for devices on network.
    def stop(self):
        self._zero_conf.close()

    def getPrinters(self):
        return self._printers

    def reCheckConnections(self):
        active_machine = Application.getInstance().getGlobalContainerStack()
        if not active_machine:
            return

        for key in self._printers:
            if key == active_machine.getMetaDataEntry("um_network_key"):
                self._printers[key].connect()
                self._printers[key].connectionStateChanged.connect(self._onPrinterConnectionStateChanged)
            else:
                self._printers[key].close()

    ##  Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
    def addPrinter(self, name, address, properties):
        printer = NetworkPrinterOutputDevice.NetworkPrinterOutputDevice(name, address, properties)
        self._printers[printer.getKey()] = printer
        global_container_stack = Application.getInstance().getGlobalContainerStack()
        if global_container_stack and printer.getKey() == global_container_stack.getMetaDataEntry("um_network_key"):
            self._printers[printer.getKey()].connect()
            printer.connectionStateChanged.connect(self._onPrinterConnectionStateChanged)

    ##  Handler for when the connection state of one of the detected printers changes
    def _onPrinterConnectionStateChanged(self, key):
        if self._printers[key].isConnected():
            self.getOutputDeviceManager().addOutputDevice(self._printers[key])
        else:
            self.getOutputDeviceManager().removeOutputDevice(key)

    ##  Handler for zeroConf detection
    def _onServiceChanged(self, zeroconf, service_type, name, state_change):
        if state_change == ServiceStateChange.Added:
            info = zeroconf.get_service_info(service_type, name)
            if info:
                if info.properties.get(b"type", None) == b'printer':
                    address = '.'.join(map(lambda n: str(n), info.address))
                    self.addPrinterSignal.emit(str(name), address, info.properties)

        elif state_change == ServiceStateChange.Removed:
            pass
            # TODO; This isn't testable right now. We need to also decide how to handle
            #<commit_msg>Only close connection if it's open when re-checking connections<commit_after>from UM.OutputDevice.OutputDevicePlugin import OutputDevicePlugin
from . import NetworkPrinterOutputDevice

from zeroconf import Zeroconf, ServiceBrowser, ServiceStateChange
from UM.Signal import Signal, signalemitter
from UM.Application import Application

##      This plugin handles the connection detection & creation of output device objects for the UM3 printer.
#       Zero-Conf is used to detect printers, which are saved in a dict.
#       If we discover a printer that has the same key as the active machine instance a connection is made.
@signalemitter
class NetworkPrinterOutputDevicePlugin(OutputDevicePlugin):
    def __init__(self):
        super().__init__()
        self._zero_conf = Zeroconf()
        self._browser = None
        self._printers = {}

        # Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
        self.addPrinterSignal.connect(self.addPrinter)
        Application.getInstance().globalContainerStackChanged.connect(self.reCheckConnections)

    addPrinterSignal = Signal()

    ##  Start looking for devices on network.
    def start(self):
        self._browser = ServiceBrowser(self._zero_conf, u'_ultimaker._tcp.local.', [self._onServiceChanged])

    ##  Stop looking for devices on network.
    def stop(self):
        self._zero_conf.close()

    def getPrinters(self):
        return self._printers

    def reCheckConnections(self):
        active_machine = Application.getInstance().getGlobalContainerStack()
        if not active_machine:
            return

        for key in self._printers:
            if key == active_machine.getMetaDataEntry("um_network_key"):
                self._printers[key].connect()
                self._printers[key].connectionStateChanged.connect(self._onPrinterConnectionStateChanged)
            else:
                if self._printers[key].isConnected():
                    self._printers[key].close()

    ##  Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
    def addPrinter(self, name, address, properties):
        printer = NetworkPrinterOutputDevice.NetworkPrinterOutputDevice(name, address, properties)
        self._printers[printer.getKey()] = printer
        global_container_stack = Application.getInstance().getGlobalContainerStack()
        if global_container_stack and printer.getKey() == global_container_stack.getMetaDataEntry("um_network_key"):
            self._printers[printer.getKey()].connect()
            printer.connectionStateChanged.connect(self._onPrinterConnectionStateChanged)

    ##  Handler for when the connection state of one of the detected printers changes
    def _onPrinterConnectionStateChanged(self, key):
        if self._printers[key].isConnected():
            self.getOutputDeviceManager().addOutputDevice(self._printers[key])
        else:
            self.getOutputDeviceManager().removeOutputDevice(key)

    ##  Handler for zeroConf detection
    def _onServiceChanged(self, zeroconf, service_type, name, state_change):
        if state_change == ServiceStateChange.Added:
            info = zeroconf.get_service_info(service_type, name)
            if info:
                if info.properties.get(b"type", None) == b'printer':
                    address = '.'.join(map(lambda n: str(n), info.address))
                    self.addPrinterSignal.emit(str(name), address, info.properties)

        elif state_change == ServiceStateChange.Removed:
            pass
            # TODO; This isn't testable right now. We need to also decide how to handle
            #<|endoftext|>


<commit_before>
<commit_msg>Delete __init__.py<commit_after><|endoftext|>


<commit_before>import re

from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import connections

from corehq.form_processor.utils.sql import fetchall_as_namedtuple
from corehq.sql_db.config import ShardMeta, plproxy_config, plproxy_standby_config

SHARD_OPTION_RX = re.compile(r'^p[\d+]')

SERVER_TEMPLATE = """
    CREATE SERVER {server_name} FOREIGN DATA WRAPPER plproxy
    OPTIONS (
        connection_lifetime '1800',
        disable_binary '1',
        {partitions}
    );
"""

USER_MAPPING_TEMPLATE = """
    CREATE USER MAPPING FOR {USER} SERVER {server_name} OPTIONS (user '{USER}', password '{PASSWORD}');
"""

ALTER_SERVER_TEMPLATE = """
    ALTER SERVER {server_name} OPTIONS (
        {options}
    );
"""

class Command(BaseCommand):
    help = 'Creates or updates the pl_proxy cluster configuration'

    def add_arguments(self, parser):
        parser.add_argument(
            '--verbose',
            action='store_true',
            dest='verbose',
            default=False,
        )
        parser.add_argument(
            '--create_only',
            action='store_true',
            dest='create_only',
            default=False,
        )

    def handle(self, **options):
        if not settings.USE_PARTITIONED_DATABASE:
            print("System not configured to use a partitioned database")

        verbose = options['verbose']
        create_or_update_cluster(plproxy_config, verbose, options['create_only'])
        create_or_update_cluster(plproxy_standby_config, verbose, options['create_only'])

def create_or_update_cluster(cluster_config, verbose, create_only):
    existing_config = _get_existing_cluster_config(cluster_config)
    if existing_config:
        if create_only:
            return
        if _confirm(f"Cluster configuration already exists on '{cluster_config.proxy_db}'."
                    f" Are you sure you want to change it?"):
            _update_pl_proxy_cluster(cluster_config, existing_config, verbose)
    else:
        create_pl_proxy_cluster(cluster_config, verbose)

def parse_existing_shard(shard_option):
    shard_name, options = shard_option.split('=', 1)
    assert shard_name[0] == 'p'
    shard_id = int(shard_name[1:])
    options = options.split(' ')
    option_kwargs = dict(tuple(option.split('=')) for option in options)
    if 'port' in option_kwargs:
        option_kwargs['port'] = int(option_kwargs['port'])
    return ShardMeta(id=shard_id, **option_kwargs)

def _get_current_shards(existing_config):
    existing_shards = [
        parse_existing_shard(option)
        for option in existing_config.srvoptions if _is_shard_option(option)
    ]
    return existing_shards

def _is_shard_option(option):
    return SHARD_OPTION_RX.match(option)

def _update_pl_proxy_cluster(cluster_config, existing_config, verbose):
    existing_shards = _get_current_shards(existing_config)
    new_shard_configs = cluster_config.get_shards()

    if verbose:
        print('{0} Existing config {0}'.format('-' * 42))
        print(existing_config)
        print('-' * 100)

    shards_to_update = get_shards_to_update(existing_shards, new_shard_configs)

    if not shards_to_update:
        print('No changes. Exiting.')
    else:
        print("Shards to update:")
        existing_shards_by_id = {shard.id: shard for shard in existing_shards}
        for new in shards_to_update:
            print("    {}  ->   {}".format(
                existing_shards_by_id[new.id].get_server_option_string(),
                new.get_server_option_string()
            ))
        if _confirm("Update these shards?"):
            alter_sql = _get_alter_server_sql(cluster_config.cluster_name, shards_to_update)
            if verbose:
                print(alter_sql)

            with connections[cluster_config.proxy_db].cursor() as cursor:
                cursor.execute(alter_sql)
        else:
            print('Abort')

def _get_alter_server_sql(cluster_name, shards_to_update):
    shard_option_template = "SET {}"
    shards_sql = []
    for shard in shards_to_update:
        shards_sql.append(shard_option_template.format(shard.get_server_option_string()))

    return ALTER_SERVER_TEMPLATE.format(
        server_name=cluster_name,
        options=',\n'.join(shards_sql)
    )

def create_pl_proxy_cluster(cluster_config, verbose=False, drop_existing=False):
    proxy_db = cluster_config.proxy_db

    sql = get_sql_to_create_pl_proxy_cluster(cluster_config, drop_existing)

    with connections[proxy_db].cursor() as cursor:
        for command in sql:
            if verbose:
                print(f'\t{command}')
            cursor.execute(command)

def get_sql_to_create_pl_proxy_cluster(cluster_config, drop_existing=False):
    sql = [get_drop_server_sql(cluster_config.cluster_name)] if drop_existing else []
    sql.extend([
        get_pl_proxy_server_config_sql(cluster_config.cluster_name, cluster_config.get_shards()),
        get_user_mapping_sql(cluster_config)
    ])
    return sql

def get_drop_server_sql(cluster_name):
    return 'DROP SERVER IF EXISTS {} CASCADE;'.format(cluster_name)

def _get_existing_cluster_config(cluster_config):
    proxy_db = cluster_config.proxy_db
    with connections[proxy_db].cursor() as cursor:
        cursor.execute('SELECT * from pg_foreign_server where srvname = %s', [cluster_config.cluster_name])
        results = list(fetchall_as_namedtuple(cursor))
        if results:
            return results[0]

def get_pl_proxy_server_config_sql(cluster_name, shards):
    shard_configs_strings = [shard.get_server_option_string() for shard in shards]

    server_sql = SERVER_TEMPLATE.format(
        server_name=cluster_name,
        partitions=',\n'.join(shard_configs_strings)
    )

    return server_sql

def get_user_mapping_sql(cluster_config):
    proxy_db = cluster_config.proxy_db
    proxy_db_config = settings.DATABASES[proxy_db].copy()
    proxy_db_config['server_name'] = cluster_config.cluster_name
    return USER_MAPPING_TEMPLATE.format(**proxy_db_config)

def _confirm(msg):
    confirm_update = input(msg + ' [yes / no] ')
    return confirm_update == 'yes'

def get_shards_to_update(existing_shards, new_shards):
    assert len(existing_shards) == len(new_shards)
    shards_to_update = []
    for existing, new in zip(existing_shards, new_shards):
        assert existing.id == new.id, '{} != {}'.format(existing.id, new.id)
        if existing != new:
            shards_to_update.append(new)

    return shards_to_update
<commit_msg>use a transaction<commit_after>import re

from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import connections, transaction

from corehq.form_processor.utils.sql import fetchall_as_namedtuple
from corehq.sql_db.config import ShardMeta, plproxy_config, plproxy_standby_config

SHARD_OPTION_RX = re.compile(r'^p[\d+]')

SERVER_TEMPLATE = """
    CREATE SERVER {server_name} FOREIGN DATA WRAPPER plproxy
    OPTIONS (
        connection_lifetime '1800',
        disable_binary '1',
        {partitions}
    );
"""

USER_MAPPING_TEMPLATE = """
    CREATE USER MAPPING FOR {USER} SERVER {server_name} OPTIONS (user '{USER}', password '{PASSWORD}');
"""

ALTER_SERVER_TEMPLATE = """
    ALTER SERVER {server_name} OPTIONS (
        {options}
    );
"""

class Command(BaseCommand):
    help = 'Creates or updates the pl_proxy cluster configuration'

    def add_arguments(self, parser):
        parser.add_argument(
            '--verbose',
            action='store_true',
            dest='verbose',
            default=False,
        )
        parser.add_argument(
            '--create_only',
            action='store_true',
            dest='create_only',
            default=False,
        )

    def handle(self, **options):
        if not settings.USE_PARTITIONED_DATABASE:
            print("System not configured to use a partitioned database")

        verbose = options['verbose']
        create_or_update_cluster(plproxy_config, verbose, options['create_only'])
        create_or_update_cluster(plproxy_standby_config, verbose, options['create_only'])

def create_or_update_cluster(cluster_config, verbose, create_only):
    existing_config = _get_existing_cluster_config(cluster_config)
    if existing_config:
        if create_only:
            return
        if _confirm(f"Cluster configuration already exists on '{cluster_config.proxy_db}'."
                    f" Are you sure you want to change it?"):
            _update_pl_proxy_cluster(cluster_config, existing_config, verbose)
    else:
        create_pl_proxy_cluster(cluster_config, verbose)

def parse_existing_shard(shard_option):
    shard_name, options = shard_option.split('=', 1)
    assert shard_name[0] == 'p'
    shard_id = int(shard_name[1:])
    options = options.split(' ')
    option_kwargs = dict(tuple(option.split('=')) for option in options)
    if 'port' in option_kwargs:
        option_kwargs['port'] = int(option_kwargs['port'])
    return ShardMeta(id=shard_id, **option_kwargs)

def _get_current_shards(existing_config):
    existing_shards = [
        parse_existing_shard(option)
        for option in existing_config.srvoptions if _is_shard_option(option)
    ]
    return existing_shards

def _is_shard_option(option):
    return SHARD_OPTION_RX.match(option)

def _update_pl_proxy_cluster(cluster_config, existing_config, verbose):
    existing_shards = _get_current_shards(existing_config)
    new_shard_configs = cluster_config.get_shards()

    if verbose:
        print('{0} Existing config {0}'.format('-' * 42))
        print(existing_config)
        print('-' * 100)

    shards_to_update = get_shards_to_update(existing_shards, new_shard_configs)

    if not shards_to_update:
        print('No changes. Exiting.')
    else:
        print("Shards to update:")
        existing_shards_by_id = {shard.id: shard for shard in existing_shards}
        for new in shards_to_update:
            print("    {}  ->   {}".format(
                existing_shards_by_id[new.id].get_server_option_string(),
                new.get_server_option_string()
            ))
        if _confirm("Update these shards?"):
            alter_sql = _get_alter_server_sql(cluster_config.cluster_name, shards_to_update)
            if verbose:
                print(alter_sql)

            with connections[cluster_config.proxy_db].cursor() as cursor:
                cursor.execute(alter_sql)
        else:
            print('Abort')

def _get_alter_server_sql(cluster_name, shards_to_update):
    shard_option_template = "SET {}"
    shards_sql = []
    for shard in shards_to_update:
        shards_sql.append(shard_option_template.format(shard.get_server_option_string()))

    return ALTER_SERVER_TEMPLATE.format(
        server_name=cluster_name,
        options=',\n'.join(shards_sql)
    )

def create_pl_proxy_cluster(cluster_config, verbose=False, drop_existing=False):
    proxy_db = cluster_config.proxy_db

    sql = get_sql_to_create_pl_proxy_cluster(cluster_config, drop_existing)

    with transaction.atomic(proxy_db), connections[proxy_db].cursor() as cursor:
        for command in sql:
            if verbose:
                print(f'\t{command}')
            cursor.execute(command)

def get_sql_to_create_pl_proxy_cluster(cluster_config, drop_existing=False):
    sql = [get_drop_server_sql(cluster_config.cluster_name)] if drop_existing else []
    sql.extend([
        get_pl_proxy_server_config_sql(cluster_config.cluster_name, cluster_config.get_shards()),
        get_user_mapping_sql(cluster_config)
    ])
    return sql

def get_drop_server_sql(cluster_name):
    return 'DROP SERVER IF EXISTS {} CASCADE;'.format(cluster_name)

def _get_existing_cluster_config(cluster_config):
    proxy_db = cluster_config.proxy_db
    with connections[proxy_db].cursor() as cursor:
        cursor.execute('SELECT * from pg_foreign_server where srvname = %s', [cluster_config.cluster_name])
        results = list(fetchall_as_namedtuple(cursor))
        if results:
            return results[0]

def get_pl_proxy_server_config_sql(cluster_name, shards):
    shard_configs_strings = [shard.get_server_option_string() for shard in shards]

    server_sql = SERVER_TEMPLATE.format(
        server_name=cluster_name,
        partitions=',\n'.join(shard_configs_strings)
    )

    return server_sql

def get_user_mapping_sql(cluster_config):
    proxy_db = cluster_config.proxy_db
    proxy_db_config = settings.DATABASES[proxy_db].copy()
    proxy_db_config['server_name'] = cluster_config.cluster_name
    return USER_MAPPING_TEMPLATE.format(**proxy_db_config)

def _confirm(msg):
    confirm_update = input(msg + ' [yes / no] ')
    return confirm_update == 'yes'

def get_shards_to_update(existing_shards, new_shards):
    assert len(existing_shards) == len(new_shards)
    shards_to_update = []
    for existing, new in zip(existing_shards, new_shards):
        assert existing.id == new.id, '{} != {}'.format(existing.id, new.id)
        if existing != new:
            shards_to_update.append(new)

    return shards_to_update
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'dan'

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import YamlPatcher
from cosmo_tester.framework.handlers.openstack import (
    openstack_clients,
    openstack_infra_state,
    openstack_infra_state_delta
)

class NeutronGaloreTest(TestCase):

    def test_neutron_galore(self):

        blueprint_path = self.copy_blueprint('neutron-galore')
        self.blueprint_yaml = blueprint_path / 'blueprint.yaml'
        self.modify_blueprint()

        before, after = self.upload_deploy_and_execute_install()

        node_states = self.get_delta_node_states(before, after)
        self.post_install_assertions(node_states)

        self._test_use_external_resource()

        self.execute_uninstall()

        self.post_uninstall_assertions()

    def modify_blueprint(self):
        with YamlPatcher(self.blueprint_yaml) as patch:
            vm_path = 'node_templates.nova_server.properties'
            patch.merge_obj('{0}.server'.format(vm_path), {
                'name': 'novaservertest',
                'image_name': self.env.ubuntu_image_name,
                'flavor_name': self.env.flavor_name,
                'security_groups': ['neutron_test_security_group_dst'],
            })

    def post_install_assertions(self, node_states):

        def p(name):
            return '{}{}'.format(self.env.resources_prefix, name)

        openstack = self.get_openstack_components(node_states)

        port_assigned_addr = openstack['server']['addresses'][
            p('neutron_network_test')][0]['addr']
        port_security_group_id = openstack['port']['security_groups'][0]
        port_fixed_ip = openstack['port']['fixed_ips'][0]['ip_address']
        port_network_id = openstack['port']['network_id']
        port_subnet_id = openstack['port']['fixed_ips'][0]['subnet_id']
        router_network_id = openstack['router']['external_gateway_info'][
            'network_id']
        sg_src_id = openstack['sg_src']['id']
        network_subnet_id = openstack['network']['subnets'][0]

        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]
                         ['OS-EXT-IPS:type'], 'fixed')
        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][1]
                         ['OS-EXT-IPS:type'], 'floating')
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]
                         ['OS-EXT-IPS:type'], 'fixed')
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]
                         ['version'], 4)
        self.assertTrue(port_assigned_addr.startswith('10.10.10.'))
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p('neutron_test_security_group_dst')})
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p(self.env.agents_security_group)})
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p('neutron_test_security_group_src')})
        self.assertEqual(openstack['server']['name'], p('novaservertest'))
        self.assertEqual(openstack['port']['name'], p('neutron_test_port'))
        self.assertEqual(port_fixed_ip, port_assigned_addr)

        self.assertEqual(openstack['floatingip']['floating_network_id'],
                         router_network_id)
        self.assertEqual(openstack['router']['name'], p('neutron_router_test'))
        self.assertEqual(openstack['sg_src']['name'],
                         p('neutron_test_security_group_src'))
        self.assertEqual(port_security_group_id, sg_src_id)
        self.assertEqual(openstack['network']['name'],
                         p('neutron_network_test'))
        self.assertEqual(port_network_id, openstack['network']['id'])
        self.assertEqual(openstack['subnet']['name'],
                         p('neutron_subnet_test'))
        self.assertEqual(openstack['subnet']['cidr'], '10.10.10.0/24')
        self.assertEqual(network_subnet_id, openstack['subnet']['id'])
        self.assertEqual(port_subnet_id, openstack['subnet']['id'])
        self.assertEqual(openstack['sg_dst']['name'],
                         p('neutron_test_security_group_dst'))
        self.assertEqual(4, len(openstack['sg_dst']['security_group_rules']))
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '1.2.3.0/24',
             'port_range_min': 80,
             'port_range_max': 80,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '2.3.4.0/24',
             'port_range_min': 65500,
             'port_range_max': 65510,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_group_id': sg_src_id,
             'port_range_min': 65521,
             'port_range_max': 65521,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '3.4.5.0/24',
             'port_range_min': 443,
             'port_range_max': 443,
             'direction': 'egress'})
        self.assertEqual(node_states['floatingip']['floating_ip_address'],
                         openstack['floatingip']['floating_ip_address'])
        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]['addr'],
                         node_states['server']['networks']
                         [p(self.env.management_network_name)][0])
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]['addr'],
                         node_states['server']['networks']
                         [p('neutron_network_test')][0])
        self.assertEqual(node_states['server']['ip'],
                         openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]['addr'])
        self.assert_router_connected_to_subnet(openstack['router']['id'],
                                               openstack['router_ports'],
                                               openstack['subnet']['id'])

    def post_uninstall_assertions(self):
        leftovers = self._test_cleanup_context.get_resources_to_teardown()
        self.assertTrue(all([len(g) == 0 for g in leftovers.values()]))

    def _test_use_external_resource(self):
        before_openstack_infra_state = openstack_infra_state(
            self.env.cloudify_config)

        self._modify_blueprint_use_external_resource()

        bp_and_dep_name = self.test_id + '-use-external-resource'
        _, after = self.upload_deploy_and_execute_install(
            blueprint_id=bp_and_dep_name, deployment_id=bp_and_dep_name)

        self._post_use_external_resource_install_assertions(
            bp_and_dep_name, before_openstack_infra_state, after['node_state'])

        self.execute_uninstall(bp_and_dep_name)

        self._post_use_external_resource_uninstall_assertions(bp_and_dep_name)

    def _modify_blueprint_use_external_resource(self):
        node_instances = self.client.node_instances().list(
            deployment_id=self.test_id)

        node_id_to_external_resource_id = {
            node_instance.node_id: node_instance.runtime_properties[
                'external_id'] for node_instance in node_instances
        }

        with YamlPatcher(self.blueprint_yaml) as patch:
            for node_id, resource_id in node_id_to_external_resource_id:
                patch.merge_obj(
                    'node_templates.{0}.properties'.format(node_id),
                    {
                        'use_external_resource': True,
                        'resource_id': resource_id
                    })

    def _post_use_external_resource_install_assertions(
            self, use_external_resource_deployment_id,
            before_openstack_infra_state, after_nodes_state):
        # verify there aren't any new resources on Openstack
        after_openstack_infra_state = openstack_infra_state(
            self.env.cloudify_config)
        delta = openstack_infra_state_delta(before_openstack_infra_state,
                                            after_openstack_infra_state)
        for delta_resources_of_single_type in delta.values():
            self.assertFalse(delta_resources_of_single_type)

        # verify the runtime properties of the new deployment's nodes
        original_deployment_node_states = self.get_node_states(
            after_nodes_state, self.test_id)
        use_external_resource_deployment_node_states = self.get_node_states(
            after_nodes_state, use_external_resource_deployment_id)
        self.assertDictEqual(original_deployment_node_states,
                             use_external_resource_deployment_node_states)

    def _post_use_external_resource_uninstall_assertions(
            self, use_external_resource_deployment_id):
        # verify the external resources are all still up and running
        original_deployment_node_states = self.get_node_states(
            self.get_manager_state(), self.test_id)
        self.post_install_assertions(original_deployment_node_states)

        # verify the use_external_resource deployment has no runtime
        # properties on the nodes
        node_instances = self.client.node_instances.list(
            deployment_id=use_external_resource_deployment_id)
        instances_with_runtime_props = [instance for instance in node_instances
                                        if instance.runtime_properties]
        self.assertEquals(0, len(instances_with_runtime_props))

    def get_node_states(self, node_states, deployment_id):
        return {
            'server': self._node_state('nova_server', node_states,
                                       deployment_id),
            'network': self._node_state('neutron_network', node_states,
                                        deployment_id),
            'subnet': self._node_state('neutron_subnet', node_states,
                                       deployment_id),
            'router': self._node_state('neutron_router', node_states,
                                       deployment_id),
            'port': self._node_state('neutron_port', node_states,
                                     deployment_id),
            'sg_src': self._node_state('security_group_src', node_states,
                                       deployment_id),
            'sg_dst': self._node_state('security_group_dst', node_states,
                                       deployment_id),
            'floatingip': self._node_state('floatingip', node_states,
                                           deployment_id)
        }

    def get_delta_node_states(self, before_state, after_state):
        delta = self.get_manager_state_delta(before_state, after_state)
        node_states = self.get_node_states(delta['node_state'], self.test_id)
        return node_states

    def get_openstack_components(self, states):
        nova, neutron = openstack_clients(self.env.cloudify_config)
        eid = 'external_id'
        sg = 'security_group'
        i = 'floatingip'
        rid = states['router'][eid]
        return {
            'server': nova.servers.get(states['server'][eid]).to_dict(),
            'network': neutron.show_network(states['network'][eid])['network'],
            'subnet': neutron.show_subnet(states['subnet'][eid])['subnet'],
            'router': neutron.show_router(rid)['router'],
            'port': neutron.show_port(states['port'][eid])['port'],
            'sg_src': neutron.show_security_group(states['sg_src'][eid])[sg],
            'sg_dst': neutron.show_security_group(states['sg_dst'][eid])[sg],
            'floatingip': neutron.show_floatingip(states[i][eid])[i],
            'router_ports': neutron.list_ports(device_id=rid)['ports']
        }

    def _node_state(self, starts_with, node_states, deployment_id):
        node_states = node_states[deployment_id].values()
        state = [state for state in node_states
                 if state['id'].startswith(starts_with)][0]
        self.assertEqual(state['state'], 'started')
        return state['runtime_properties']

    def assert_obj_list_contains_subset(self, obj_list, subset):
        for obj in obj_list:
            if all([obj.get(key) == value for key, value in subset.items()]):
                    return
        self.fail('Could not find {0} in {1}'.format(subset, obj_list))

    def assert_router_connected_to_subnet(self, router_id,
                                          router_ports, subnet_id):
        for port in router_ports:
            for fixed_ip in port.get('fixed_ips', []):
                if fixed_ip.get('subnet_id') == subnet_id:
                    return
        self.fail('router {0} is not connected to subnet {1}'
                  .format(router_id, subnet_id))
<commit_msg>Fix for neutron galore test<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

__author__ = 'dan'

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import YamlPatcher
from cosmo_tester.framework.handlers.openstack import (
    openstack_clients,
    openstack_infra_state,
    openstack_infra_state_delta
)

class NeutronGaloreTest(TestCase):

    def test_neutron_galore(self):

        blueprint_path = self.copy_blueprint('neutron-galore')
        self.blueprint_yaml = blueprint_path / 'blueprint.yaml'
        self.modify_blueprint()

        before, after = self.upload_deploy_and_execute_install()

        node_states = self.get_delta_node_states(before, after)
        self.post_install_assertions(node_states)

        self._test_use_external_resource()

        self.execute_uninstall()

        self.post_uninstall_assertions()

    def modify_blueprint(self):
        with YamlPatcher(self.blueprint_yaml) as patch:
            vm_path = 'node_templates.nova_server.properties'
            patch.merge_obj('{0}.server'.format(vm_path), {
                'name': 'novaservertest',
                'image_name': self.env.ubuntu_image_name,
                'flavor_name': self.env.flavor_name,
                'security_groups': ['neutron_test_security_group_dst'],
            })

    def post_install_assertions(self, node_states):

        def p(name):
            return '{}{}'.format(self.env.resources_prefix, name)

        openstack = self.get_openstack_components(node_states)

        port_assigned_addr = openstack['server']['addresses'][
            p('neutron_network_test')][0]['addr']
        port_security_group_id = openstack['port']['security_groups'][0]
        port_fixed_ip = openstack['port']['fixed_ips'][0]['ip_address']
        port_network_id = openstack['port']['network_id']
        port_subnet_id = openstack['port']['fixed_ips'][0]['subnet_id']
        router_network_id = openstack['router']['external_gateway_info'][
            'network_id']
        sg_src_id = openstack['sg_src']['id']
        network_subnet_id = openstack['network']['subnets'][0]

        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]
                         ['OS-EXT-IPS:type'], 'fixed')
        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][1]
                         ['OS-EXT-IPS:type'], 'floating')
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]
                         ['OS-EXT-IPS:type'], 'fixed')
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]
                         ['version'], 4)
        self.assertTrue(port_assigned_addr.startswith('10.10.10.'))
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p('neutron_test_security_group_dst')})
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p(self.env.agents_security_group)})
        self.assert_obj_list_contains_subset(
            openstack['server']['security_groups'],
            {'name': p('neutron_test_security_group_src')})
        self.assertEqual(openstack['server']['name'], p('novaservertest'))
        self.assertEqual(openstack['port']['name'], p('neutron_test_port'))
        self.assertEqual(port_fixed_ip, port_assigned_addr)

        self.assertEqual(openstack['floatingip']['floating_network_id'],
                         router_network_id)
        self.assertEqual(openstack['router']['name'], p('neutron_router_test'))
        self.assertEqual(openstack['sg_src']['name'],
                         p('neutron_test_security_group_src'))
        self.assertEqual(port_security_group_id, sg_src_id)
        self.assertEqual(openstack['network']['name'],
                         p('neutron_network_test'))
        self.assertEqual(port_network_id, openstack['network']['id'])
        self.assertEqual(openstack['subnet']['name'],
                         p('neutron_subnet_test'))
        self.assertEqual(openstack['subnet']['cidr'], '10.10.10.0/24')
        self.assertEqual(network_subnet_id, openstack['subnet']['id'])
        self.assertEqual(port_subnet_id, openstack['subnet']['id'])
        self.assertEqual(openstack['sg_dst']['name'],
                         p('neutron_test_security_group_dst'))
        self.assertEqual(4, len(openstack['sg_dst']['security_group_rules']))
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '1.2.3.0/24',
             'port_range_min': 80,
             'port_range_max': 80,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '2.3.4.0/24',
             'port_range_min': 65500,
             'port_range_max': 65510,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_group_id': sg_src_id,
             'port_range_min': 65521,
             'port_range_max': 65521,
             'direction': 'ingress'})
        self.assert_obj_list_contains_subset(
            openstack['sg_dst']['security_group_rules'],
            {'remote_ip_prefix': '3.4.5.0/24',
             'port_range_min': 443,
             'port_range_max': 443,
             'direction': 'egress'})
        self.assertEqual(node_states['floatingip']['floating_ip_address'],
                         openstack['floatingip']['floating_ip_address'])
        self.assertEqual(openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]['addr'],
                         node_states['server']['networks']
                         [p(self.env.management_network_name)][0])
        self.assertEqual(openstack['server']['addresses']
                         [p('neutron_network_test')][0]['addr'],
                         node_states['server']['networks']
                         [p('neutron_network_test')][0])
        self.assertEqual(node_states['server']['ip'],
                         openstack['server']['addresses']
                         [p(self.env.management_network_name)][0]['addr'])
        self.assert_router_connected_to_subnet(openstack['router']['id'],
                                               openstack['router_ports'],
                                               openstack['subnet']['id'])

    def post_uninstall_assertions(self):
        leftovers = self._test_cleanup_context.get_resources_to_teardown()
        self.assertTrue(all([len(g) == 0 for g in leftovers.values()]))

    def _test_use_external_resource(self):
        before_openstack_infra_state = openstack_infra_state(
            self.env.cloudify_config)

        self._modify_blueprint_use_external_resource()

        bp_and_dep_name = self.test_id + '-use-external-resource'
        _, after = self.upload_deploy_and_execute_install(
            blueprint_id=bp_and_dep_name, deployment_id=bp_and_dep_name)

        self._post_use_external_resource_install_assertions(
            bp_and_dep_name, before_openstack_infra_state, after['node_state'])

        self.execute_uninstall(bp_and_dep_name)

        self._post_use_external_resource_uninstall_assertions(bp_and_dep_name)

    def _modify_blueprint_use_external_resource(self):
        node_instances = self.client.node_instances.list(
            deployment_id=self.test_id)

        node_id_to_external_resource_id = {
            node_instance.node_id: node_instance.runtime_properties[
                'external_id'] for node_instance in node_instances
        }

        with YamlPatcher(self.blueprint_yaml) as patch:
            for node_id, resource_id in node_id_to_external_resource_id:
                patch.merge_obj(
                    'node_templates.{0}.properties'.format(node_id),
                    {
                        'use_external_resource': True,
                        'resource_id': resource_id
                    })

    def _post_use_external_resource_install_assertions(
            self, use_external_resource_deployment_id,
            before_openstack_infra_state, after_nodes_state):
        # verify there aren't any new resources on Openstack
        after_openstack_infra_state = openstack_infra_state(
            self.env.cloudify_config)
        delta = openstack_infra_state_delta(before_openstack_infra_state,
                                            after_openstack_infra_state)
        for delta_resources_of_single_type in delta.values():
            self.assertFalse(delta_resources_of_single_type)

        # verify the runtime properties of the new deployment's nodes
        original_deployment_node_states = self.get_node_states(
            after_nodes_state, self.test_id)
        use_external_resource_deployment_node_states = self.get_node_states(
            after_nodes_state, use_external_resource_deployment_id)
        self.assertDictEqual(original_deployment_node_states,
                             use_external_resource_deployment_node_states)

    def _post_use_external_resource_uninstall_assertions(
            self, use_external_resource_deployment_id):
        # verify the external resources are all still up and running
        original_deployment_node_states = self.get_node_states(
            self.get_manager_state(), self.test_id)
        self.post_install_assertions(original_deployment_node_states)

        # verify the use_external_resource deployment has no runtime
        # properties on the nodes
        node_instances = self.client.node_instances.list(
            deployment_id=use_external_resource_deployment_id)
        instances_with_runtime_props = [instance for instance in node_instances
                                        if instance.runtime_properties]
        self.assertEquals(0, len(instances_with_runtime_props))

    def get_node_states(self, node_states, deployment_id):
        return {
            'server': self._node_state('nova_server', node_states,
                                       deployment_id),
            'network': self._node_state('neutron_network', node_states,
                                        deployment_id),
            'subnet': self._node_state('neutron_subnet', node_states,
                                       deployment_id),
            'router': self._node_state('neutron_router', node_states,
                                       deployment_id),
            'port': self._node_state('neutron_port', node_states,
                                     deployment_id),
            'sg_src': self._node_state('security_group_src', node_states,
                                       deployment_id),
            'sg_dst': self._node_state('security_group_dst', node_states,
                                       deployment_id),
            'floatingip': self._node_state('floatingip', node_states,
                                           deployment_id)
        }

    def get_delta_node_states(self, before_state, after_state):
        delta = self.get_manager_state_delta(before_state, after_state)
        node_states = self.get_node_states(delta['node_state'], self.test_id)
        return node_states

    def get_openstack_components(self, states):
        nova, neutron = openstack_clients(self.env.cloudify_config)
        eid = 'external_id'
        sg = 'security_group'
        i = 'floatingip'
        rid = states['router'][eid]
        return {
            'server': nova.servers.get(states['server'][eid]).to_dict(),
            'network': neutron.show_network(states['network'][eid])['network'],
            'subnet': neutron.show_subnet(states['subnet'][eid])['subnet'],
            'router': neutron.show_router(rid)['router'],
            'port': neutron.show_port(states['port'][eid])['port'],
            'sg_src': neutron.show_security_group(states['sg_src'][eid])[sg],
            'sg_dst': neutron.show_security_group(states['sg_dst'][eid])[sg],
            'floatingip': neutron.show_floatingip(states[i][eid])[i],
            'router_ports': neutron.list_ports(device_id=rid)['ports']
        }

    def _node_state(self, starts_with, node_states, deployment_id):
        node_states = node_states[deployment_id].values()
        state = [state for state in node_states
                 if state['id'].startswith(starts_with)][0]
        self.assertEqual(state['state'], 'started')
        return state['runtime_properties']

    def assert_obj_list_contains_subset(self, obj_list, subset):
        for obj in obj_list:
            if all([obj.get(key) == value for key, value in subset.items()]):
                    return
        self.fail('Could not find {0} in {1}'.format(subset, obj_list))

    def assert_router_connected_to_subnet(self, router_id,
                                          router_ports, subnet_id):
        for port in router_ports:
            for fixed_ip in port.get('fixed_ips', []):
                if fixed_ip.get('subnet_id') == subnet_id:
                    return
        self.fail('router {0} is not connected to subnet {1}'
                  .format(router_id, subnet_id))
<|endoftext|>