<commit_before>from fontTools.varLib.models import supportScalar, normalizeValue
from fontTools.misc.fixedTools import MAX_F2DOT14

def _negate(*values):
    yield from (-1 * v for v in values)

def _revnegate(v):
    return (-v[2], -v[1], -v[0])

def _solvePinned(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    assert axisMin == axisDef == axisMax

    support = {'tag': tent}
    scalar = supportScalar({'tag': axisDef}, support)
    if scalar == 0.0:
        return []
    return [(scalar, (-1, 0, +1))]

def _solveDefaultUnmoved(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    negative = lower < 0
    if negative:
        if axisMin == -1.0:
            return [(1, tent)]
        elif axisMin == 0.0:
            return []
    else:
        if axisMax == 1.0:
            return [(1, tent)]
        elif axisMax == 0.0:
            return []

    limit = axisMin if negative else axisMax

    # Rebase axis bounds onto the new limit, which then becomes the new -1.0 or +1.0.
    # The results are always positive, because both dividend and divisor are either
    # all positive or all negative.
    newLower = lower / limit
    newPeak = peak / limit
    newUpper = upper / limit
    # for negative TupleVariation, swap lower and upper to simplify procedure
    if negative:
        newLower, newUpper = newUpper, newLower

    # special case when innermost bound == peak == limit
    if newLower == newPeak == 1.0:
        loc = (-1.0, -1.0, -1.0) if negative else (1.0, 1.0, 1.0)
        return [(1, loc)]

    # case 1: the whole deltaset falls outside the new limit; we can drop it
    elif newLower >= 1.0:
        return []

    # case 2: only the peak and outermost bound fall outside the new limit;
    # we keep the deltaset, update peak and outermost bound and and scale deltas
    # by the scalar value for the restricted axis at the new limit.
    elif newPeak >= 1.0:
        scalar = supportScalar({'tag': limit}, {'tag': (lower, peak, upper)})
        newPeak = 1.0
        newUpper = 1.0
        if negative:
            newLower, newPeak, newUpper = _negate(newUpper, newPeak, newLower)
        loc = (newLower, newPeak, newUpper)
        return [(scalar, loc)]

    # case 3: peak falls inside but outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    elif newUpper <= 2.0:
        if negative:
            newLower, newPeak, newUpper = _negate(newUpper, newPeak, newLower)
        elif MAX_F2DOT14 < newUpper <= 2.0:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            newUpper = MAX_F2DOT14
        loc = (newLower, newPeak, newUpper)
        return [(1, loc)]

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:
        # duplicate the tent, then adjust lower/peak/upper so that the outermost limit
        # of the original tent is +/-2.0, whereas the new tent's starts as the old
        # one peaks and maxes out at +/-1.0.
        if negative:
            loc = (-2.0, -1 * newPeak, -1 * newLower)
            newloc = (-1.0, -1.0, -1 * newPeak)
        else:
            loc = (newLower, newPeak, MAX_F2DOT14)
            newloc = (newPeak, 1.0, 1.0)
        # the new tent's deltas are scaled by the difference between the scalar value
        # for the old tent at the desired limit...
        scalar1 = supportScalar({tag: limit}, {tag: (lower, peak, upper)})
        # ... and the scalar value for the clamped tent (with outer limit +/-2.0),
        # which can be simplified like this:
        scalar2 = 1 / (2 - newPeak)

        return [(scalar1, loc), (scalar2, newloc)]

def _solveWithoutGain(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # axisMin <= axisDef <= lower < peak <= axisMax

    # case 3: outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    if axisDef + (axisMax - axisDef) * 2 >= upper:

        if axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            upper = axisDef + (axisMax - axisDef) * MAX_F2DOT14

        return [(1, (lower, peak, upper))]

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:

        loc1 = (lower, peak, axisMax)
        scalar1 = 1

        loc2 = (peak, axisMax, axisMax)
        scalar2 = supportScalar({tag: axisMax}, {tag: tent})

        return [(scalar1, loc1), (scalar2, loc2)]

def _solveWithGain(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # lower <= axisDef <= peak <= axisMax

    gain = supportScalar({tag: axisDef}, {tag: tent})
    out = [(gain, axisLimit)]

    # First, the positive side

    # case 3: outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    if axisDef + (axisMax - axisDef) * 2 >= upper:

        if axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            upper = axisDef + (axisMax - axisDef) * MAX_F2DOT14

        out.append((1 - gain, (axisDef, peak, upper)))

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:

        loc1 = (axisDef, peak, axisMax)
        scalar1 = 1

        loc2 = (peak, axisMax, axisMax)
        scalar2 = supportScalar({tag: axisMax}, {tag: tent})

        out.append((scalar1 - gain, loc1))
        out.append((scalar2 - gain, loc2))

    # Now, the negative side

    # case 1neg: lower extends beyond axisMin: we chop.
    if lower <= axisMin:
        loc = (axisMin, axisMin, axisDef)
        scalar = supportScalar({tag: axisMin}, {tag: tent})

        out.append((scalar - gain, loc))

    # case 2neg: lower is betwen axisMin and axisDef: we add two deltasets to
    # keep it "up" all the way to end.
    else:
        loc1 = (axisMin, lower, axisDef)
        scalar1 = 0

        loc2 = (axisMin, axisMin, lower)
        scalar2 = 0

        out.append((scalar1 - gain, loc1))
        out.append((scalar2 - gain, loc2))

    return out

def _solveGeneral(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # Mirror the problem such that axisDef is always <= peak
    if axisDef > peak:
        return [(scalar, _revnegate(t))
                for scalar,t
                in _solveGeneral(_revnegate(tent),
                                 _revnegate(axisLimit))]
    # axisDef <= peak

    # case 1: the whole deltaset falls outside the new limit; we can drop it
    if axisMax <= lower and axisMax < peak:
        return [] # No overlap

    # case 2: only the peak and outermost bound fall outside the new limit;
    # we keep the deltaset, update peak and outermost bound and and scale deltas
    # by the scalar value for the restricted axis at the new limit.
    if axisMax < peak:
        mult = supportScalar({'tag': axisMax}, {'tag': tent})
        tent = (lower, axisMax, axisMax)
        return [(scalar*mult, t) for scalar,t in _solveGeneral(tent, axisLimit)]

    # axisDef <= peak <= axisMax

    if axisDef <= lower and axisDef < peak:
        # No gain to carry
        return _solveWithoutGain(tent, axisLimit)
    else:
        return _solveWithGain(tent, axisLimit)

    raise NotImplementedError

def rebaseTent(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    assert -1 <= axisMin <= axisDef <= axisMax <= +1

    lower, peak, upper = tent
    assert -2 <= lower <= peak <= upper <= +2

    assert peak != 0

    # Get the pinned case out of the way
    if axisMin == axisMax:
        return _solvePinned(tent, axisLimit)

    # If default isn't moving, get that out of the way as well
    if False and axisDef == 0:
        return _solveDefaultUnmoved(tent, axisLimit)

    sols = _solveGeneral(tent, axisLimit)
    n = lambda v: normalizeValue(v, axisLimit, extrapolate=True)
    sols = [(scalar, (n(v[0]), n(v[1]), n(v[2]))) for scalar,v in sols if scalar != 0]
    return sols
<commit_msg>[instancer.solver] Fix up, also route pinned through general<commit_after>from fontTools.varLib.models import supportScalar, normalizeValue
from fontTools.misc.fixedTools import MAX_F2DOT14

def _negate(*values):
    yield from (-1 * v for v in values)

def _revnegate(v):
    return (-v[2], -v[1], -v[0])

def _solvePinned(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    assert axisMin == axisDef == axisMax

    support = {'tag': tent}
    scalar = supportScalar({'tag': axisDef}, support)
    if scalar == 0.0:
        return []
    return [(scalar, (-1, 0, +1))]

def _solveDefaultUnmoved(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    negative = lower < 0
    if negative:
        if axisMin == -1.0:
            return [(1, tent)]
        elif axisMin == 0.0:
            return []
    else:
        if axisMax == 1.0:
            return [(1, tent)]
        elif axisMax == 0.0:
            return []

    limit = axisMin if negative else axisMax

    # Rebase axis bounds onto the new limit, which then becomes the new -1.0 or +1.0.
    # The results are always positive, because both dividend and divisor are either
    # all positive or all negative.
    newLower = lower / limit
    newPeak = peak / limit
    newUpper = upper / limit
    # for negative TupleVariation, swap lower and upper to simplify procedure
    if negative:
        newLower, newUpper = newUpper, newLower

    # special case when innermost bound == peak == limit
    if newLower == newPeak == 1.0:
        loc = (-1.0, -1.0, -1.0) if negative else (1.0, 1.0, 1.0)
        return [(1, loc)]

    # case 1: the whole deltaset falls outside the new limit; we can drop it
    elif newLower >= 1.0:
        return []

    # case 2: only the peak and outermost bound fall outside the new limit;
    # we keep the deltaset, update peak and outermost bound and and scale deltas
    # by the scalar value for the restricted axis at the new limit.
    elif newPeak >= 1.0:
        scalar = supportScalar({'tag': limit}, {'tag': (lower, peak, upper)})
        newPeak = 1.0
        newUpper = 1.0
        if negative:
            newLower, newPeak, newUpper = _negate(newUpper, newPeak, newLower)
        loc = (newLower, newPeak, newUpper)
        return [(scalar, loc)]

    # case 3: peak falls inside but outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    elif newUpper <= 2.0:
        if negative:
            newLower, newPeak, newUpper = _negate(newUpper, newPeak, newLower)
        elif MAX_F2DOT14 < newUpper <= 2.0:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            newUpper = MAX_F2DOT14
        loc = (newLower, newPeak, newUpper)
        return [(1, loc)]

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:
        # duplicate the tent, then adjust lower/peak/upper so that the outermost limit
        # of the original tent is +/-2.0, whereas the new tent's starts as the old
        # one peaks and maxes out at +/-1.0.
        if negative:
            loc = (-2.0, -1 * newPeak, -1 * newLower)
            newloc = (-1.0, -1.0, -1 * newPeak)
        else:
            loc = (newLower, newPeak, MAX_F2DOT14)
            newloc = (newPeak, 1.0, 1.0)
        # the new tent's deltas are scaled by the difference between the scalar value
        # for the old tent at the desired limit...
        scalar1 = supportScalar({tag: limit}, {tag: (lower, peak, upper)})
        # ... and the scalar value for the clamped tent (with outer limit +/-2.0),
        # which can be simplified like this:
        scalar2 = 1 / (2 - newPeak)

        return [(scalar1, loc), (scalar2, newloc)]

def _solveWithoutGain(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # axisMin <= axisDef <= lower < peak <= axisMax

    # case 3: outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    if axisDef + (axisMax - axisDef) * 2 >= upper:

        if axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            upper = axisDef + (axisMax - axisDef) * MAX_F2DOT14

        return [(1, (lower, peak, upper))]

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:

        loc1 = (lower, peak, axisMax)
        scalar1 = 1

        loc2 = (peak, axisMax, axisMax)
        scalar2 = supportScalar({tag: axisMax}, {tag: tent})

        return [(scalar1, loc1), (scalar2, loc2)]

def _solveWithGain(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # lower <= axisDef <= peak <= axisMax

    gain = supportScalar({'tag': axisDef}, {'tag': tent})
    out = [(gain, axisLimit)]

    # First, the positive side

    # case 3: outermost limit still fits within F2Dot14 bounds;
    # we keep deltas as is and only scale the axes bounds. Deltas beyond -1.0
    # or +1.0 will never be applied as implementations must clamp to that range.
    if axisDef + (axisMax - axisDef) * 2 >= upper:

        if axisDef + (axisMax - axisDef) * MAX_F2DOT14 < upper:
            # we clamp +2.0 to the max F2Dot14 (~1.99994) for convenience
            upper = axisDef + (axisMax - axisDef) * MAX_F2DOT14

        out.append((1 - gain, (axisDef, peak, upper)))

    # case 4: new limit doesn't fit; we need to chop the deltaset into two 'tents',
    # because the shape of a triangle with part of one side cut off cannot be
    # represented as a triangle itself. It can be represented as sum of two triangles.
    # NOTE: This increases the file size!
    else:

        loc1 = (axisDef, peak, axisMax)
        scalar1 = 1

        loc2 = (peak, axisMax, axisMax)
        scalar2 = supportScalar({'tag': axisMax}, {'tag': tent})

        out.append((scalar1 - gain, loc1))
        out.append((scalar2 - gain, loc2))

    # Now, the negative side

    # case 1neg: lower extends beyond axisMin: we chop.
    if lower <= axisMin:
        loc = (axisMin, axisMin, axisDef)
        scalar = supportScalar({'tag': axisMin}, {'tag': tent})

        out.append((scalar - gain, loc))

    # case 2neg: lower is betwen axisMin and axisDef: we add two deltasets to
    # keep it "up" all the way to end.
    else:
        loc1 = (axisMin, lower, axisDef)
        scalar1 = 0

        loc2 = (axisMin, axisMin, lower)
        scalar2 = 0

        out.append((scalar1 - gain, loc1))
        out.append((scalar2 - gain, loc2))

    return out

def _solveGeneral(tent, axisLimit):
    axisMin, axisDef, axisMax = axisLimit
    lower, peak, upper = tent

    # Mirror the problem such that axisDef is always <= peak
    if axisDef > peak:
        return [(scalar, _revnegate(t))
                for scalar,t
                in _solveGeneral(_revnegate(tent),
                                 _revnegate(axisLimit))]
    # axisDef <= peak

    # case 1: the whole deltaset falls outside the new limit; we can drop it
    if axisMax <= lower and axisMax < peak:
        return [] # No overlap

    # case 2: only the peak and outermost bound fall outside the new limit;
    # we keep the deltaset, update peak and outermost bound and and scale deltas
    # by the scalar value for the restricted axis at the new limit.
    if axisMax < peak:
        mult = supportScalar({'tag': axisMax}, {'tag': tent})
        tent = (lower, axisMax, axisMax)
        return [(scalar*mult, t) for scalar,t in _solveGeneral(tent, axisLimit)]

    # axisDef <= peak <= axisMax

    if axisDef <= lower and axisDef < peak:
        # No gain to carry
        return _solveWithoutGain(tent, axisLimit)
    else:
        return _solveWithGain(tent, axisLimit)

    raise NotImplementedError

def rebaseTent(tent, axisLimit):

    axisMin, axisDef, axisMax = axisLimit
    assert -1 <= axisMin <= axisDef <= axisMax <= +1

    lower, peak, upper = tent
    assert -2 <= lower <= peak <= upper <= +2

    assert peak != 0

    # Get the pinned case out of the way
    if False and axisMin == axisMax:
        return _solvePinned(tent, axisLimit)

    # If default isn't moving, get that out of the way as well
    if False and axisDef == 0:
        return _solveDefaultUnmoved(tent, axisLimit)

    sols = _solveGeneral(tent, axisLimit)
    n = lambda v: normalizeValue(v, axisLimit, extrapolate=True)
    sols = [(scalar, (n(v[0]), n(v[1]), n(v[2]))) for scalar,v in sols if scalar != 0]
    return sols
<|endoftext|>


<commit_before>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Load up an adjacency matrix given G_fn, lcc & roiRoot

import mrpaths
import argparse
import mrcap.lcc as lcc
from getBaseName import getBaseName
import os
import sys

def loadAdjMat(G_fn, lcc_fn, roiRootName = None):
  '''
  Load adjacency matrix given lcc_fn & G_fn. lcc has z-indicies corresponding to the lcc :
  G_fn - the .mat file holding graph
  lcc_fn - the largest connected component .npy z-ordering
  '''
  print "Loading adjacency matrix..."
  if not roiRootName:
    roiRootName =  os.path.join(getRoiRoot(G_fn),getBaseName(G_fn))
    #import pdb; pdb.set_trace()
  vcc = lcc.ConnectedComponent(fn = lcc_fn)
  try:
    fg = lcc._load_fibergraph(roiRootName , G_fn)
    G = vcc.induced_subgraph(fg.spcscmat)
    G = G+G.T # Symmetrize
  except:
    if not os.path.exists(roiRootName):
      print "%s Doesn't exist" % roiRootName

    if not os.path.exists(lcc_fn):
      print "%s Doesn't exist" % lcc_fn

    if not os.path.exists(G_fn):
      print "%s Doesn't exist" % G_fn

    if os.path.exist(roiRootName) and os.path.exist(lcc_fn) and os.path.exist(G_fn):
      print "****Some wild Problem loading real lcc & graph****"
    sys.exit(-1)

  return G

def getRoiRoot(G_fn):
  '''
  Get the roi root name form G_fn
  G_fn - full filename of graph (e.g of format /{User}/{disa}/{graphs}/filename_fiber.mat)
  * {} - Not necessary
  '''
  roiRoot = ''
  for i in G_fn.split('/')[1:-3]:
    roiRoot = os.path.join(roiRoot, i)
  roiRoot = os.path.join(roiRoot, "base", "roi")
  return roiRoot

def main():

    parser = argparse.ArgumentParser(description='Calculate Max Avg Degree estimate as max eigenvalue for biggraphs')
    parser.add_argument('G_fn', action='store',help='Full filename sparse graph (.mat)')
    parser.add_argument('lcc_fn', action='store',help='Full filename of largest connected component (.npy)')
    parser.add_argument('roiRootName', action='store',help='Full path of roi director + baseName')

    result = parser.parse_args()
    getMaxAveDegree(result.G_fn, result.lcc_fn, result.roiRootName)

if __name__ == '__main__':
  main()

<commit_msg>Minor<commit_after>#!/usr/bin/python

# Author: Disa Mhembere, Johns Hopkins University
# Separated: 10/2/2012
# Load up an adjacency matrix given G_fn, lcc & roiRoot

import mrpaths
import argparse
import mrcap.lcc as lcc
from getBaseName import getBaseName
import os
import sys

def loadAdjMat(G_fn, lcc_fn, roiRootName = None):
  '''
  Load adjacency matrix given lcc_fn & G_fn. lcc has z-indicies corresponding to the lcc :
  G_fn - the .mat file holding graph
  lcc_fn - the largest connected component .npy z-ordering
  '''
  print "Loading adjacency matrix..."
  if not roiRootName:
    roiRootName =  os.path.join(getRoiRoot(G_fn),getBaseName(G_fn))
    #import pdb; pdb.set_trace()
  vcc = lcc.ConnectedComponent(fn = lcc_fn)
  try:
    fg = lcc._load_fibergraph(roiRootName , G_fn)
    G = vcc.induced_subgraph(fg.spcscmat)
    G = G+G.T # Symmetrize
  except:
    if not os.path.exists(roiRootName):
      print "%s Doesn't exist" % roiRootName

    if not os.path.exists(lcc_fn):
      print "%s Doesn't exist" % lcc_fn

    if not os.path.exists(G_fn):
      print "%s Doesn't exist" % G_fn

    if os.path.exists(roiRootName) and os.path.exists(lcc_fn) and os.path.exists(G_fn):
      print "****Some wild Problem loading real lcc & graph****"
    sys.exit(-1)

  return G

def getRoiRoot(G_fn):
  '''
  Get the roi root name form G_fn
  G_fn - full filename of graph (e.g of format /{User}/{disa}/{graphs}/filename_fiber.mat)
  * {} - Not necessary
  '''
  roiRoot = ''
  for i in G_fn.split('/')[1:-3]:
    roiRoot = os.path.join(roiRoot, i)
  roiRoot = os.path.join(roiRoot, "base", "roi")
  return roiRoot

def main():

    parser = argparse.ArgumentParser(description='Calculate Max Avg Degree estimate as max eigenvalue for biggraphs')
    parser.add_argument('G_fn', action='store',help='Full filename sparse graph (.mat)')
    parser.add_argument('lcc_fn', action='store',help='Full filename of largest connected component (.npy)')
    parser.add_argument('roiRootName', action='store',help='Full path of roi director + baseName')

    result = parser.parse_args()
    getMaxAveDegree(result.G_fn, result.lcc_fn, result.roiRootName)

if __name__ == '__main__':
  main()

<|endoftext|>


<commit_before>import numpy as np

class ToyXValidationData():
    """
    Class that generates random data for XValidation (list of matrices/lists)
    to test the pipeline

    @author Heiko
    """
    @staticmethod
    def get():
        X_list = []
        y_list = []
        dim = 100
        for _ in range(10):
            n = np.random.randint(100, 1000)
            X_list += [np.random.randn(n, dim)]
            y_list += [(np.random.rand(n)>.5).astype(np.int64)]

        return X_list, y_list
<commit_msg>copy paste error<commit_after>import numpy as np

class RandomXValidationData():
    """
    Class that generates random data for XValidation (list of matrices/lists)
    to test the pipeline

    @author Heiko
    """
    @staticmethod
    def get():
        X_list = []
        y_list = []
        dim = 100
        for _ in range(10):
            n = np.random.randint(100, 1000)
            X_list += [np.random.randn(n, dim)]
            y_list += [(np.random.rand(n)>.5).astype(np.int64)]

        return X_list, y_list
<|endoftext|>


<commit_before>import pytest

from course_modes.models import CourseMode
from custom_settings.models import CustomSettings
from custom_settings.signals.handlers import initialize_course_settings
from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
from openedx.features.philu_utils.py_test import does_not_raise

@pytest.mark.django_db
@pytest.mark.parametrize('created', [False, True])
def test_get_course_open_date_from_settings(created):
    """Assert that CustomSettings and CourseMode objects are created when handler is called"""
    course_overview = CourseOverviewFactory(display_name='test_overview')

    with does_not_raise():
        initialize_course_settings(None, course_overview, created)

        if created:
            assert CustomSettings.objects.filter(id=course_overview.id).count() == 1
            assert CourseMode.objects.filter(course_id=course_overview.id, mode_slug='honor',
                                             mode_display_name='test_overview').count() == 1
<commit_msg>LP-2180 Fixing test method name<commit_after>import pytest

from course_modes.models import CourseMode
from custom_settings.models import CustomSettings
from custom_settings.signals.handlers import initialize_course_settings
from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
from openedx.features.philu_utils.py_test import does_not_raise

@pytest.mark.django_db
@pytest.mark.parametrize('created', [False, True])
def test_initialize_course_settings(created):
    """Assert that CustomSettings and CourseMode objects are created when handler is called"""
    course_overview = CourseOverviewFactory(display_name='test_overview')

    with does_not_raise():
        initialize_course_settings(None, course_overview, created)

        if created:
            assert CustomSettings.objects.filter(id=course_overview.id).count() == 1
            assert CourseMode.objects.filter(course_id=course_overview.id, mode_slug='honor',
                                             mode_display_name='test_overview').count() == 1
<|endoftext|>


<commit_before>import json
from mock import Mock
import unittest

from xmodule.self_assessment_module import SelfAssessmentModule
from xmodule.modulestore import Location
from lxml import etree

from . import test_system

class SelfAssessmentTest(unittest.TestCase):

    definition = {'rubric': 'A rubric',
                  'prompt': 'Who?',
                  'submitmessage': 'Shall we submit now?',
                  'hintprompt': 'Consider this...',
                  }

    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = {'attempts': '10'}

    descriptor = Mock()

    def test_import(self):
        state = json.dumps({'student_answers': ["Answer 1", "answer 2", "answer 3"],
                            'scores': [0, 1],
                            'hints': ['o hai'],
                            'state': SelfAssessmentModule.INITIAL,
                            'attempts': 2})

        rubric = '''<rubric><rubric>
            <category>
			<description>Response Quality</description>
			<option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
		    </category>
             </rubric></rubric>'''

        prompt = etree.XML("<prompt>Text</prompt>")
        static_data = {
                'max_attempts': 10,
                'rubric': etree.XML(rubric),
                'prompt': prompt,
                'max_score': 1
                }

        module = SelfAssessmentModule(test_system, self.location,
                                      self.definition, self.descriptor,
                                      static_data, state, metadata=self.metadata)

        self.assertEqual(module.get_score()['score'], 0)

        module.save_answer({'student_answer': "I am an answer"}, test_system)
        self.assertEqual(module.state, module.ASSESSING)

        module.save_assessment({'assessment': '0'}, test_system)
        self.assertEqual(module.state, module.POST_ASSESSMENT)
        module.save_hint({'hint': 'this is a hint'}, test_system)
        self.assertEqual(module.state, module.DONE)

        d = module.reset({})
        self.assertTrue(d['success'])
        self.assertEqual(module.state, module.INITIAL)

        # if we now assess as right, skip the REQUEST_HINT state
        module.save_answer({'student_answer': 'answer 4'}, test_system)
        module.save_assessment({'assessment': '1'}, test_system)
        self.assertEqual(module.state, module.DONE)
<commit_msg>Update tests for new behavior<commit_after>import json
from mock import Mock
import unittest

from xmodule.self_assessment_module import SelfAssessmentModule
from xmodule.modulestore import Location
from lxml import etree

from . import test_system

class SelfAssessmentTest(unittest.TestCase):

    definition = {'rubric': 'A rubric',
                  'prompt': 'Who?',
                  'submitmessage': 'Shall we submit now?',
                  'hintprompt': 'Consider this...',
                  }

    location = Location(["i4x", "edX", "sa_test", "selfassessment",
                         "SampleQuestion"])

    metadata = {'attempts': '10'}

    descriptor = Mock()

    def test_import(self):
        state = json.dumps({'student_answers': ["Answer 1", "answer 2", "answer 3"],
                            'scores': [0, 1],
                            'hints': ['o hai'],
                            'state': SelfAssessmentModule.INITIAL,
                            'attempts': 2})

        rubric = '''<rubric><rubric>
            <category>
			<description>Response Quality</description>
			<option>The response is not a satisfactory answer to the question.  It either fails to address the question or does so in a limited way, with no evidence of higher-order thinking.</option>
		    </category>
             </rubric></rubric>'''

        prompt = etree.XML("<prompt>Text</prompt>")
        static_data = {
                'max_attempts': 10,
                'rubric': etree.XML(rubric),
                'prompt': prompt,
                'max_score': 1,
                'display_name': "Name"
                }

        module = SelfAssessmentModule(test_system, self.location,
                                      self.definition, self.descriptor,
                                      static_data, state, metadata=self.metadata)

        self.assertEqual(module.get_score()['score'], 0)

        module.save_answer({'student_answer': "I am an answer"}, test_system)
        self.assertEqual(module.state, module.ASSESSING)

        module.save_assessment({'assessment': '0'}, test_system)
        self.assertEqual(module.state, module.DONE)

        d = module.reset({})
        self.assertTrue(d['success'])
        self.assertEqual(module.state, module.INITIAL)

        # if we now assess as right, skip the REQUEST_HINT state
        module.save_answer({'student_answer': 'answer 4'}, test_system)
        module.save_assessment({'assessment': '1'}, test_system)
        self.assertEqual(module.state, module.DONE)
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000139"
    addresses_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019kest.tsv"
    stations_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019kest.tsv"
    elections = ["local.2019-05-02"]
    csv_delimiter = "\t"
    csv_encoding = "latin-1"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10006530403",  # NG348HB -> NG348HH : York House Officers Mess, R A F College, Cranwell, Sleaford
            "10006510362",  # LN41DJ -> LN41JD : Brinkle Springs, Acre Dyke Lane, Heighington, Lincoln
            "10006510378",  # LN41JG -> LN41JQ : Timperley, Bardney Road, Branston Booths, Lincoln
            "10006526117",  # LN41JH -> LN41AE : Ronsway, Fen Road, Heighington, Lincoln
            "10006505994",  # LN43RA -> LN43RR : North Moor Farm, Linwood Road, Martin, Lincoln
            "10006507278",  # NG340BU -> NG340BN : Hill Top Farmhouse, Burton Gorse, Sleaford, Lincs
            "10006515656",  # LN50DW -> LN50BN : Devonshire Grange, The Heath, Wellingore, Lincoln
            "10006516242",  # LN41JA -> LN41HZ : Claremont, Moor Lane, Branston, Lincoln
            "10006522464",  # LN44BB -> LN44BL : 1 the Hurn, Parsons Drove, Billinghay, Lincoln
            "10006527563",  # LN106XJ -> LN106XL : New Road Farm, Blankney Dales, Woodhall Spa, Lincs
            "10006527907",  # PE203QF -> PE203PZ : Rakes Farm, Swineshead Bridge, Boston
            "10006529717",  # LN50SS -> LN50SR : 1 the Cottages Carlton Lowfield Farm, Brant Broughton, Carlton Le Moorland, Lincoln
            "10006529720",  # LN50SS -> LN50SR : 2 the Cottages Carlton Lowfield Farm, Brant Broughton, Carlton Le Moorland, Lincoln
            "10006529962",  # LN44BW -> LN44BP : Lowe`s Dwelling, Tattershall Road, Billinghay, Lincoln
            "10006530120",  # PE203QF -> PE203QA : Mobile Home, Maize Farm, East Heckington, Boston, Lincs
            "10006534005",  # LN44JL -> LN44JW : Chapel Bungalow Tattershall Bridge Road, Tattershall Bridge, Lincoln
            "10006536127",  # NG340DA -> NG340BN : Fir Nature, White Cross Lane, Burton Pedwardine, Sleaford
            "10006539008",  # LN50DP -> LN50DW : Walnut Tree Barn, Pottergate Road, Wellingore, Lincoln
            "10006527449",  # LN65UU -> LN68HB : 34A Lincoln Road, Skellingthorpe, Lincoln
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001146077",  # NG348HB -> NG348EL : Daedalus Officers Mess, R A F College, Cranwell, Sleaford
            "10006506295",  # LN43LT -> LN43JF : 1 Church End, Rowston, Lincoln
        ]:
            rec["accept_suggestion"] = False

        return rec

    def station_record_to_dict(self, record):

        if record.polling_place_id == "5063":
            record = record._replace(polling_place_easting="502596")
            record = record._replace(polling_place_northing="370730")

        """
        File supplied contained obviously inaccurate point
        remove it and fall back to geocoding
        """
        if record.polling_place_id == "5181":
            record = record._replace(polling_place_easting="0")
            record = record._replace(polling_place_northing="0")

        return super().station_record_to_dict(record)
<commit_msg>postcode/point correction for Sleaford Leisure Centre<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id = "E07000139"
    addresses_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019kest.tsv"
    stations_name = "local.2019-05-02/Version 1/Democracy_Club__02May2019kest.tsv"
    elections = ["local.2019-05-02"]
    csv_delimiter = "\t"
    csv_encoding = "latin-1"

    def address_record_to_dict(self, record):
        rec = super().address_record_to_dict(record)
        uprn = record.property_urn.strip().lstrip("0")

        if uprn in [
            "10006530403",  # NG348HB -> NG348HH : York House Officers Mess, R A F College, Cranwell, Sleaford
            "10006510362",  # LN41DJ -> LN41JD : Brinkle Springs, Acre Dyke Lane, Heighington, Lincoln
            "10006510378",  # LN41JG -> LN41JQ : Timperley, Bardney Road, Branston Booths, Lincoln
            "10006526117",  # LN41JH -> LN41AE : Ronsway, Fen Road, Heighington, Lincoln
            "10006505994",  # LN43RA -> LN43RR : North Moor Farm, Linwood Road, Martin, Lincoln
            "10006507278",  # NG340BU -> NG340BN : Hill Top Farmhouse, Burton Gorse, Sleaford, Lincs
            "10006515656",  # LN50DW -> LN50BN : Devonshire Grange, The Heath, Wellingore, Lincoln
            "10006516242",  # LN41JA -> LN41HZ : Claremont, Moor Lane, Branston, Lincoln
            "10006522464",  # LN44BB -> LN44BL : 1 the Hurn, Parsons Drove, Billinghay, Lincoln
            "10006527563",  # LN106XJ -> LN106XL : New Road Farm, Blankney Dales, Woodhall Spa, Lincs
            "10006527907",  # PE203QF -> PE203PZ : Rakes Farm, Swineshead Bridge, Boston
            "10006529717",  # LN50SS -> LN50SR : 1 the Cottages Carlton Lowfield Farm, Brant Broughton, Carlton Le Moorland, Lincoln
            "10006529720",  # LN50SS -> LN50SR : 2 the Cottages Carlton Lowfield Farm, Brant Broughton, Carlton Le Moorland, Lincoln
            "10006529962",  # LN44BW -> LN44BP : Lowe`s Dwelling, Tattershall Road, Billinghay, Lincoln
            "10006530120",  # PE203QF -> PE203QA : Mobile Home, Maize Farm, East Heckington, Boston, Lincs
            "10006534005",  # LN44JL -> LN44JW : Chapel Bungalow Tattershall Bridge Road, Tattershall Bridge, Lincoln
            "10006536127",  # NG340DA -> NG340BN : Fir Nature, White Cross Lane, Burton Pedwardine, Sleaford
            "10006539008",  # LN50DP -> LN50DW : Walnut Tree Barn, Pottergate Road, Wellingore, Lincoln
            "10006527449",  # LN65UU -> LN68HB : 34A Lincoln Road, Skellingthorpe, Lincoln
        ]:
            rec["accept_suggestion"] = True

        if uprn in [
            "200001146077",  # NG348HB -> NG348EL : Daedalus Officers Mess, R A F College, Cranwell, Sleaford
            "10006506295",  # LN43LT -> LN43JF : 1 Church End, Rowston, Lincoln
        ]:
            rec["accept_suggestion"] = False

        return rec

    def station_record_to_dict(self, record):

        if record.polling_place_id == "5063":
            record = record._replace(polling_place_easting="502596")
            record = record._replace(polling_place_northing="370730")

        if record.polling_place_id == "5181":
            record = record._replace(polling_place_postcode="NG34 7HH")
            record = record._replace(polling_place_easting="507252")
            record = record._replace(polling_place_northing="345802")

        return super().station_record_to_dict(record)
<|endoftext|>


<commit_before># Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache License.

import zipfile, time

from azurelinuxagent.common.protocol.restapi import ExtHandler, ExtHandlerProperties, ExtHandlerPackage, ExtHandlerVersionUri
from azurelinuxagent.common.protocol.wire import WireProtocol
from azurelinuxagent.ga.exthandlers import ExtHandlerInstance, NUMBER_OF_DOWNLOAD_RETRIES
from azurelinuxagent.common.exception import ExtensionDownloadError, ExtensionErrorCodes
from tests.tools import *

class DownloadExtensionTestCase(AgentTestCase):
    """
    Test cases for launch_command
    """
    @classmethod
    def setUpClass(cls):
        AgentTestCase.setUpClass()
        cls.mock_cgroups = patch("azurelinuxagent.ga.exthandlers.CGroupConfigurator")
        cls.mock_cgroups.start()

    @classmethod
    def tearDownClass(cls):
        cls.mock_cgroups.stop()

        AgentTestCase.tearDownClass()

    def setUp(self):
        AgentTestCase.setUp(self)

        ext_handler_properties = ExtHandlerProperties()
        ext_handler_properties.version = "1.0.0"
        ext_handler = ExtHandler(name='Microsoft.CPlat.Core.RunCommandLinux')
        ext_handler.properties = ext_handler_properties

        protocol = WireProtocol("http://Microsoft.CPlat.Core.RunCommandLinux/foo-bar")

        self.pkg = ExtHandlerPackage()
        self.pkg.uris = [ ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri() ]
        self.pkg.uris[0].uri = 'https://zrdfepirv2cy4prdstr00a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[1].uri = 'https://zrdfepirv2cy4prdstr01a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[2].uri = 'https://zrdfepirv2cy4prdstr02a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[3].uri = 'https://zrdfepirv2cy4prdstr03a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[4].uri = 'https://zrdfepirv2cy4prdstr04a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'

        self.ext_handler_instance = ExtHandlerInstance(ext_handler=ext_handler, protocol=protocol)
        self.ext_handler_instance.pkg = self.pkg

        self.extension_dir = os.path.join(self.tmp_dir, "Microsoft.CPlat.Core.RunCommandLinux-1.0.0")
        self.mock_get_base_dir = patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", return_value=self.extension_dir)
        self.mock_get_base_dir.start()

        self.mock_get_log_dir = patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", return_value=self.tmp_dir)
        self.mock_get_log_dir.start()

        self.agent_dir = self.tmp_dir
        self.mock_get_lib_dir = patch("azurelinuxagent.ga.exthandlers.conf.get_lib_dir", return_value=self.agent_dir)
        self.mock_get_lib_dir.start()

    def tearDown(self):
        self.mock_get_lib_dir.stop()
        self.mock_get_log_dir.stop()
        self.mock_get_base_dir.stop()

        AgentTestCase.tearDown(self)

    _extension_command = "RunCommandLinux.sh"

    @staticmethod
    def _create_zip_file(filename):
        file = None
        try:
            file = zipfile.ZipFile(filename, "w")
            info = zipfile.ZipInfo(DownloadExtensionTestCase._extension_command)
            info.date_time = time.localtime(time.time())[:6]
            info.compress_type = zipfile.ZIP_DEFLATED
            file.writestr(info, "#!/bin/sh\necho 'RunCommandLinux executed successfully'\n")
        finally:
            if file is not None:
                file.close()

    @staticmethod
    def _create_invalid_zip_file(filename):
        with open(filename, "w") as file:
            file.write("An invalid ZIP file\n")

    def _get_extension_package_file(self):
        return os.path.join(self.agent_dir, os.path.basename(self.pkg.uris[0].uri) + ".zip")

    def _get_extension_command_file(self):
        return os.path.join(self.extension_dir, DownloadExtensionTestCase._extension_command)

    def _assert_download_and_expand_succeeded(self):
        self.assertTrue(os.path.exists(self._get_extension_package_file()), "The extension package was not downloaded to the expected location")
        self.assertTrue(os.path.exists(self._get_extension_command_file()), "The extension package was not expanded to the expected location")

    def test_it_should_download_and_expand_extension_package(self):
        def download_ext_handler_pkg(_uri, destination):
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.report_event") as mock_report_event:
                self.ext_handler_instance.download()

        # first download attempt should succeed
        mock_download_ext_handler_pkg.assert_called_once()
        mock_report_event.assert_called_once()

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_existing_extension_package_when_already_downloaded(self):
        DownloadExtensionTestCase._create_zip_file(self._get_extension_package_file())

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg") as mock_download_ext_handler_pkg:
            with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.report_event") as mock_report_event:
                self.ext_handler_instance.download()

        mock_download_ext_handler_pkg.assert_not_called()
        mock_report_event.assert_not_called()

        self.assertTrue(os.path.exists(self._get_extension_command_file()), "The extension package was not expanded to the expected location")

    def test_it_should_ignore_existing_extension_package_when_it_is_invalid(self):
        def download_ext_handler_pkg(_uri, destination):
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        DownloadExtensionTestCase._create_invalid_zip_file(self._get_extension_package_file())

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        mock_download_ext_handler_pkg.assert_called_once()

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_download_fails(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                return False
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_download_raises_an_exception(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                raise Exception("Download failed")
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_it_downloads_an_invalid_package(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                DownloadExtensionTestCase._create_invalid_zip_file(destination)
            else:
                DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_raise_an_exception_when_all_downloads_fail(self):
        def download_ext_handler_pkg(_uri, _destination):
            DownloadExtensionTestCase._create_invalid_zip_file(self._get_extension_package_file())
            return True

        with patch("time.sleep", lambda *_: None):
            with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
                with self.assertRaises(ExtensionDownloadError) as context_manager:
                    self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, NUMBER_OF_DOWNLOAD_RETRIES * len(self.pkg.uris))

        self.assertRegex(str(context_manager.exception), "Failed to download extension")
        self.assertEquals(context_manager.exception.code, ExtensionErrorCodes.PluginManifestDownloadError)

        self.assertFalse(os.path.exists(self.extension_dir), "The extension directory was not removed")
        self.assertFalse(os.path.exists(self._get_extension_package_file()), "The extension package was not removed")

<commit_msg>nit fix; incorrect zipfile name being passed to the tests.<commit_after># Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache License.

import zipfile, time

from azurelinuxagent.common.protocol.restapi import ExtHandler, ExtHandlerProperties, ExtHandlerPackage, ExtHandlerVersionUri
from azurelinuxagent.common.protocol.wire import WireProtocol
from azurelinuxagent.ga.exthandlers import ExtHandlerInstance, NUMBER_OF_DOWNLOAD_RETRIES
from azurelinuxagent.common.exception import ExtensionDownloadError, ExtensionErrorCodes
from tests.tools import *

class DownloadExtensionTestCase(AgentTestCase):
    """
    Test cases for launch_command
    """
    @classmethod
    def setUpClass(cls):
        AgentTestCase.setUpClass()
        cls.mock_cgroups = patch("azurelinuxagent.ga.exthandlers.CGroupConfigurator")
        cls.mock_cgroups.start()

    @classmethod
    def tearDownClass(cls):
        cls.mock_cgroups.stop()

        AgentTestCase.tearDownClass()

    def setUp(self):
        AgentTestCase.setUp(self)

        ext_handler_properties = ExtHandlerProperties()
        ext_handler_properties.version = "1.0.0"
        ext_handler = ExtHandler(name='Microsoft.CPlat.Core.RunCommandLinux')
        ext_handler.properties = ext_handler_properties

        protocol = WireProtocol("http://Microsoft.CPlat.Core.RunCommandLinux/foo-bar")

        self.pkg = ExtHandlerPackage()
        self.pkg.uris = [ ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri(), ExtHandlerVersionUri() ]
        self.pkg.uris[0].uri = 'https://zrdfepirv2cy4prdstr00a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[1].uri = 'https://zrdfepirv2cy4prdstr01a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[2].uri = 'https://zrdfepirv2cy4prdstr02a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[3].uri = 'https://zrdfepirv2cy4prdstr03a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'
        self.pkg.uris[4].uri = 'https://zrdfepirv2cy4prdstr04a.blob.core.windows.net/f72653efd9e349ed9842c8b99e4c1712-foobar/Microsoft.CPlat.Core__RunCommandLinux__1.0.0'

        self.ext_handler_instance = ExtHandlerInstance(ext_handler=ext_handler, protocol=protocol)
        self.ext_handler_instance.pkg = self.pkg

        self.extension_dir = os.path.join(self.tmp_dir, "Microsoft.CPlat.Core.RunCommandLinux-1.0.0")
        self.mock_get_base_dir = patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_base_dir", return_value=self.extension_dir)
        self.mock_get_base_dir.start()

        self.mock_get_log_dir = patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.get_log_dir", return_value=self.tmp_dir)
        self.mock_get_log_dir.start()

        self.agent_dir = self.tmp_dir
        self.mock_get_lib_dir = patch("azurelinuxagent.ga.exthandlers.conf.get_lib_dir", return_value=self.agent_dir)
        self.mock_get_lib_dir.start()

    def tearDown(self):
        self.mock_get_lib_dir.stop()
        self.mock_get_log_dir.stop()
        self.mock_get_base_dir.stop()

        AgentTestCase.tearDown(self)

    _extension_command = "RunCommandLinux.sh"

    @staticmethod
    def _create_zip_file(filename):
        file = None
        try:
            file = zipfile.ZipFile(filename, "w")
            info = zipfile.ZipInfo(DownloadExtensionTestCase._extension_command)
            info.date_time = time.localtime(time.time())[:6]
            info.compress_type = zipfile.ZIP_DEFLATED
            file.writestr(info, "#!/bin/sh\necho 'RunCommandLinux executed successfully'\n")
        finally:
            if file is not None:
                file.close()

    @staticmethod
    def _create_invalid_zip_file(filename):
        with open(filename, "w") as file:
            file.write("An invalid ZIP file\n")

    def _get_extension_package_file(self):
        return os.path.join(self.agent_dir, self.ext_handler_instance.get_extension_package_zipfile_name())

    def _get_extension_command_file(self):
        return os.path.join(self.extension_dir, DownloadExtensionTestCase._extension_command)

    def _assert_download_and_expand_succeeded(self):
        self.assertTrue(os.path.exists(self._get_extension_package_file()), "The extension package was not downloaded to the expected location")
        self.assertTrue(os.path.exists(self._get_extension_command_file()), "The extension package was not expanded to the expected location")

    def test_it_should_download_and_expand_extension_package(self):
        def download_ext_handler_pkg(_uri, destination):
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.report_event") as mock_report_event:
                self.ext_handler_instance.download()

        # first download attempt should succeed
        mock_download_ext_handler_pkg.assert_called_once()
        mock_report_event.assert_called_once()

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_existing_extension_package_when_already_downloaded(self):
        DownloadExtensionTestCase._create_zip_file(self._get_extension_package_file())

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg") as mock_download_ext_handler_pkg:
            with patch("azurelinuxagent.ga.exthandlers.ExtHandlerInstance.report_event") as mock_report_event:
                self.ext_handler_instance.download()

        mock_download_ext_handler_pkg.assert_not_called()
        mock_report_event.assert_not_called()

        self.assertTrue(os.path.exists(self._get_extension_command_file()), "The extension package was not expanded to the expected location")

    def test_it_should_ignore_existing_extension_package_when_it_is_invalid(self):
        def download_ext_handler_pkg(_uri, destination):
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        DownloadExtensionTestCase._create_invalid_zip_file(self._get_extension_package_file())

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        mock_download_ext_handler_pkg.assert_called_once()

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_download_fails(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                return False
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_download_raises_an_exception(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                raise Exception("Download failed")
            DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_use_alternate_uris_when_it_downloads_an_invalid_package(self):
        self.download_failures = 0

        def download_ext_handler_pkg(_uri, destination):
            # fail a few times, then succeed
            if self.download_failures < 3:
                self.download_failures += 1
                DownloadExtensionTestCase._create_invalid_zip_file(destination)
            else:
                DownloadExtensionTestCase._create_zip_file(destination)
            return True

        with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
            self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, self.download_failures + 1)

        self._assert_download_and_expand_succeeded()

    def test_it_should_raise_an_exception_when_all_downloads_fail(self):
        def download_ext_handler_pkg(_uri, _destination):
            DownloadExtensionTestCase._create_invalid_zip_file(self._get_extension_package_file())
            return True

        with patch("time.sleep", lambda *_: None):
            with patch("azurelinuxagent.common.protocol.wire.WireProtocol.download_ext_handler_pkg", side_effect=download_ext_handler_pkg) as mock_download_ext_handler_pkg:
                with self.assertRaises(ExtensionDownloadError) as context_manager:
                    self.ext_handler_instance.download()

        self.assertEquals(mock_download_ext_handler_pkg.call_count, NUMBER_OF_DOWNLOAD_RETRIES * len(self.pkg.uris))

        self.assertRegex(str(context_manager.exception), "Failed to download extension")
        self.assertEquals(context_manager.exception.code, ExtensionErrorCodes.PluginManifestDownloadError)

        self.assertFalse(os.path.exists(self.extension_dir), "The extension directory was not removed")
        self.assertFalse(os.path.exists(self._get_extension_package_file()), "The extension package was not removed")

<|endoftext|>


<commit_before>from unittest import TestCase

from wallace.interval_storage import IntervalStorage
from wallace.settings import AbstractSettings

class IntervalStorageTest(TestCase):
    def test_interval_storage_with_a_single_interval(self):
        interval_storage = IntervalStorage()
        self.assertFalse(interval_storage.has_intersection(0.0, 1.0))
        interval_storage.add_interval("1", 0.0, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.5))

        with self.assertRaises(ValueError):
            interval_storage.get_entry(1.5)

    def test_interval_storage_for_invalid_intervals(self):
        interval_storage = IntervalStorage()
        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", 0.0, 1.2)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", -3.2, 0.8)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", -3.2, 1.4)

    def test_interval_storage_for_completely_containing_intervals(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.5, 0.7)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("2", 0.55, 0.65)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("3", 0.45, 0.75)

    def test_interval_storage_with_two_intervals_and_midpoint_get(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.0, 0.5)
        interval_storage.add_interval("2", 0.5, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.25))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("2", interval_storage.get_entry(0.8))

    def test_interval_storage_with_three_intervals(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.0, 0.3)
        interval_storage.add_interval("2", 0.3, 0.6)
        interval_storage.add_interval("3", 0.6, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.0))
        self.assertEqual("1", interval_storage.get_entry(0.2))
        self.assertEqual("2", interval_storage.get_entry(0.3))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("3", interval_storage.get_entry(0.6))
        self.assertEqual("3", interval_storage.get_entry(0.85))

        with self.assertRaises(ValueError):
            interval_storage.get_entry(1.0)

        with self.assertRaises(ValueError):
            interval_storage.get_entry(-0.01)

    def test_initializing_interval_storage_with_a_map(self):
        interval_storage = IntervalStorage({
            "1": (0.0, 0.3),
            "2": (0.3, 0.6),
            "3": (0.6, 1.0)
            })

        self.assertEqual("1", interval_storage.get_entry(0.0))
        self.assertEqual("1", interval_storage.get_entry(0.2))
        self.assertEqual("2", interval_storage.get_entry(0.3))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("3", interval_storage.get_entry(0.6))
        self.assertEqual("3", interval_storage.get_entry(0.85))

    def test_initializing_interval_storage_with_a_map_with_invalid_intervals(self):
        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (0.0, 0.5),
                "2": (0.4, 1.0)
                })

        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (-10.0, 20.5),
                "2": (0.4, 1.0)
                })

        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (0.0, 0.5),
                "2": (0.5, 1.2)
                })
<commit_msg>Add tests for has_entry on the IntervalStorage class.<commit_after>from unittest import TestCase

from wallace.interval_storage import IntervalStorage
from wallace.settings import AbstractSettings

class IntervalStorageTest(TestCase):
    def test_interval_storage_with_a_single_interval(self):
        interval_storage = IntervalStorage()
        self.assertFalse(interval_storage.has_intersection(0.0, 1.0))
        interval_storage.add_interval("1", 0.0, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.5))

        with self.assertRaises(ValueError):
            interval_storage.get_entry(1.5)

    def test_interval_storage_for_invalid_intervals(self):
        interval_storage = IntervalStorage()
        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", 0.0, 1.2)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", -3.2, 0.8)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("1", -3.2, 1.4)

    def test_interval_storage_for_completely_containing_intervals(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.5, 0.7)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("2", 0.55, 0.65)

        with self.assertRaises(ValueError):
            interval_storage.add_interval("3", 0.45, 0.75)

    def test_interval_storage_for_has_entry(self):
        interval_storage = IntervalStorage()
        self.assertFalse(interval_storage.has_entry(0.5))
        self.assertFalse(interval_storage.has_entry(1.5))

        interval_storage.add_interval("1", 0.3, 0.7)
        self.assertTrue(interval_storage.has_entry(0.5))
        self.assertFalse(interval_storage.has_entry(0.8))
        self.assertFalse(interval_storage.has_entry(0.2))
        self.assertFalse(interval_storage.has_entry(-0.2))

    def test_interval_storage_with_two_intervals_and_midpoint_get(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.0, 0.5)
        interval_storage.add_interval("2", 0.5, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.25))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("2", interval_storage.get_entry(0.8))

    def test_interval_storage_with_three_intervals(self):
        interval_storage = IntervalStorage()
        interval_storage.add_interval("1", 0.0, 0.3)
        interval_storage.add_interval("2", 0.3, 0.6)
        interval_storage.add_interval("3", 0.6, 1.0)

        self.assertEqual("1", interval_storage.get_entry(0.0))
        self.assertEqual("1", interval_storage.get_entry(0.2))
        self.assertEqual("2", interval_storage.get_entry(0.3))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("3", interval_storage.get_entry(0.6))
        self.assertEqual("3", interval_storage.get_entry(0.85))

        with self.assertRaises(ValueError):
            interval_storage.get_entry(1.0)

        with self.assertRaises(ValueError):
            interval_storage.get_entry(-0.01)

    def test_initializing_interval_storage_with_a_map(self):
        interval_storage = IntervalStorage({
            "1": (0.0, 0.3),
            "2": (0.3, 0.6),
            "3": (0.6, 1.0)
            })

        self.assertEqual("1", interval_storage.get_entry(0.0))
        self.assertEqual("1", interval_storage.get_entry(0.2))
        self.assertEqual("2", interval_storage.get_entry(0.3))
        self.assertEqual("2", interval_storage.get_entry(0.5))
        self.assertEqual("3", interval_storage.get_entry(0.6))
        self.assertEqual("3", interval_storage.get_entry(0.85))

    def test_initializing_interval_storage_with_a_map_with_invalid_intervals(self):
        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (0.0, 0.5),
                "2": (0.4, 1.0)
                })

        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (-10.0, 20.5),
                "2": (0.4, 1.0)
                })

        with self.assertRaises(ValueError):
            interval_storage = IntervalStorage({
                "1": (0.0, 0.5),
                "2": (0.5, 1.2)
                })
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

""" Assumes fabric environment already set up """

__author__ = 'ilyash'

import subprocess
import time
import os
from zipfile import ZipFile

from fabric import operations
import fabric.api
from path import path
import sh

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import YamlPatcher, get_actual_keypath
from cosmo_tester.framework.util import get_blueprint_path

CHEF_SERVER_COOKBOOK_ZIP_URL = (
    'https://github.com/opscode-cookbooks/chef-server/archive/'
    'c588a4c401d3fac14f70d3285fe49eb4dccd9759.zip'
)

KNIFE_PARAMS = '-u admin -k ~/admin.pem'

def _use_cookbook(cookbook_name,
                  cookbook_local_tar_path):
    """ Downloads cookbook from given url and uploads it to the Chef server """
    fabric.api.run('mkdir -p ~/cookbooks/{0}'.format(cookbook_name))
    fabric.api.put(local_path=cookbook_local_tar_path,
                   remote_path='/tmp/{0}.tar.gz'.format(cookbook_name))
    fabric.api.run('tar -xzvf /tmp/{0}.tar.gz --strip-components=1'
                   ' -C ~/cookbooks/{0}'.format(cookbook_name))
    fabric.api.run('knife cookbook upload {0} --cookbook-path ~/cookbooks {1}'
                   .format(KNIFE_PARAMS, cookbook_name))
    fabric.api.run('knife cookbook list {0} | grep -F {1}'
                   .format(KNIFE_PARAMS, cookbook_name))

def _userize_file(original_path):
    """ Places the file under user's home directory and make it
        permissions-wise accessible """
    fabric.api.sudo("cp -a {path} ~{user}/ && chown {user} ~{user}/{basename}"
                    .format(path=original_path,
                            basename=str(path(original_path).basename()),
                            user=fabric.api.env['user']))

def setup_chef_server(local_dir, cookbooks):
    _userize_file("/etc/chef-server/admin.pem")
    for cb in cookbooks:
        _use_cookbook(*cb)

    _userize_file("/etc/chef-server/chef-validator.pem")
    operations.get('~/chef-validator.pem', str(local_dir))

def find_node_state(node_name, nodes_state):
    pfx = node_name + '_'
    matches = [v for k, v in nodes_state.items() if k.startswith(pfx)]
    if len(matches) != 1:
        raise RuntimeError("Failed to find node {0}".format(node_name))
    return matches[0]

def get_nodes_of_type(blueprint, type_):
    return [node_obj for _, node_obj in blueprint.obj[
        'node_templates'].iteritems() if node_obj['type'] == type_]

def update_blueprint(env, blueprint, hostname, userdata_vars=None):
    hostname_base = 'system-test-{0}-{1}'.format(
        time.strftime("%Y%m%d-%H%M"), hostname)
    vms = get_nodes_of_type(blueprint, 'cloudify.openstack.server')
    if len(vms) > 1:
        hostnames = ['{0}-{1:2}'.format(hostname_base, i)
                     for i in range(0, len(vms))]
    else:
        hostnames = [hostname_base]

    users = []
    for vm_idx, vm in enumerate(vms):
        vm_hostname = hostnames[vm_idx]
        # vm['properties']['server'] does not exist when using existing one
        if 'server' in vm['properties']:
            vm['properties']['server'].update({
                'flavor_name': env.flavor_name,
                'image_name': env.ubuntu_image_name,
                'name': vm_hostname,
            })
            props = vm['properties']['server']
            if 'userdata' in props:
                props['userdata'] = props['userdata'].format(
                    hostname='{0}{1}'.format(env.resources_prefix,
                                             vm_hostname).replace('_', '-'),
                    **(userdata_vars or {}))
        users.append('ubuntu')
    return {'hostnames': hostnames, 'users': users}

class ChefPluginClientTest(TestCase):

    def setUp(self, *args, **kwargs):

        super(ChefPluginClientTest, self).setUp(*args, **kwargs)
        agent_key_file = get_actual_keypath(self.env,
                                            self.env.agent_key_path)

        blueprint_dir = self.copy_blueprint('chef-plugin')
        self.blueprint_yaml = blueprint_dir / 'chef-server-by-chef-solo.yaml'

        with YamlPatcher(self.blueprint_yaml) as blueprint:
            bp_info = update_blueprint(self.env, blueprint, 'chef-server')

        self.chef_server_hostname = bp_info['hostnames'][0]

        cookbooks_dir = blueprint_dir / 'cookbooks'

        def run(*args, **kwargs):
            return subprocess.check_output(*args, **kwargs)

        with cookbooks_dir:
            run([
                'wget', '-q', '-O', 'chef-server.zip',
                CHEF_SERVER_COOKBOOK_ZIP_URL,
                ])
            ZipFile('chef-server.zip').extractall()
            chef_cookbook_dir = cookbooks_dir.glob('chef-server-*')[0]
            run(['mv', chef_cookbook_dir, 'chef-server'])
            # Next line because Chef cookbooks are required
            # to declare all dependencies, even if they don't use them.
            # We don't need git, it's only used in chef-cookbook::dev recipe.
            run(['sed', '-i', "/depends 'git'/d", 'chef-server/metadata.rb'])

        with blueprint_dir:
            run(['tar', 'czf', 'cookbooks.tar.gz', 'cookbooks'])

        self.chef_server_id = self.test_id + '-chef-server'
        id_ = self.chef_server_id
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        self.chef_server_ip = fip_node['runtime_properties'][
            'floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': bp_info['users'][0],
            'key_filename': str(agent_key_file),
            'host_string': self.chef_server_ip,
        })

        cookbook_local_path = os.path.abspath(
            os.path.join(get_blueprint_path('chef-plugin'),
                         'cookbook-create-file.tar.gz'))
        setup_chef_server(blueprint_dir, [[
            'create-file',
            cookbook_local_path,
        ]])
        self.blueprint_dir = blueprint_dir

    def tearDown(self, *args, **kwargs):
        self.execute_uninstall(self.chef_server_id)
        super(ChefPluginClientTest, self).tearDown(*args, **kwargs)

    def test_chef_client(self):
        blueprint_dir = self.blueprint_dir
        self.blueprint_yaml = blueprint_dir / 'chef-client-test.yaml'
        with YamlPatcher(self.blueprint_yaml) as blueprint:
            update_blueprint(self.env, blueprint, 'chef-server', {
                'chef_server_ip': self.chef_server_ip,
                'chef_server_hostname': self.chef_server_hostname,
            })
            chef_node = get_nodes_of_type(blueprint,
                                          'cloudify.types.chef.db_server')[0]
            chef_config = chef_node['properties']['chef_config']
            chef_config['chef_server_url'] = 'https://{0}:443'.format(
                self.chef_server_ip)
            chef_config['validation_client_name'] = 'chef-validator'
            chef_config['validation_key'] = (
                path(blueprint_dir) / 'chef-validator.pem').text()

        id_ = self.test_id + '-chef-client-' + str(int(time.time()))
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        chef_client_ip = fip_node['runtime_properties']['floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            # XXX: sometime - same user for connection is accidental
            #      todo: replace it with update_blueprint()'s bp_info,
            #            as in setUp()
            'host_string': chef_client_ip,
        })

        out = fabric.api.run('cat /tmp/blueprint.txt')
        self.assertEquals(out, 'Great success!')

        self.execute_uninstall(id_)

class ChefPluginSoloTest(TestCase):

    def setUp(self, *args, **kwargs):

        super(ChefPluginSoloTest, self).setUp(*args, **kwargs)

        self.blueprint_dir = self.copy_blueprint('chef-plugin')

        # Get resources
        with self.blueprint_dir:
            for res in 'cookbooks', 'data_bags', 'environments', 'roles':
                sh.tar('czf', res+'.tar.gz', res)

    def test_chef_solo(self):
        agent_key_file = get_actual_keypath(self.env,
                                            self.env.agent_key_path)
        blueprint_dir = self.blueprint_dir
        self.blueprint_yaml = blueprint_dir / 'chef-solo-test.yaml'
        with YamlPatcher(self.blueprint_yaml) as blueprint:
            bp_info = update_blueprint(self.env, blueprint, 'chef-solo')

        id_ = self.test_id + '-chef-solo-' + str(int(time.time()))
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        chef_solo_ip = fip_node['runtime_properties']['floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': bp_info['users'][0],
            'key_filename': str(agent_key_file),
            'host_string': chef_solo_ip,
        })

        expected_files_contents = (
            ('/tmp/blueprint.txt', 'Great success number #2 !'),
            ('/tmp/blueprint2.txt', '/tmp/blueprint.txt'),
            ('/tmp/chef_node_env.e1.txt', 'env:e1'),
            ('/tmp/chef_node_data_bag_user.db1.i1.txt', 'db1-i1-k1'),
        )

        for file_name, expected_content in expected_files_contents:
            actual_content = fabric.api.run('cat ' + file_name)
            msg = "File '{0}' should have content '{1}' but has '{2}'".format(
                file_name, expected_content, actual_content)
            self.assertEquals(actual_content, expected_content, msg)

        self.execute_uninstall(id_)
<commit_msg>append resources prefix to chef server hostname<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

""" Assumes fabric environment already set up """

__author__ = 'ilyash'

import subprocess
import time
import os
from zipfile import ZipFile

from fabric import operations
import fabric.api
from path import path
import sh

from cosmo_tester.framework.testenv import TestCase
from cosmo_tester.framework.util import YamlPatcher, get_actual_keypath
from cosmo_tester.framework.util import get_blueprint_path

CHEF_SERVER_COOKBOOK_ZIP_URL = (
    'https://github.com/opscode-cookbooks/chef-server/archive/'
    'c588a4c401d3fac14f70d3285fe49eb4dccd9759.zip'
)

KNIFE_PARAMS = '-u admin -k ~/admin.pem'

def _use_cookbook(cookbook_name,
                  cookbook_local_tar_path):
    """ Downloads cookbook from given url and uploads it to the Chef server """
    fabric.api.run('mkdir -p ~/cookbooks/{0}'.format(cookbook_name))
    fabric.api.put(local_path=cookbook_local_tar_path,
                   remote_path='/tmp/{0}.tar.gz'.format(cookbook_name))
    fabric.api.run('tar -xzvf /tmp/{0}.tar.gz --strip-components=1'
                   ' -C ~/cookbooks/{0}'.format(cookbook_name))
    fabric.api.run('knife cookbook upload {0} --cookbook-path ~/cookbooks {1}'
                   .format(KNIFE_PARAMS, cookbook_name))
    fabric.api.run('knife cookbook list {0} | grep -F {1}'
                   .format(KNIFE_PARAMS, cookbook_name))

def _userize_file(original_path):
    """ Places the file under user's home directory and make it
        permissions-wise accessible """
    fabric.api.sudo("cp -a {path} ~{user}/ && chown {user} ~{user}/{basename}"
                    .format(path=original_path,
                            basename=str(path(original_path).basename()),
                            user=fabric.api.env['user']))

def setup_chef_server(local_dir, cookbooks):
    _userize_file("/etc/chef-server/admin.pem")
    for cb in cookbooks:
        _use_cookbook(*cb)

    _userize_file("/etc/chef-server/chef-validator.pem")
    operations.get('~/chef-validator.pem', str(local_dir))

def find_node_state(node_name, nodes_state):
    pfx = node_name + '_'
    matches = [v for k, v in nodes_state.items() if k.startswith(pfx)]
    if len(matches) != 1:
        raise RuntimeError("Failed to find node {0}".format(node_name))
    return matches[0]

def get_nodes_of_type(blueprint, type_):
    return [node_obj for _, node_obj in blueprint.obj[
        'node_templates'].iteritems() if node_obj['type'] == type_]

def update_blueprint(env, blueprint, hostname, userdata_vars=None):
    hostname_base = 'system-test-{0}-{1}'.format(
        time.strftime("%Y%m%d-%H%M"), hostname)
    vms = get_nodes_of_type(blueprint, 'cloudify.openstack.server')
    if len(vms) > 1:
        hostnames = ['{0}-{1:2}'.format(hostname_base, i)
                     for i in range(0, len(vms))]
    else:
        hostnames = [hostname_base]

    users = []
    for vm_idx, vm in enumerate(vms):
        vm_hostname = hostnames[vm_idx]
        # vm['properties']['server'] does not exist when using existing one
        if 'server' in vm['properties']:
            vm['properties']['server'].update({
                'flavor_name': env.flavor_name,
                'image_name': env.ubuntu_image_name,
                'name': vm_hostname,
            })
            props = vm['properties']['server']
            if 'userdata' in props:
                props['userdata'] = props['userdata'].format(
                    hostname='{0}{1}'.format(env.resources_prefix,
                                             vm_hostname).replace('_', '-'),
                    **(userdata_vars or {}))
        users.append('ubuntu')
    return {'hostnames': hostnames, 'users': users}

class ChefPluginClientTest(TestCase):

    def setUp(self, *args, **kwargs):

        super(ChefPluginClientTest, self).setUp(*args, **kwargs)
        agent_key_file = get_actual_keypath(self.env,
                                            self.env.agent_key_path)

        blueprint_dir = self.copy_blueprint('chef-plugin')
        self.blueprint_yaml = blueprint_dir / 'chef-server-by-chef-solo.yaml'

        with YamlPatcher(self.blueprint_yaml) as blueprint:
            bp_info = update_blueprint(self.env, blueprint, 'chef-server')

        self.chef_server_hostname = '{0}{1}'.format(self.env.resources_prefix,
                                                    bp_info['hostnames'][0])

        cookbooks_dir = blueprint_dir / 'cookbooks'

        def run(*args, **kwargs):
            return subprocess.check_output(*args, **kwargs)

        with cookbooks_dir:
            run([
                'wget', '-q', '-O', 'chef-server.zip',
                CHEF_SERVER_COOKBOOK_ZIP_URL,
                ])
            ZipFile('chef-server.zip').extractall()
            chef_cookbook_dir = cookbooks_dir.glob('chef-server-*')[0]
            run(['mv', chef_cookbook_dir, 'chef-server'])
            # Next line because Chef cookbooks are required
            # to declare all dependencies, even if they don't use them.
            # We don't need git, it's only used in chef-cookbook::dev recipe.
            run(['sed', '-i', "/depends 'git'/d", 'chef-server/metadata.rb'])

        with blueprint_dir:
            run(['tar', 'czf', 'cookbooks.tar.gz', 'cookbooks'])

        self.chef_server_id = self.test_id + '-chef-server'
        id_ = self.chef_server_id
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        self.chef_server_ip = fip_node['runtime_properties'][
            'floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': bp_info['users'][0],
            'key_filename': str(agent_key_file),
            'host_string': self.chef_server_ip,
        })

        cookbook_local_path = os.path.abspath(
            os.path.join(get_blueprint_path('chef-plugin'),
                         'cookbook-create-file.tar.gz'))
        setup_chef_server(blueprint_dir, [[
            'create-file',
            cookbook_local_path,
        ]])
        self.blueprint_dir = blueprint_dir

    def tearDown(self, *args, **kwargs):
        self.execute_uninstall(self.chef_server_id)
        super(ChefPluginClientTest, self).tearDown(*args, **kwargs)

    def test_chef_client(self):
        blueprint_dir = self.blueprint_dir
        self.blueprint_yaml = blueprint_dir / 'chef-client-test.yaml'
        with YamlPatcher(self.blueprint_yaml) as blueprint:
            update_blueprint(self.env, blueprint, 'chef-server', {
                'chef_server_ip': self.chef_server_ip,
                'chef_server_hostname': self.chef_server_hostname,
            })
            chef_node = get_nodes_of_type(blueprint,
                                          'cloudify.types.chef.db_server')[0]
            chef_config = chef_node['properties']['chef_config']
            chef_config['chef_server_url'] = 'https://{0}:443'.format(
                self.chef_server_ip)
            chef_config['validation_client_name'] = 'chef-validator'
            chef_config['validation_key'] = (
                path(blueprint_dir) / 'chef-validator.pem').text()

        id_ = self.test_id + '-chef-client-' + str(int(time.time()))
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        chef_client_ip = fip_node['runtime_properties']['floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            # XXX: sometime - same user for connection is accidental
            #      todo: replace it with update_blueprint()'s bp_info,
            #            as in setUp()
            'host_string': chef_client_ip,
        })

        out = fabric.api.run('cat /tmp/blueprint.txt')
        self.assertEquals(out, 'Great success!')

        self.execute_uninstall(id_)

class ChefPluginSoloTest(TestCase):

    def setUp(self, *args, **kwargs):

        super(ChefPluginSoloTest, self).setUp(*args, **kwargs)

        self.blueprint_dir = self.copy_blueprint('chef-plugin')

        # Get resources
        with self.blueprint_dir:
            for res in 'cookbooks', 'data_bags', 'environments', 'roles':
                sh.tar('czf', res+'.tar.gz', res)

    def test_chef_solo(self):
        agent_key_file = get_actual_keypath(self.env,
                                            self.env.agent_key_path)
        blueprint_dir = self.blueprint_dir
        self.blueprint_yaml = blueprint_dir / 'chef-solo-test.yaml'
        with YamlPatcher(self.blueprint_yaml) as blueprint:
            bp_info = update_blueprint(self.env, blueprint, 'chef-solo')

        id_ = self.test_id + '-chef-solo-' + str(int(time.time()))
        before, after = self.upload_deploy_and_execute_install(id_, id_)

        fip_node = find_node_state('ip', after['node_state'][id_])
        chef_solo_ip = fip_node['runtime_properties']['floating_ip_address']

        fabric_env = fabric.api.env
        fabric_env.update({
            'timeout': 30,
            'user': bp_info['users'][0],
            'key_filename': str(agent_key_file),
            'host_string': chef_solo_ip,
        })

        expected_files_contents = (
            ('/tmp/blueprint.txt', 'Great success number #2 !'),
            ('/tmp/blueprint2.txt', '/tmp/blueprint.txt'),
            ('/tmp/chef_node_env.e1.txt', 'env:e1'),
            ('/tmp/chef_node_data_bag_user.db1.i1.txt', 'db1-i1-k1'),
        )

        for file_name, expected_content in expected_files_contents:
            actual_content = fabric.api.run('cat ' + file_name)
            msg = "File '{0}' should have content '{1}' but has '{2}'".format(
                file_name, expected_content, actual_content)
            self.assertEquals(actual_content, expected_content, msg)

        self.execute_uninstall(id_)
<|endoftext|>


<commit_before>import os

from django.conf import settings
from django.core.management.base import BaseCommand
from optparse import make_option

from create_indexes import create_indexes
from create_locations import add_location_objects
from setconfig import create_config_if_not_exist
from create_default_dashboard import create_dashboard

from crits.core.crits_mongoengine import Action
from crits.core.user_role import UserRole
from crits.domains.domain import TLD
from crits.raw_data.raw_data import RawDataType
from crits.signatures.signature import SignatureType

class Command(BaseCommand):
    """
    Script Class.
    """

    option_list = BaseCommand.option_list + (
        make_option('--drop',
                    '-d',
                    dest='drop',
                    action="store_true",
                    default=False,
                    help='Drop existing content before adding.'),
    )
    help = 'Creates default CRITs collections in MongoDB.'

    def handle(self, *args, **options):
        """
        Script Execution.
        """

        drop = options.get('drop')
        if drop:
            print "Dropping enabled. Will drop content before adding!"
        else:
            print "Drop protection enabled. Will not drop existing content!"
        populate_user_roles(drop)
        populate_actions(drop)
        populate_raw_data_types(drop)
        populate_signature_types(drop)
        # The following will always occur with every run of this script:
        #   - tlds are based off of a Mozilla TLD list so it should never
        #     contain  entries outside of the ones provided.
        populate_tlds(drop)
        add_location_objects(drop)
        create_dashboard(drop)
        create_config_if_not_exist()
        create_indexes()

def populate_user_roles(drop):
    """
    Populate default set of user roles into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your user roles here
    # note: you MUST have Administrator, Read Only, and a third option
    # available!
    user_roles = ['Administrator', 'Analyst', 'Read Only']
    if drop:
        UserRole.drop_collection()
    if len(UserRole.objects()) < 1:
        for role in user_roles:
            ur = UserRole()
            ur.name = role
            ur.save()
        print "User Roles: added %s roles!" % len(user_roles)
    else:
        print "User Roles: existing documents detected. skipping!"

def populate_actions(drop):
    """
    Populate default set of Actions into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your Actions here
    actions = ['Blocked Outbound At Firewall', 'Blocked Outbound At Desktop Firewall']
    if drop:
        Action.drop_collection()
    if len(Action.objects()) < 1:
        for action in actions:
            ia = Action()
            ia.name = action
            ia.save()
        print "Actions: added %s actions!" % len(actions)
    else:
        print "Actions: existing documents detected. skipping!"

def populate_raw_data_types(drop):
    """
    Populate default set of raw data types into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your raw data types here
    data_types = ['Text', 'JSON']
    if drop:
        RawDataType.drop_collection()
    if len(RawDataType.objects()) < 1:
        for data_type in data_types:
            dt = RawDataType()
            dt.name = data_type
            dt.save()
        print "Raw Data Types: added %s types!" % len(data_types)
    else:
        print "Raw Data Types: existing documents detected. skipping!"

def populate_signature_types(drop):
    """
    Populate default set of signature types into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your signature types here
    data_types = ['Bro', 'Snort', 'Yara']
    if drop:
        SignatureType.drop_collection()
    if len(SignatureType.objects()) < 1:
        for data_type in data_types:
            dt = SignatureType()
            dt.name = data_type
            dt.save()
        print "Signature Types: added %s types!" % len(data_types)
    else:
        print "Signature Types: existing documents detected. skipping!"

def populate_tlds(drop):
    """
    Populate default set of TLDs into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    if not drop:
        print "Drop protection does not apply to effective TLDs"
    TLD.drop_collection()
    f = os.path.join(settings.SITE_ROOT, '..', 'extras', 'effective_tld_names.dat')
    count = 0
    for line in open(f, 'r').readlines():
        line = line.strip()
        if line and not line.startswith('//'):
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
            count += 1
    print "Effective TLDs: added %s TLDs!" % count
<commit_msg>Update create_default_collections.py<commit_after>import os

from django.conf import settings
from django.core.management.base import BaseCommand
from optparse import make_option

from create_indexes import create_indexes
from create_locations import add_location_objects
from setconfig import create_config_if_not_exist
from create_default_dashboard import create_dashboard

from crits.core.crits_mongoengine import Action
from crits.core.user_role import UserRole
from crits.domains.domain import TLD
from crits.raw_data.raw_data import RawDataType
from crits.signatures.signature import SignatureType

class Command(BaseCommand):
    """
    Script Class.
    """

    option_list = BaseCommand.option_list + (
        make_option('--drop',
                    '-d',
                    dest='drop',
                    action="store_true",
                    default=False,
                    help='Drop existing content before adding.'),
    )
    help = 'Creates default CRITs collections in MongoDB.'

    def handle(self, *args, **options):
        """
        Script Execution.
        """

        drop = options.get('drop')
        if drop:
            print "Dropping enabled. Will drop content before adding!"
        else:
            print "Drop protection enabled. Will not drop existing content!"
        populate_user_roles(drop)
        populate_actions(drop)
        populate_raw_data_types(drop)
        populate_signature_types(drop)
        # The following will always occur with every run of this script:
        #   - tlds are based off of a Mozilla TLD list so it should never
        #     contain  entries outside of the ones provided.
        populate_tlds(drop)
        add_location_objects(drop)
        create_dashboard(drop)
        create_config_if_not_exist()
        create_indexes()

def populate_user_roles(drop):
    """
    Populate default set of user roles into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your user roles here
    # note: you MUST have Administrator, Read Only, and a third option
    # available!
    user_roles = ['Administrator', 'Analyst', 'Read Only']
    if drop:
        UserRole.drop_collection()
    if len(UserRole.objects()) < 1:
        for role in user_roles:
            ur = UserRole()
            ur.name = role
            ur.save()
        print "User Roles: added %s roles!" % len(user_roles)
    else:
        print "User Roles: existing documents detected. skipping!"

def populate_actions(drop):
    """
    Populate default set of Actions into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your Actions here
    actions = ['Blocked Outbound At Firewall', 'Blocked Outbound At Desktop Firewall']
    if drop:
        Action.drop_collection()
    if len(Action.objects()) < 1:
        for action in actions:
            ia = Action()
            ia.name = action
            ia.save()
        print "Actions: added %s actions!" % len(actions)
    else:
        print "Actions: existing documents detected. skipping!"

def populate_raw_data_types(drop):
    """
    Populate default set of raw data types into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your raw data types here
    data_types = ['Text', 'JSON']
    if drop:
        RawDataType.drop_collection()
    if len(RawDataType.objects()) < 1:
        for data_type in data_types:
            dt = RawDataType()
            dt.name = data_type
            dt.save()
        print "Raw Data Types: added %s types!" % len(data_types)
    else:
        print "Raw Data Types: existing documents detected. skipping!"

def populate_signature_types(drop):
    """
    Populate default set of signature types into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    # define your signature types here
    data_types = ['Bro', 'Snort', 'Yara']
    if drop:
        SignatureType.drop_collection()
    if len(SignatureType.objects()) < 1:
        for data_type in data_types:
            dt = SignatureType()
            dt.name = data_type
            dt.save()
        print "Signature Types: added %s types!" % len(data_types)
    else:
        print "Signature Types: existing documents detected. skipping!"

def populate_tlds(drop):
    """
    Populate default set of TLDs into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    if not drop:
        print "Drop protection does not apply to effective TLDs"
    TLD.drop_collection()
    f = os.path.join(settings.SITE_ROOT, '..', 'extras', 'effective_tld_names.dat')
    count = 0
    for line in open(f, 'r').readlines():
        line = line.strip()
        if line and not line.startswith('//'):
            line = line.replace("*.", "")
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
            count += 1
    print "Effective TLDs: added %s TLDs!" % count
<|endoftext|>