<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2020, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import unittest

import torch.nn.functional as F

from nupic.research.frameworks.dendrites import AbsoluteMaxGatingDendriticLayer
from nupic.research.frameworks.dendrites.routing import (
    init_dataloader,
    init_optimizer,
    init_test_scenario,
    train_dendrite_model,
)

class LearnToRouteTest(unittest.TestCase):
    """
    This test suite tests whether dendrite layer weights are changing when learning the
    routing task
    """

    def test_forward_weights(self):
        """
        Feed-forward weights should not be modified when `mode == "dendrites"`, and
        should be modified when `mode == "all"`
        """

        for mode in ("dendrites", "all", "learn_context"):

            r, dendrite_layer, context_model, context_vectors, device = \
                init_test_scenario(
                    mode=mode,
                    dim_in=100,
                    dim_out=100,
                    num_contexts=10,
                    dim_context=100,
                    dendrite_module=AbsoluteMaxGatingDendriticLayer
                )

            dataloader = init_dataloader(
                routing_function=r,
                context_vectors=context_vectors,
                device=device,
                batch_size=64,
                x_min=-2.0,
                x_max=2.0
            )

            optimizer = init_optimizer(
                mode=mode, layer=dendrite_layer, context_model=context_model
            )

            forward_weights_before = copy.deepcopy(
                dendrite_layer.module.weight.data
            )

            # Perform a single training epoch
            train_dendrite_model(
                model=dendrite_layer,
                context_model=context_model,
                loader=dataloader,
                optimizer=optimizer,
                device=device,
                criterion=F.l1_loss
            )

            forward_weights_after = copy.deepcopy(
                dendrite_layer.module.weight.data
            )

            expected = (forward_weights_before == forward_weights_after).all()

            # If training both feed-forward and dendrite weights, we expect the
            # dendrite weights to change
            if mode == "all" or "learn_context":
                expected = not expected

            self.assertTrue(expected)

    def test_dendrite_weights(self):
        """
        Dendrite weights should be modified both when `mode == "dendrites"` and when
        `mode == "all"`
        """

        for mode in ("dendrites", "all", "learn_context"):

            r, dendrite_layer, context_model, context_vectors, device = \
                init_test_scenario(
                    mode=mode,
                    dim_in=100,
                    dim_out=100,
                    num_contexts=10,
                    dim_context=100,
                    dendrite_module=AbsoluteMaxGatingDendriticLayer
                )

            dataloader = init_dataloader(
                routing_function=r,
                context_vectors=context_vectors,
                device=device,
                batch_size=64,
                x_min=-2.0,
                x_max=2.0
            )

            optimizer = init_optimizer(
                mode=mode, layer=dendrite_layer, context_model=context_model
            )

            dendrite_weights_before = copy.deepcopy(
                dendrite_layer.segments.weights.data
            )

            # Perform a single training epoch
            train_dendrite_model(
                model=dendrite_layer,
                context_model=context_model,
                loader=dataloader,
                optimizer=optimizer,
                device=device,
                criterion=F.l1_loss
            )

            dendrite_weights_after = copy.deepcopy(
                dendrite_layer.segments.weights.data
            )

            expected = (dendrite_weights_before != dendrite_weights_after).any()
            self.assertTrue(expected)

    def test_context_model(self):
        """
        Context model should be learned only when `mode == "learn_context"`
        """
        for mode in ("dendrites", "all", "learn_context"):

            r, dendrite_layer, context_model, context_vectors, device =\
                init_test_scenario(
                    mode=mode,
                    dim_in=100,
                    dim_out=100,
                    num_contexts=10,
                    dim_context=100,
                    dendrite_module=AbsoluteMaxGatingDendriticLayer
                )

            dataloader = init_dataloader(
                routing_function=r,
                context_vectors=context_vectors,
                device=device,
                batch_size=64,
                x_min=-2.0,
                x_max=2.0
            )

            optimizer = init_optimizer(
                mode=mode, layer=dendrite_layer, context_model=context_model
            )

            if mode != "learn_context":
                assert context_model is None
            else:
                context_weights_before = copy.deepcopy(
                    context_model.linear1.weight.data
                )
                # Perform a single training epoch
                train_dendrite_model(
                    model=dendrite_layer,
                    context_model=context_model,
                    loader=dataloader,
                    optimizer=optimizer,
                    device=device,
                    criterion=F.l1_loss
                )

                context_weights_after = copy.deepcopy(
                    context_model.linear1.weight.data
                )

                expected = (context_weights_before != context_weights_after).any()
                self.assertTrue(expected)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Made routing test slightly faster<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2020, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

import copy
import unittest

import torch.nn.functional as F

from nupic.research.frameworks.dendrites import AbsoluteMaxGatingDendriticLayer
from nupic.research.frameworks.dendrites.routing import (
    init_dataloader,
    init_optimizer,
    init_test_scenario,
    train_dendrite_model,
)

class LearnToRouteTest(unittest.TestCase):
    """
    This test suite tests whether dendrite layer weights are changing when learning the
    routing task
    """

    def test_forward_weights(self):
        """
        Feed-forward weights should not be modified when `mode == "dendrites"`, and
        should be modified when `mode == "all"`
        """

        for mode in ("dendrites", "all", "learn_context"):

            r, dendrite_layer, context_model, context_vectors, device = \
                init_test_scenario(
                    mode=mode,
                    dim_in=10,
                    dim_out=10,
                    num_contexts=2,
                    dim_context=10,
                    dendrite_module=AbsoluteMaxGatingDendriticLayer
                )

            dataloader = init_dataloader(
                routing_function=r,
                context_vectors=context_vectors,
                device=device,
                batch_size=64,
                x_min=-2.0,
                x_max=2.0
            )

            optimizer = init_optimizer(
                mode=mode, layer=dendrite_layer, context_model=context_model
            )

            forward_weights_before = copy.deepcopy(
                dendrite_layer.module.weight.data
            )

            # Perform a single training epoch
            train_dendrite_model(
                model=dendrite_layer,
                context_model=context_model,
                loader=dataloader,
                optimizer=optimizer,
                device=device,
                criterion=F.l1_loss
            )

            forward_weights_after = copy.deepcopy(
                dendrite_layer.module.weight.data
            )

            expected = (forward_weights_before == forward_weights_after).all()

            # If training both feed-forward and dendrite weights, we expect the
            # dendrite weights to change
            if mode == "all" or "learn_context":
                expected = not expected

            self.assertTrue(expected)

    def test_dendrite_weights(self):
        """
        Dendrite weights should be modified both when `mode == "dendrites"` and when
        `mode == "all"`
        """

        for mode in ("dendrites", "all", "learn_context"):

            r, dendrite_layer, context_model, context_vectors, device = \
                init_test_scenario(
                    mode=mode,
                    dim_in=10,
                    dim_out=10,
                    num_contexts=2,
                    dim_context=10,
                    dendrite_module=AbsoluteMaxGatingDendriticLayer
                )

            dataloader = init_dataloader(
                routing_function=r,
                context_vectors=context_vectors,
                device=device,
                batch_size=64,
                x_min=-2.0,
                x_max=2.0
            )

            optimizer = init_optimizer(
                mode=mode, layer=dendrite_layer, context_model=context_model
            )

            dendrite_weights_before = copy.deepcopy(
                dendrite_layer.segments.weights.data
            )

            # Perform a single training epoch
            train_dendrite_model(
                model=dendrite_layer,
                context_model=context_model,
                loader=dataloader,
                optimizer=optimizer,
                device=device,
                criterion=F.l1_loss
            )

            dendrite_weights_after = copy.deepcopy(
                dendrite_layer.segments.weights.data
            )

            expected = (dendrite_weights_before != dendrite_weights_after).any()
            self.assertTrue(expected)

    def test_context_model(self):
        """
        Context model should be learned only when `mode == "learn_context"`
        """
        r, dendrite_layer, context_model, context_vectors, device =\
            init_test_scenario(
                mode="learn_context",
                dim_in=10,
                dim_out=10,
                num_contexts=2,
                dim_context=10,
                dendrite_module=AbsoluteMaxGatingDendriticLayer
            )

        dataloader = init_dataloader(
            routing_function=r,
            context_vectors=context_vectors,
            device=device,
            batch_size=64,
            x_min=-2.0,
            x_max=2.0
        )

        optimizer = init_optimizer(
            mode="learn_context", layer=dendrite_layer, context_model=context_model
        )

        context_weights_before = copy.deepcopy(
            context_model.linear1.weight.data
        )
        # Perform a single training epoch
        train_dendrite_model(
            model=dendrite_layer,
            context_model=context_model,
            loader=dataloader,
            optimizer=optimizer,
            device=device,
            criterion=F.l1_loss
        )

        context_weights_after = copy.deepcopy(
            context_model.linear1.weight.data
        )

        expected = (context_weights_before != context_weights_after).any()
        self.assertTrue(expected)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>#
# Script to get source from SVN and build SansView
#
# Read the release notes to make ensure that the required software is installed.
#
# SVN must be installed:
# http://subversion.tigris.org/servlets/ProjectDocumentList?folderID=91
#
#
# On Windows:
#   - make sure svn.exe in on the path. You might need to log out and log back in again after installing SVN.
#
#   - Inno Setup must be installed
#
#   - py2exe must be installed
#
#   - mingw must be installed
#
# On Mac:
#   - py2app must be installed
#   - macholib must be installed to use py2app
#   - modulegraph must be installed to use py2app
#
# Usage:
# python build_sansview [command]
# [command] can be any of the following:
#   -h: lists the command line options
#   -r: Builds a SansView using the released modules.
#   -t: Builds SansView from the trubuild_sank.
#   -i: Builds a Windows installer from the release version.
#   -n: Print out the dependencies for the release notes

import os
import sys
import shutil
import logging

# Installation folder
import time
timestamp = int(time.time())
CWD    = os.getcwd()
INSTALL_FOLDER = "install_%s" % str(timestamp)

# On Windows, the python executable is not always on the path.
# Use its most frequent location as the default.
if sys.platform == 'win32':
    PYTHON = "c:\python25\python"
    LIB_FOLDER = "%s/%s" % (CWD, INSTALL_FOLDER)
else:
    PYTHON = 'python'

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s',
                    filename='build_%s.log' % str(timestamp),
                    filemode='w')

SVN    = "svn"
INNO   = "\"c:\Program Files\Inno Setup 5\ISCC\""

# Release version 1.9
SANSMODELS = "0.9.1"
DATALOADER = "0.9"
GUIFRAME   = "0.9.1"
SANSVIEW   = "1.9.1"
PLOTTOOLS  = "0.9.1"
UTIL       = "0.9.1"
PARK       = "1.2.1"
PARK_INTEG = "0.9.1"
PRVIEW     = "0.9.1"
PR_INV     = "0.9"
CALCULATOR = "0.9.1"
CALC_VIEW  = "0.9"
INVARIANT  = "0.9.1"
INV_VIEW   = "0.9"
FIT_VIEW   = "0.9.1"

# URLs for SVN repos
SANSMODELS_URL = "svn://danse.us/sans/releases/sansmodels-%s" % SANSMODELS
DATALOADER_URL = "svn://danse.us/sans/releases/sansdataloader-%s" % DATALOADER
GUIFRAME_URL = "svn://danse.us/sans/releases/sansguiframe-%s" % GUIFRAME
PLOTTOOLS_URL = "svn://danse.us/common/releases/plottools-%s" % PLOTTOOLS
UTIL_URL = "svn://danse.us/common/releases/util-%s" % UTIL
SANSVIEW_URL = "svn://danse.us/sans/releases/sansview-%s" % SANSVIEW
FIT_URL = "svn://danse.us/sans/releases/fittingview-%s" % FIT_VIEW
PARK_INTEG_URL = "svn://danse.us/sans/releases/park_integration-%s" % PARK_INTEG
PARK_URL = "svn://danse.us/park/releases/park-%s" % PARK
PRVIEW_URL = "svn://danse.us/sans/releases/inversionview-%s" % PRVIEW
PR_INV_URL = "svn://danse.us/sans/releases/pr_inversion-%s" % PR_INV
CALC_URL = "svn://danse.us/sans/releases/sanscalculator-%s" % CALCULATOR
CALC_VIEW_URL = "svn://danse.us/sans/releases/calculatorview-%s" % CALC_VIEW
INV_URL = "svn://danse.us/sans/releases/sansinvariant-%s" % INVARIANT
INV_VIEW_URL = "svn://danse.us/sans/releases/invariantview-%s" % INV_VIEW

def check_system():
    """
        Checks that the system has the necessary modules.
    """
    try:
        import wx
    except:
        logging.error("wxpython missing")

    try:
        import matplotlib
    except:
        logging.error("matplotlib missing")

    try:
        import numpy
    except:
        logging.error("numpy missing")

    try:
        import scipy
    except:
        logging.error("scipy missing")

    try:
        import periodictable
    except:
        logging.error("periodictable missing")

    if os.system("gcc -dumpversion")==1:
        logging.error("missing mingw")

def install_pkg(install_dir, setup_dir, url):
    """
        Check a package out and install it

        @param install_dir: directory to put the code in
        @param setup_dir: relative location of the setup.py script
        @param url: URL of the SVN repo
    """
    print "PYTHON, LIB_FOLDER=====",PYTHON, LIB_FOLDER
    logging.info("Installing %s" % url)
    try:
        if not os.path.isdir(install_dir):
            os.mkdir(install_dir)
        os.chdir(install_dir)
        os.system("%s checkout -q %s" % (SVN, url))
        os.chdir(setup_dir)
        if sys.platform == 'win32':
            os.system("%s setup.py -q build -cmingw32" % PYTHON)
            os.system("%s setup.py -q install --root \"%s\"" % (PYTHON, LIB_FOLDER))
        else:
            os.system("%s setup.py build" % PYTHON)
            os.system("%s setup.py install --prefix ~/.local" % PYTHON)
    except:
        logging.error("Install failed for %s" % url)
        logging.error(sys.exc_value)
        raw_input("Press enter to continue\n")
        sys.exit()

def checkout(release=False):
    """
        Check the SansView code out
    """
    wd = os.getcwd()

    os.chdir(wd)
    if release:
        install_pkg(".", "sansdataloader-%s" % DATALOADER, DATALOADER_URL)
    else:
        install_pkg(".", "sansdataloader", "svn://danse.us/sans/trunk/sansdataloader")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansmodels-%s" % SANSMODELS, SANSMODELS_URL)
    else:
        install_pkg(".", "sansmodels", "svn://danse.us/sans/trunk/sansmodels")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansguiframe-%s" % GUIFRAME, GUIFRAME_URL)
    else:
        install_pkg(".", "sansguiframe", "svn://danse.us/sans/trunk/sansguiframe")

    os.chdir(wd)
    if release:
        install_pkg(".", "plottools-%s" % PLOTTOOLS, PLOTTOOLS_URL)
    else:
        install_pkg(".", "plottools", "svn://danse.us/common/plottools")

    os.chdir(wd)
    if release:
        install_pkg(".", "util-%s" % UTIL, UTIL_URL)
    else:
        install_pkg(".", "util", "svn://danse.us/common/util")

    os.chdir(wd)
    if release:
        install_pkg(".", "park_integration-%s" % PARK_INTEG, PARK_INTEG_URL)
    else:
        install_pkg(".", "park_integration", "svn://danse.us/sans/trunk/park_integration")

    os.chdir(wd)
    if release:
        install_pkg(".", "inversionview-%s" % PRVIEW, PRVIEW_URL)
    else:
        install_pkg(".", "inversionview", "svn://danse.us/sans/trunk/inversionview")

    os.chdir(wd)
    if release:
        install_pkg(".", "pr_inversion-%s" % PR_INV, PR_INV_URL)
    else:
        install_pkg(".", "pr_inversion", "svn://danse.us/sans/trunk/pr_inversion")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansinvariant-%s" % INVARIANT, INV_URL)
    else:
        install_pkg(".", "sansinvariant", "svn://danse.us/sans/trunk/sansinvariant")

    os.chdir(wd)
    if release:
        install_pkg(".", "invariantview-%s" % INV_VIEW, INV_VIEW_URL)
    else:
        install_pkg(".", "invariantview", "svn://danse.us/sans/trunk/invariantview")

    os.chdir(wd)
    if release:
        install_pkg(".", "calculatorview-%s" % CALC_VIEW, CALC_VIEW_URL)
    else:
        install_pkg(".", "calculatorview", "svn://danse.us/sans/trunk/calculatorview")

    os.chdir(wd)
    if release:
        install_pkg(".", "sanscalculator-%s" % CALCULATOR, CALC_URL)
    else:
        install_pkg(".", "sanscalculator", "svn://danse.us/sans/trunk/sanscalculator")

    os.chdir(wd)
    if release:
        install_pkg(".", "park-%s" % PARK, PARK_URL)
    else:
        install_pkg(".", "park-1.2", "svn://danse.us/park/branches/park-1.2")

    os.chdir(wd)
    if release:
        install_pkg(".", "fittingview-%s" % FIT_VIEW, FIT_URL)
    else:
        install_pkg(".", "fittingview", "svn://danse.us/sans/trunk/fittingview")

    os.chdir(wd)
    if release:
        os.system("%s checkout -q %s" % (SVN, SANSVIEW_URL))
    else:
        os.system("%s checkout -q svn://danse.us/sans/trunk/sansview" % SVN)

def prepare(wipeout = False):
    """
        Prepare the build

        @param wipeout: If True, the DANSE modules in the standard site-packages will be
        removed to avoid conflicts.
    """
    # Remove existing libraries
    if wipeout == True:
        logging.info("Deleting DANSES modules in site-packages")
        from distutils.sysconfig import get_python_lib
        libdir = get_python_lib()
        old_dirs = [os.path.join(libdir, 'danse'),
                    os.path.join(libdir, 'data_util'),
                    os.path.join(libdir, 'park'),
                    os.path.join(libdir, 'sans'),
                    ]
        for d in old_dirs:
            if os.path.isdir(d):
                shutil.rmtree(d)

    # Create a fresh installation folder
    if os.path.isdir(INSTALL_FOLDER):
        shutil.rmtree(INSTALL_FOLDER)

    os.mkdir(INSTALL_FOLDER)

    # Check that the dependencies are properly installed
    check_system()

    # Move to the installation folder
    os.chdir(INSTALL_FOLDER)

def warning():
    """
        The build script will wipe out part of the site-packages.
        Ask the user whether he wants to proceed.
    """
    print "WARNING!\n"
    print "In order to build a clean version of SansView, this script"
    print "deletes anything found under site-packages for the following"
    print "modules:"
    print "   - danse"
    print "   - data_util"
    print "   - DataLoader"
    print "   - park"
    print "   - sans"
    print "   - sans_extension\n"
    answer = raw_input("Do you want to delete those modules [Y] or continue with a dirty installation [N]? [Y|N]")
    return answer.upper()=="Y"

if __name__ == "__main__":
    print "Build script for SansView %s" % SANSVIEW

    if len(sys.argv)==1:
        # If there is no argument, build the installer
        sys.argv.append("-i")

    if len(sys.argv)>1:
        # Help
        if sys.argv[1]=="-h":
            print "Usage:"
            print "    python build_sansview [command]\n"
            print "[command] can be any of the following:"
            print "    -h: lists the command line options"
            print "    -r: Builds a SansView using the released modules"
            print "    -t: Builds SansView from the trunk"
            print "    -i: Builds an installer from the release version [Windows only]"
            print "    -n: Print out the dependencies for the release notes"
        elif sys.argv[1]=="-n":
            # Print out release URLs
            print SANSMODELS_URL
            print DATALOADER_URL
            print GUIFRAME_URL
            print PLOTTOOLS_URL
            print UTIL_URL
            print SANSVIEW_URL
            print PARK_INTEG_URL
            print PARK_URL
            print PRVIEW
            print PR_INV
            print FIT_URL
        else:
            logging.info("Build script for SansView %s" % SANSVIEW)

            # Prepare installation folder
            prepare(warning())

            # Check the command line argument
            if sys.argv[1]=="-t":
                logging.info("Building trunk version")
                checkout()
            elif sys.argv[1]=="-r":
                logging.info("Building release version")
                checkout(True)
            elif sys.argv[1]=="-i":
                logging.info("Building release version")
                checkout(True)
                if sys.platform=='win32':
                    logging.info("Building Windows installer from release version")
                    os.chdir("sansview-%s" % (SANSVIEW))
                    os.system("%s setup_exe.py py2exe --extrapath \"%s\python25\lib\site-packages\"" % (PYTHON, LIB_FOLDER))
                    os.system("%s/Q installer.iss" % INNO)
                    shutil.copy2(os.path.join("Output","setupSansView.exe"),
                                 os.path.join(CWD, "setupSansView_%s.exe" % str(timestamp)))
                elif sys.platform=='darwin':
                    logging.info("Building Mac application from release version")
                    os.chdir("sansview-%s" % (SANSVIEW))
                    os.system("%s setup_mac.py py2app" % PYTHON)

    raw_input("Press enter to continue\n")

<commit_msg>removed a print statement<commit_after>#
# Script to get source from SVN and build SansView
#
# Read the release notes to make ensure that the required software is installed.
#
# SVN must be installed:
# http://subversion.tigris.org/servlets/ProjectDocumentList?folderID=91
#
#
# On Windows:
#   - make sure svn.exe in on the path. You might need to log out and log back in again after installing SVN.
#
#   - Inno Setup must be installed
#
#   - py2exe must be installed
#
#   - mingw must be installed
#
# On Mac:
#   - py2app must be installed
#   - macholib must be installed to use py2app
#   - modulegraph must be installed to use py2app
#
# Usage:
# python build_sansview [command]
# [command] can be any of the following:
#   -h: lists the command line options
#   -r: Builds a SansView using the released modules.
#   -t: Builds SansView from the trubuild_sank.
#   -i: Builds a Windows installer from the release version.
#   -n: Print out the dependencies for the release notes

import os
import sys
import shutil
import logging

# Installation folder
import time
timestamp = int(time.time())
CWD    = os.getcwd()
INSTALL_FOLDER = "install_%s" % str(timestamp)

# On Windows, the python executable is not always on the path.
# Use its most frequent location as the default.
if sys.platform == 'win32':
    PYTHON = "c:\python25\python"
    LIB_FOLDER = "%s/%s" % (CWD, INSTALL_FOLDER)
else:
    PYTHON = 'python'

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s',
                    filename='build_%s.log' % str(timestamp),
                    filemode='w')

SVN    = "svn"
INNO   = "\"c:\Program Files\Inno Setup 5\ISCC\""

# Release version 1.9
SANSMODELS = "0.9.1"
DATALOADER = "0.9"
GUIFRAME   = "0.9.1"
SANSVIEW   = "1.9.1"
PLOTTOOLS  = "0.9.1"
UTIL       = "0.9.1"
PARK       = "1.2.1"
PARK_INTEG = "0.9.1"
PRVIEW     = "0.9.1"
PR_INV     = "0.9"
CALCULATOR = "0.9.1"
CALC_VIEW  = "0.9"
INVARIANT  = "0.9.1"
INV_VIEW   = "0.9"
FIT_VIEW   = "0.9.1"

# URLs for SVN repos
SANSMODELS_URL = "svn://danse.us/sans/releases/sansmodels-%s" % SANSMODELS
DATALOADER_URL = "svn://danse.us/sans/releases/sansdataloader-%s" % DATALOADER
GUIFRAME_URL = "svn://danse.us/sans/releases/sansguiframe-%s" % GUIFRAME
PLOTTOOLS_URL = "svn://danse.us/common/releases/plottools-%s" % PLOTTOOLS
UTIL_URL = "svn://danse.us/common/releases/util-%s" % UTIL
SANSVIEW_URL = "svn://danse.us/sans/releases/sansview-%s" % SANSVIEW
FIT_URL = "svn://danse.us/sans/releases/fittingview-%s" % FIT_VIEW
PARK_INTEG_URL = "svn://danse.us/sans/releases/park_integration-%s" % PARK_INTEG
PARK_URL = "svn://danse.us/park/releases/park-%s" % PARK
PRVIEW_URL = "svn://danse.us/sans/releases/inversionview-%s" % PRVIEW
PR_INV_URL = "svn://danse.us/sans/releases/pr_inversion-%s" % PR_INV
CALC_URL = "svn://danse.us/sans/releases/sanscalculator-%s" % CALCULATOR
CALC_VIEW_URL = "svn://danse.us/sans/releases/calculatorview-%s" % CALC_VIEW
INV_URL = "svn://danse.us/sans/releases/sansinvariant-%s" % INVARIANT
INV_VIEW_URL = "svn://danse.us/sans/releases/invariantview-%s" % INV_VIEW

def check_system():
    """
        Checks that the system has the necessary modules.
    """
    try:
        import wx
    except:
        logging.error("wxpython missing")

    try:
        import matplotlib
    except:
        logging.error("matplotlib missing")

    try:
        import numpy
    except:
        logging.error("numpy missing")

    try:
        import scipy
    except:
        logging.error("scipy missing")

    try:
        import periodictable
    except:
        logging.error("periodictable missing")

    if os.system("gcc -dumpversion")==1:
        logging.error("missing mingw")

def install_pkg(install_dir, setup_dir, url):
    """
        Check a package out and install it

        @param install_dir: directory to put the code in
        @param setup_dir: relative location of the setup.py script
        @param url: URL of the SVN repo
    """
    print "PYTHON, LIB_FOLDER=====",PYTHON, LIB_FOLDER
    logging.info("Installing %s" % url)
    try:
        if not os.path.isdir(install_dir):
            os.mkdir(install_dir)
        os.chdir(install_dir)
        os.system("%s checkout -q %s" % (SVN, url))
        os.chdir(setup_dir)
        if sys.platform == 'win32':
            os.system("%s setup.py -q build -cmingw32" % PYTHON)
            os.system("%s setup.py -q install --root \"%s\"" % (PYTHON, LIB_FOLDER))
        else:
            os.system("%s setup.py build" % PYTHON)
            os.system("%s setup.py install --prefix ~/.local" % PYTHON)
    except:
        logging.error("Install failed for %s" % url)
        logging.error(sys.exc_value)
        raw_input("Press enter to continue\n")
        sys.exit()

def checkout(release=False):
    """
        Check the SansView code out
    """
    wd = os.getcwd()

    os.chdir(wd)
    if release:
        install_pkg(".", "sansdataloader-%s" % DATALOADER, DATALOADER_URL)
    else:
        install_pkg(".", "sansdataloader", "svn://danse.us/sans/trunk/sansdataloader")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansmodels-%s" % SANSMODELS, SANSMODELS_URL)
    else:
        install_pkg(".", "sansmodels", "svn://danse.us/sans/trunk/sansmodels")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansguiframe-%s" % GUIFRAME, GUIFRAME_URL)
    else:
        install_pkg(".", "sansguiframe", "svn://danse.us/sans/trunk/sansguiframe")

    os.chdir(wd)
    if release:
        install_pkg(".", "plottools-%s" % PLOTTOOLS, PLOTTOOLS_URL)
    else:
        install_pkg(".", "plottools", "svn://danse.us/common/plottools")

    os.chdir(wd)
    if release:
        install_pkg(".", "util-%s" % UTIL, UTIL_URL)
    else:
        install_pkg(".", "util", "svn://danse.us/common/util")

    os.chdir(wd)
    if release:
        install_pkg(".", "park_integration-%s" % PARK_INTEG, PARK_INTEG_URL)
    else:
        install_pkg(".", "park_integration", "svn://danse.us/sans/trunk/park_integration")

    os.chdir(wd)
    if release:
        install_pkg(".", "inversionview-%s" % PRVIEW, PRVIEW_URL)
    else:
        install_pkg(".", "inversionview", "svn://danse.us/sans/trunk/inversionview")

    os.chdir(wd)
    if release:
        install_pkg(".", "pr_inversion-%s" % PR_INV, PR_INV_URL)
    else:
        install_pkg(".", "pr_inversion", "svn://danse.us/sans/trunk/pr_inversion")

    os.chdir(wd)
    if release:
        install_pkg(".", "sansinvariant-%s" % INVARIANT, INV_URL)
    else:
        install_pkg(".", "sansinvariant", "svn://danse.us/sans/trunk/sansinvariant")

    os.chdir(wd)
    if release:
        install_pkg(".", "invariantview-%s" % INV_VIEW, INV_VIEW_URL)
    else:
        install_pkg(".", "invariantview", "svn://danse.us/sans/trunk/invariantview")

    os.chdir(wd)
    if release:
        install_pkg(".", "calculatorview-%s" % CALC_VIEW, CALC_VIEW_URL)
    else:
        install_pkg(".", "calculatorview", "svn://danse.us/sans/trunk/calculatorview")

    os.chdir(wd)
    if release:
        install_pkg(".", "sanscalculator-%s" % CALCULATOR, CALC_URL)
    else:
        install_pkg(".", "sanscalculator", "svn://danse.us/sans/trunk/sanscalculator")

    os.chdir(wd)
    if release:
        install_pkg(".", "park-%s" % PARK, PARK_URL)
    else:
        install_pkg(".", "park-1.2", "svn://danse.us/park/branches/park-1.2")

    os.chdir(wd)
    if release:
        install_pkg(".", "fittingview-%s" % FIT_VIEW, FIT_URL)
    else:
        install_pkg(".", "fittingview", "svn://danse.us/sans/trunk/fittingview")

    os.chdir(wd)
    if release:
        os.system("%s checkout -q %s" % (SVN, SANSVIEW_URL))
    else:
        os.system("%s checkout -q svn://danse.us/sans/trunk/sansview" % SVN)

def prepare(wipeout = False):
    """
        Prepare the build

        @param wipeout: If True, the DANSE modules in the standard site-packages will be
        removed to avoid conflicts.
    """
    # Remove existing libraries
    if wipeout == True:
        logging.info("Deleting DANSES modules in site-packages")
        from distutils.sysconfig import get_python_lib
        libdir = get_python_lib()
        old_dirs = [os.path.join(libdir, 'danse'),
                    os.path.join(libdir, 'data_util'),
                    os.path.join(libdir, 'park'),
                    os.path.join(libdir, 'sans'),
                    ]
        for d in old_dirs:
            if os.path.isdir(d):
                shutil.rmtree(d)

    # Create a fresh installation folder
    if os.path.isdir(INSTALL_FOLDER):
        shutil.rmtree(INSTALL_FOLDER)

    os.mkdir(INSTALL_FOLDER)

    # Check that the dependencies are properly installed
    check_system()

    # Move to the installation folder
    os.chdir(INSTALL_FOLDER)

def warning():
    """
        The build script will wipe out part of the site-packages.
        Ask the user whether he wants to proceed.
    """
    print "WARNING!\n"
    print "In order to build a clean version of SansView, this script"
    print "deletes anything found under site-packages for the following"
    print "modules:"
    print "   - danse"
    print "   - data_util"
    print "   - DataLoader"
    print "   - park"
    print "   - sans"
    print "   - sans_extension\n"
    answer = raw_input("Do you want to delete those modules [Y] or continue with a dirty installation [N]? [Y|N]")
    return answer.upper()=="Y"

if __name__ == "__main__":

    if len(sys.argv)==1:
        # If there is no argument, build the installer
        sys.argv.append("-i")

    if len(sys.argv)>1:
        # Help
        if sys.argv[1]=="-h":
            print "Usage:"
            print "    python build_sansview [command]\n"
            print "[command] can be any of the following:"
            print "    -h: lists the command line options"
            print "    -r: Builds a SansView using the released modules"
            print "    -t: Builds SansView from the trunk"
            print "    -i: Builds an installer from the release version [Windows only]"
            print "    -n: Print out the dependencies for the release notes"
        elif sys.argv[1]=="-n":
            # Print out release URLs
            print SANSMODELS_URL
            print DATALOADER_URL
            print GUIFRAME_URL
            print PLOTTOOLS_URL
            print UTIL_URL
            print SANSVIEW_URL
            print PARK_INTEG_URL
            print PARK_URL
            print PRVIEW
            print PR_INV
            print FIT_URL
        else:
            logging.info("Build script for SansView %s" % SANSVIEW)

            # Prepare installation folder
            prepare(warning())

            # Check the command line argument
            if sys.argv[1]=="-t":
                logging.info("Building trunk version")
                checkout()
            elif sys.argv[1]=="-r":
                logging.info("Building release version")
                checkout(True)
            elif sys.argv[1]=="-i":
                logging.info("Building release version")
                checkout(True)
                if sys.platform=='win32':
                    logging.info("Building Windows installer from release version")
                    os.chdir("sansview-%s" % (SANSVIEW))
                    os.system("%s setup_exe.py py2exe --extrapath \"%s\python25\lib\site-packages\"" % (PYTHON, LIB_FOLDER))
                    os.system("%s/Q installer.iss" % INNO)
                    shutil.copy2(os.path.join("Output","setupSansView.exe"),
                                 os.path.join(CWD, "setupSansView_%s.exe" % str(timestamp)))
                elif sys.platform=='darwin':
                    logging.info("Building Mac application from release version")
                    os.chdir("sansview-%s" % (SANSVIEW))
                    os.system("%s setup_mac.py py2app" % PYTHON)

    raw_input("Press enter to continue\n")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    sartoris.testsuite
    ~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: (c) 2013 by Wikimedia Foundation.
    :license: BSD, see LICENSE for more details.
"""

import unittest

class TestNullHandler(unittest.TestCase):
    def test_emit(self):
        # null_handler = NullHandler()
        # self.assertEqual(expected, null_handler.emit(record))
        assert False # TODO: implement your test here

class TestParseargs(unittest.TestCase):
    def test_parseargs(self):
        # self.assertEqual(expected, parseargs(argv))
        assert False # TODO: implement your test here

class TestSartoris(unittest.TestCase):
    def test___init__(self):
        # sartoris = Sartoris(*args, **kwargs)
        assert False # TODO: implement your test here

    def test___new__(self):
        # sartoris = Sartoris(*args, **kwargs)
        assert False # TODO: implement your test here

    def test_abort(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.abort(args))
        assert False # TODO: implement your test here

    def test_diff(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.diff(args))
        assert False # TODO: implement your test here

    def test_log_deploys(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.log_deploys(args))
        assert False # TODO: implement your test here

    def test_resync(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.resync(args))
        assert False # TODO: implement your test here

    def test_revert(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.revert(args))
        assert False # TODO: implement your test here

    def test_show_tag(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.show_tag(args))
        assert False # TODO: implement your test here

    def test_start(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.start(args))
        assert False # TODO: implement your test here

    def test_sync(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.sync(args, no_deps, force))
        assert False # TODO: implement your test here

class TestMain(unittest.TestCase):
    def test_main(self):
        # self.assertEqual(expected, main(argv, out, err))
        assert False # TODO: implement your test here

class TestCli(unittest.TestCase):
    def test_cli(self):
        # self.assertEqual(expected, cli())
        assert False # TODO: implement your test here

if __name__ == '__main__':
    unittest.main()
<commit_msg>Conform to PEP 8<commit_after># -*- coding: utf-8 -*-
"""
    sartoris.testsuite
    ~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: (c) 2013 by Wikimedia Foundation.
    :license: BSD, see LICENSE for more details.
"""

import unittest

class TestNullHandler(unittest.TestCase):
    def test_emit(self):
        # null_handler = NullHandler()
        # self.assertEqual(expected, null_handler.emit(record))
        assert False  # TODO: implement your test here

class TestParseargs(unittest.TestCase):
    def test_parseargs(self):
        # self.assertEqual(expected, parseargs(argv))
        assert False  # TODO: implement your test here

class TestSartoris(unittest.TestCase):
    def test___init__(self):
        # sartoris = Sartoris(*args, **kwargs)
        assert False  # TODO: implement your test here

    def test___new__(self):
        # sartoris = Sartoris(*args, **kwargs)
        assert False  # TODO: implement your test here

    def test_abort(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.abort(args))
        assert False  # TODO: implement your test here

    def test_diff(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.diff(args))
        assert False  # TODO: implement your test here

    def test_log_deploys(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.log_deploys(args))
        assert False  # TODO: implement your test here

    def test_resync(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.resync(args))
        assert False  # TODO: implement your test here

    def test_revert(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.revert(args))
        assert False  # TODO: implement your test here

    def test_show_tag(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.show_tag(args))
        assert False  # TODO: implement your test here

    def test_start(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.start(args))
        assert False  # TODO: implement your test here

    def test_sync(self):
        # sartoris = Sartoris(*args, **kwargs)
        # self.assertEqual(expected, sartoris.sync(args, no_deps, force))
        assert False  # TODO: implement your test here

class TestMain(unittest.TestCase):
    def test_main(self):
        # self.assertEqual(expected, main(argv, out, err))
        assert False  # TODO: implement your test here

class TestCli(unittest.TestCase):
    def test_cli(self):
        # self.assertEqual(expected, cli())
        assert False  # TODO: implement your test here

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from django.conf import settings
from django.utils.importlib import import_module

_partitioners_queue = None
_delivery_providers_queue = None

def partition(cart):
    groups = []
    remaining_items = list(cart.items.all())
    for handler in _partitioners_queue:
        handled_groups, remaining_items = handler.partition(cart,
                                                            remaining_items)
        groups += handled_groups
    if remaining_items:
        raise ImproperlyConfigured('Unhandled items remaining in cart.')
    return groups

def get_delivery_types(delivery_group):
    types = []
    for provider_path, provider in _delivery_providers_queue:
        prov_types = provider.enum_types(delivery_group=delivery_group)
        types.extend([('%s:%s' % (provider_path, t[0]), t[1]) for t in prov_types])
    return types

def get_delivery_formclass(delivery_group, typ):
    provider_path, typ_short = typ.split(':', 1)
    for prov_path, provider in _delivery_providers_queue:
        if provider_path == prov_path:
            return provider.get_formclass(delivery_group, typ_short)
    raise ValueError('No provider found for delivery type %s.' % typ)

def get_delivery_variant(delivery_group, typ, form):
    provider_path, typ_short = typ.split(':', 1)
    for prov_path, provider in _delivery_providers_queue:
        if provider_path == prov_path:
            return provider.get_variant(delivery_group, typ_short, form)
    raise ValueError('No provider found for delivery type %s.' % typ)

def init_queues():
    global _partitioners_queue
    _partitioners_queue = []
    handlers = getattr(settings, 'SATCHLESS_ORDER_PARTITIONERS', [
        'satchless.contrib.order.partitioner.simple',
    ])
    for handler in handlers:
        if isinstance(handler, str):
            mod_name, han_name = handler.rsplit('.', 1)
            module = import_module(mod_name)
            handler = getattr(module, han_name)
        _partitioners_queue.append(handler)
    global _delivery_providers_queue
    _delivery_providers_queue = []
    providers = getattr(settings, 'SATCHLESS_DELIVERY_PROVIDERS', [
        'satchless.delivery.DeliveryProvider',
    ])
    for provider_path in providers:
        mod_name, prov_name = provider_path.rsplit('.', 1)
        module = import_module(mod_name)
        provider = getattr(module, prov_name)()
        _delivery_providers_queue.append((provider_path, provider))

init_queues()
<commit_msg>Init payment providers queue<commit_after>from django.conf import settings
from django.utils.importlib import import_module

_partitioners_queue = None
_delivery_providers_queue = None
_payment_providers_queue = None

def partition(cart):
    groups = []
    remaining_items = list(cart.items.all())
    for handler in _partitioners_queue:
        handled_groups, remaining_items = handler.partition(cart,
                                                            remaining_items)
        groups += handled_groups
    if remaining_items:
        raise ImproperlyConfigured('Unhandled items remaining in cart.')
    return groups

def get_delivery_types(delivery_group):
    types = []
    for provider_path, provider in _delivery_providers_queue:
        prov_types = provider.enum_types(delivery_group=delivery_group)
        types.extend([('%s:%s' % (provider_path, t[0]), t[1]) for t in prov_types])
    return types

def get_delivery_formclass(delivery_group, typ):
    provider_path, typ_short = typ.split(':', 1)
    for prov_path, provider in _delivery_providers_queue:
        if provider_path == prov_path:
            return provider.get_formclass(delivery_group, typ_short)
    raise ValueError('No provider found for delivery type %s.' % typ)

def get_delivery_variant(delivery_group, typ, form):
    provider_path, typ_short = typ.split(':', 1)
    for prov_path, provider in _delivery_providers_queue:
        if provider_path == prov_path:
            return provider.get_variant(delivery_group, typ_short, form)
    raise ValueError('No provider found for delivery type %s.' % typ)

def init_queues():
    global _partitioners_queue
    _partitioners_queue = []
    handlers = getattr(settings, 'SATCHLESS_ORDER_PARTITIONERS', [
        'satchless.contrib.order.partitioner.simple',
    ])
    for handler in handlers:
        if isinstance(handler, str):
            mod_name, han_name = handler.rsplit('.', 1)
            module = import_module(mod_name)
            handler = getattr(module, han_name)
        _partitioners_queue.append(handler)
    global _delivery_providers_queue
    _delivery_providers_queue = []
    providers = getattr(settings, 'SATCHLESS_DELIVERY_PROVIDERS', [
        'satchless.delivery.DeliveryProvider',
    ])
    for provider_path in providers:
        mod_name, prov_name = provider_path.rsplit('.', 1)
        module = import_module(mod_name)
        provider = getattr(module, prov_name)()
        _delivery_providers_queue.append((provider_path, provider))
    global _payment_providers_queue
    _payment_providers_queue = []
    providers = getattr(settings, 'SATCHLESS_PAYMENT_PROVIDERS', [
        'satchless.payment.PaymentProvider',
    ])
    for provider_path in providers:
        mod_name, prov_name = provider_path.rsplit('.', 1)
        module = import_module(mod_name)
        provider = getattr(module, prov_name)()
        _payment_providers_queue.append((provider_path, provider))

init_queues()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import gettext
import os
import sys

import eventlet
from eventlet import wsgi

from oslo.config import cfg

# If ../savanna/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                                os.pardir,
                                                os.pardir))
if os.path.exists(os.path.join(possible_topdir,
                               'savanna',
                               '__init__.py')):
    sys.path.insert(0, possible_topdir)

gettext.install('savanna', unicode=1)

from savanna import config
from savanna.db import api as db_api
import savanna.main as server
from savanna.openstack.common import log as logging

LOG = logging.getLogger(__name__)

def main():
    dev_conf = os.path.join(possible_topdir,
                            'etc',
                            'savanna',
                            'savanna.conf')
    config_files = None
    if os.path.exists(dev_conf):
        config_files = [dev_conf]

    config.parse_configs(sys.argv[1:], config_files)
    logging.setup("savanna")

    if not db_api.setup_db():
        raise RuntimeError('Failed to create database!')

    app = server.make_app()

    wsgi.server(eventlet.listen((cfg.CONF.host, cfg.CONF.port), backlog=500),
                app)
<commit_msg>Setup logging for wsgi server<commit_after>#!/usr/bin/env python

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import gettext
import os
import sys

import eventlet
from eventlet import wsgi

from oslo.config import cfg

# If ../savanna/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
                                                os.pardir,
                                                os.pardir))
if os.path.exists(os.path.join(possible_topdir,
                               'savanna',
                               '__init__.py')):
    sys.path.insert(0, possible_topdir)

gettext.install('savanna', unicode=1)

from savanna import config
from savanna.db import api as db_api
import savanna.main as server
from savanna.openstack.common import log as logging

LOG = logging.getLogger(__name__)

def main():
    dev_conf = os.path.join(possible_topdir,
                            'etc',
                            'savanna',
                            'savanna.conf')
    config_files = None
    if os.path.exists(dev_conf):
        config_files = [dev_conf]

    config.parse_configs(sys.argv[1:], config_files)
    logging.setup("savanna")

    if not db_api.setup_db():
        raise RuntimeError('Failed to create database!')

    app = server.make_app()

    wsgi.server(eventlet.listen((cfg.CONF.host, cfg.CONF.port), backlog=500),
                app, log=logging.WritableLogger(LOG), debug=False)
<|endoftext|>


<commit_before>from django.contrib import admin

from scanning.models import PendingScan, Document, DocumentPage, Scan, ScanPage, Transcription

class ScanPageInline(admin.TabularInline):
    model = ScanPage
class ScanAdmin(admin.ModelAdmin):
    model = Scan
    inlines = [ScanPageInline]
admin.site.register(Scan, ScanAdmin)

class PendingScanAdmin(admin.ModelAdmin):
    model = PendingScan
    list_display = ('author', 'editor', 'code', 'created', 'completed')
    search_fields = ('code',)
admin.site.register(PendingScan, PendingScanAdmin)

class DocumentAdmin(admin.ModelAdmin):
    list_display = ['title', 'author', 'status', 'created']
    search_fields = ['title', 'author__profile__display_name',
                     'body', 'transcription__revisions__body']
    date_hierarchy = 'created'
    list_filter = ['type', 'status', 'author__profile__managed',
                   'author__profile__display_name']
admin.site.register(Document, DocumentAdmin)

admin.site.register(DocumentPage)
admin.site.register(Transcription)
<commit_msg>Better author filter<commit_after>from django.contrib import admin

from scanning.models import PendingScan, Document, DocumentPage, Scan, ScanPage, Transcription

class ScanPageInline(admin.TabularInline):
    model = ScanPage
class ScanAdmin(admin.ModelAdmin):
    model = Scan
    inlines = [ScanPageInline]
admin.site.register(Scan, ScanAdmin)

class PendingScanAdmin(admin.ModelAdmin):
    model = PendingScan
    list_display = ('author', 'editor', 'code', 'created', 'completed')
    search_fields = ('code',)
admin.site.register(PendingScan, PendingScanAdmin)

class AuthorFilter(admin.SimpleListFilter):
    title = 'Author'
    parameter_name = 'author__profile__display_name'

    def lookups(self, request, model_admin):
        qs = model_admin.queryset(request)
        return sorted(qs.order_by().values_list(
                    'author_id', 'author__profile__display_name'
                ).distinct(), key=lambda a: a[1])

    def queryset(self, request, queryset):
        return queryset.filter(author__profile__blogger=True)

class DocumentAdmin(admin.ModelAdmin):
    list_display = ['title', 'author', 'status', 'created']
    search_fields = ['title', 'author__profile__display_name',
                     'body', 'transcription__revisions__body']
    date_hierarchy = 'created'
    list_filter = ['type', 'status', 'author__profile__managed', AuthorFilter]
admin.site.register(Document, DocumentAdmin)

admin.site.register(DocumentPage)
admin.site.register(Transcription)
<|endoftext|>


<commit_before># Copyright 2021 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

from uw_gws import GWS

def email_addresses_from_group(group_name):
    email = []
    gws = GWS()

    for netid in gws.get_effective_members(group_name):
        email.append("{}.uw.edu".format(netid))

    return email
<commit_msg>email from group membership<commit_after># Copyright 2021 UW-IT, University of Washington
# SPDX-License-Identifier: Apache-2.0

from uw_gws import GWS

def email_addresses_from_group(group_name):
    email = []
    gws = GWS()

    for member in gws.get_effective_members(group_name):
        if member.is_uwnetid():
            email.append("{}@uw.edu".format(member.name))

    return email
<|endoftext|>


<commit_before>from __future__ import division

__author__ = 'Vladimir Iglovikov'

'''
I will try to use logistic regression from Graphlab to predict
'''

import graphlab as gl
import os
import scipy as sp
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import numpy as np
from sklearn.preprocessing import StandardScaler

def llfun(act, pred):
    epsilon = 1e-15
    pred = sp.maximum(epsilon, pred)
    pred = sp.minimum(1 - epsilon, pred)
    ll = sum(act * sp.log(pred) + sp.subtract(1, act) * sp.log(sp.subtract(1, pred)))
    ll = ll * -1.0/len(act)
    return ll

min_date = gl.SArray(['2015-05-01']).str_to_datetime()[0]

print min_date.hour

jan1 = gl.SArray(['2015-01-01']).str_to_datetime()[0]

print 'reading train'
train = gl.SFrame(os.path.join('..', 'data', 'train_ads_search'))

print
print 'train shape'
print train.shape

# print 'filling missing values'
# a = train['Price'].mean()
# train = train.fillna('Price', a)

features = ['Position',
          'HistCTR',
          # 'Price',
          # 'CategoryID',
          # 'AdID',
          # 'LocationID'
            'month',
            'day',
            'weekday',
            'days'
          ]

X = train[:10**6]

train_cut = train[:10**6]
train_cut['SearchDate'] = train_cut['SearchDate'].str_to_datetime()

train_cut['month'] = train_cut['SearchDate'].apply(lambda x: x.month)
train_cut['day'] = train_cut['SearchDate'].apply(lambda x: x.day)
train_cut['weekday'] = train_cut['SearchDate'].apply(lambda x: x.weekday())
train_cut['days'] = train_cut['SearchDate'].apply(lambda x: (x - jan1).days)

train_cut = train_cut[train_cut['SearchDate'] > min_date]

y = list(train_cut['IsClick'])

X = train_cut[features].to_dataframe()

print X.shape, len(y)

scaler = StandardScaler()
X = scaler.fit_transform(X)

random_state = 42

# Learn
clf = LogisticRegression(random_state=random_state)

skf = cross_validation.StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
scores = cross_validation.cross_val_score(clf, X, y, cv=skf, scoring='log_loss', n_jobs=-1)
print -np.mean(scores), np.std(scores)

# print 'splitting set'
# sf_train, sf_test = train.random_split(0.5, seed=42)
#
# features=['Position',
#           'HistCTR',
#           'Price',
#           # 'CategoryID',
#           # 'AdID',
#           # 'LocationID'
#           ]
#
# model = gl.logistic_classifier.create(sf_train,
#                                       target='IsClick',
#                                       features=features,
#                                       validation_set=sf_test)
#
# print 'predicting'
# prediction = model.predict(sf_test, output_type='probablility')
#
# print prediction[:30]
# print sf_test['IsClick'][:30]
# print 'calculating score'
# score = llfun(sf_test['IsClick'], prediction)
# print score<commit_msg>Now gl_lr works with trancated version of the dataset<commit_after>from __future__ import division

__author__ = 'Vladimir Iglovikov'

'''
I will try to use logistic regression from Graphlab to predict
'''

import graphlab as gl
import os
import scipy as sp
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import numpy as np
from sklearn.preprocessing import StandardScaler
import time
import pandas as pd

min_date = gl.SArray(['2015-05-10']).str_to_datetime()[0]

jan1 = gl.SArray(['2015-01-01']).str_to_datetime()[0]

print 'reading train'
train = gl.SFrame(os.path.join('..', 'data', 'train_ads_search'))[:10**5]

print
print 'train shape'
print train.shape

print 'filling missing values in Price'
a = train['Price'].mean()
train = train.fillna('Price', a)

features = ['Position',
          'HistCTR',
          'Price',
          'CategoryID',
          # 'RegionID',
          'RegionID.1',
          # 'CityID',
          'CategoryID.1',
          # 'AdID',
          # 'LocationID'
            # 'month',
            'day',
            'weekday',
            'days',
            'hour',
            'IsUserLoggedOn',
            'LocationID.1',
          ]

# train = train[:10**8]

train = train[train['SearchDate'] > min_date]

# train_cut['month'] = train_cut['SearchDate'].apply(lambda x: x.month)
train['day'] = train['SearchDate'].apply(lambda x: str(x.day))
train['weekday'] = train['SearchDate'].apply(lambda x: str(x.weekday()))
train['days'] = train['SearchDate'].apply(lambda x: (x - jan1).days)
train['hour'] = train['SearchDate'].apply(lambda x: str(x.hour))
train['CategoryID'] = train['CategoryID'].apply(lambda x: str(x))
train['CategoryID.1'] = train['CategoryID.1'].apply(lambda x: str(x))
# train['LocationID.1'] = train['LocationID.1'].apply(lambda x: str(x))
train['RegionID'] = train['RegionID'].apply(lambda x: str(x))
train['RegionID.1'] = train['RegionID.1'].apply(lambda x: str(x))
train['CityID'] = train['CityID'].apply(lambda x: str(x))
train['CityID.1'] = train['CityID.1'].apply(lambda x: str(x))

print train.shape

ind = 2
if ind == 1:
	y = list(train['IsClick'])

	X = train[features].to_dataframe()

	print X.head()
	print X.info()

	print X.shape, len(y)

	scaler = StandardScaler()
	X = scaler.fit_transform(X)

	random_state = 42

	# Learn
	clf = LogisticRegression(random_state=random_state)

	skf = cross_validation.StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
	scores = cross_validation.cross_val_score(clf, X, y, cv=skf, scoring='log_loss', n_jobs=-1)
	print -np.mean(scores), np.std(scores)
elif ind == 2:
  print 'fitting'
  model = gl.logistic_classifier.create(train,
                                        target='IsClick', features=features)
  print model.summary()

  print 'reading test'
  test = gl.SFrame(os.path.join('..', 'data', 'test_ads_search'))

  print 'adding data features'

  test['SearchDate'] = test['SearchDate'].str_to_datetime()

  test['day'] = test['SearchDate'].apply(lambda x: str(x.day))
  test['hour'] = test['SearchDate'].apply(lambda x: str(x.hour))
  test['weekday'] = test['SearchDate'].apply(lambda x: str(x.weekday()))
  test['days'] = test['SearchDate'].apply(lambda x: (x - jan1).days)
  test['CategoryID'] = test['CategoryID'].apply(lambda x: str(x))
  test['CategoryID.1'] = test['CategoryID.1'].apply(lambda x: str(x))
  # test['LocationID.1'] = test['LocationID.1'].apply(lambda x: str(x))
  test['RegionID'] = test['RegionID'].apply(lambda x: str(x))
  test['RegionID.1'] = test['RegionID.1'].apply(lambda x: str(x))

  test['CityID'] = test['CityID'].apply(lambda x: str(x))
  test['CityID.1'] = test['CityID.1'].apply(lambda x: str(x))

  test = test.fillna('Price', a)
  print test.shape
  print 'predicting'
  prediction = model.predict(test, output_type='probability')
  result = pd.DataFrame()
  result['IsClick'] = prediction
  result['ID'] = test['ID']
  result.to_csv('predictions/LR_{timestamp}.csv'.format(timestamp=time.time()), index=False)

<|endoftext|>


<commit_before># Copyright 2019 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from azure.mgmt.web.models import (Site, SiteConfig)
from c7n_azure.constants import (FUNCTION_DOCKER_VERSION, FUNCTION_EXT_VERSION)
from c7n_azure.provisioning.deployment_unit import DeploymentUnit
from c7n_azure.utils import azure_name_value_pair

class FunctionAppDeploymentUnit(DeploymentUnit):

    def __init__(self):
        super(FunctionAppDeploymentUnit, self).__init__(
            'azure.mgmt.web.WebSiteManagementClient')
        self.type = "Function Application"

    def _get(self, params):
        return self.client.web_apps.get(params['resource_group_name'], params['name'])

    def _provision(self, params):
        site_config = SiteConfig(app_settings=[])
        functionapp_def = Site(location=params['location'], site_config=site_config)

        # common function app settings
        functionapp_def.server_farm_id = params['app_service_plan_id']
        functionapp_def.reserved = True  # This implies Linux for auto-created app plans

        # consumption app plan
        if params['is_consumption_plan']:
            functionapp_def.kind = 'functionapp,linux'
        # dedicated app plan
        else:
            functionapp_def.kind = 'functionapp,linux,container'
            site_config.linux_fx_version = FUNCTION_DOCKER_VERSION
            site_config.always_on = True

        # application insights settings
        app_insights_key = params['app_insights_key']
        if app_insights_key:
            site_config.app_settings.append(
                azure_name_value_pair('APPINSIGHTS_INSTRUMENTATIONKEY', app_insights_key))

        # Don't generate pycache
        site_config.app_settings.append(
            azure_name_value_pair('PYTHONDONTWRITEBYTECODE', 1))

        # Enable server side build
        site_config.app_settings.append(
            azure_name_value_pair('ENABLE_ORYX_BUILD', 'true')
        )
        site_config.app_settings.append(
            azure_name_value_pair('SCM_DO_BUILD_DURING_DEPLOYMENT', 'true')
        )

        # general app settings
        con_string = params['storage_account_connection_string']
        site_config.app_settings.append(azure_name_value_pair('AzureWebJobsStorage', con_string))
        site_config.app_settings.append(azure_name_value_pair('FUNCTIONS_EXTENSION_VERSION',
                                                              FUNCTION_EXT_VERSION))
        site_config.app_settings.append(azure_name_value_pair('FUNCTIONS_WORKER_RUNTIME', 'python'))

        return self.client.web_apps.create_or_update(params['resource_group_name'],
                                                     params['name'],
                                                     functionapp_def).result()
<commit_msg>azure - function app policies - require client certs and configure https only (#5860)<commit_after># Copyright 2019 Microsoft Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from azure.mgmt.web.models import (Site, SiteConfig)
from c7n_azure.constants import (FUNCTION_DOCKER_VERSION, FUNCTION_EXT_VERSION)
from c7n_azure.provisioning.deployment_unit import DeploymentUnit
from c7n_azure.utils import azure_name_value_pair

class FunctionAppDeploymentUnit(DeploymentUnit):

    def __init__(self):
        super(FunctionAppDeploymentUnit, self).__init__(
            'azure.mgmt.web.WebSiteManagementClient')
        self.type = "Function Application"

    def _get(self, params):
        return self.client.web_apps.get(params['resource_group_name'], params['name'])

    def _provision(self, params):
        site_config = SiteConfig(app_settings=[])
        functionapp_def = Site(
            https_only=True,
            client_cert_enabled=True,
            location=params['location'],
            site_config=site_config)

        # common function app settings
        functionapp_def.server_farm_id = params['app_service_plan_id']
        functionapp_def.reserved = True  # This implies Linux for auto-created app plans

        # consumption app plan
        if params['is_consumption_plan']:
            functionapp_def.kind = 'functionapp,linux'
        # dedicated app plan
        else:
            functionapp_def.kind = 'functionapp,linux,container'
            site_config.linux_fx_version = FUNCTION_DOCKER_VERSION
            site_config.always_on = True

        # application insights settings
        app_insights_key = params['app_insights_key']
        if app_insights_key:
            site_config.app_settings.append(
                azure_name_value_pair('APPINSIGHTS_INSTRUMENTATIONKEY', app_insights_key))

        # Don't generate pycache
        site_config.app_settings.append(
            azure_name_value_pair('PYTHONDONTWRITEBYTECODE', 1))

        # Enable server side build
        site_config.app_settings.append(
            azure_name_value_pair('ENABLE_ORYX_BUILD', 'true')
        )
        site_config.app_settings.append(
            azure_name_value_pair('SCM_DO_BUILD_DURING_DEPLOYMENT', 'true')
        )

        # general app settings
        con_string = params['storage_account_connection_string']
        site_config.app_settings.append(azure_name_value_pair('AzureWebJobsStorage', con_string))
        site_config.app_settings.append(azure_name_value_pair('FUNCTIONS_EXTENSION_VERSION',
                                                              FUNCTION_EXT_VERSION))
        site_config.app_settings.append(azure_name_value_pair('FUNCTIONS_WORKER_RUNTIME', 'python'))

        return self.client.web_apps.create_or_update(params['resource_group_name'],
                                                     params['name'],
                                                     functionapp_def).result()
<|endoftext|>


<commit_before>'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

from unittest import TestCase
from mock.mock import patch
from resource_management import *
from resource_management.libraries.resources.execute_hadoop\
  import ExecuteHadoop

@patch.object(System, "os_family", new = 'redhat')
class TestExecuteHadoopResource(TestCase):
  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_default_args(self, execute_mock):
    '''
    Test if default arguments are passed to Execute
    '''
    with Environment() as env:
      ExecuteHadoop("command",
                    conf_dir="conf_dir",
                    user="user",
                    logoutput=True,
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': True, 'tries': 1, 'user': 'user', 'try_sleep': 0})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_unknown_conf(self, execute_mock):
    '''
    Test when UnknownConfiguration passed
    '''
    with Environment() as env:
      ExecuteHadoop("command",
                    kinit_path_local=UnknownConfiguration(name="kinit_path_local"),
                    conf_dir="conf_dir",
                    user="user",
                    keytab=UnknownConfiguration(name="keytab"),
                    security_enabled=False,
                    principal=UnknownConfiguration(name="principal")
                    )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_defined_args(self, execute_mock):
    '''
    Test if defined arguments are passed to Execute
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=2,
                    keytab="keytab",
                    security_enabled=False,
                    kinit_override=False,
                    try_sleep=2,
                    logoutput=True,
                    principal="principal"
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': True, 'tries': 2, 'user': 'user', 'try_sleep': 2})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_command_list(self, execute_mock):
    '''
    Test for "command" passed as List
    '''
    with Environment("/") as env:
      ExecuteHadoop(["command1","command2"],
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    keytab="keytab"
      )
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'hadoop --config conf_dir command1')
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command2')
      self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
                       {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})
      self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
                       {'logoutput': False, 'tries': 1, 'user': 'user', 'try_sleep': 0})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_command_tuple(self, execute_mock):
    '''
    Test for "command" passed as Tuple
    '''
    with Environment("/") as env:
      ExecuteHadoop(("command1","command2","command3"),
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    keytab="keytab"
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,
                       'hadoop --config conf_dir command1 command2 command3')

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured(self, execute_mock):
    '''
    Test security_enabled=True behaviour
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=False,
                    try_sleep=0,
                    logoutput=True
      )
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(str(execute_mock.call_args_list[0][0][0]),
                       "Execute['path -kt keytab user']")
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'path -kt keytab user')
      self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
                       {'path': ['/bin'], 'user': 'user'})
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
                       {'logoutput': True, 'tries': 1, 'user': 'user', 'try_sleep': 0})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured_kinit_override(self, execute_mock):
    '''
    Test security_enabled=True and kinit_override=True behaviour
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=True,
                    try_sleep=0,
                    logoutput=True
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'hadoop --config conf_dir command')

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured_principal(self, execute_mock):
    '''
    Test with "principal" argument
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=False,
                    try_sleep=0,
                    logoutput=True,
                    principal="principal")
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'path -kt keytab principal')
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command')<commit_msg>AMBARI-7304 Python TestExecuteHadoopResource test failures (dsen)<commit_after>'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import os

from unittest import TestCase
from mock.mock import patch
from resource_management import *
from resource_management.libraries.resources.execute_hadoop\
  import ExecuteHadoop

@patch.object(System, "os_family", new = 'redhat')
class TestExecuteHadoopResource(TestCase):
  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_default_args(self, execute_mock):
    '''
    Test if default arguments are passed to Execute
    '''
    with Environment() as env:
      ExecuteHadoop("command",
                    conf_dir="conf_dir",
                    user="user",
                    logoutput=True,
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': True,
                        'tries': 1,
                        'user': 'user',
                        'try_sleep': 0,
                        'environment': {'PATH': os.environ['PATH']}})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_unknown_conf(self, execute_mock):
    '''
    Test when UnknownConfiguration passed
    '''
    with Environment() as env:
      ExecuteHadoop("command",
                    kinit_path_local=UnknownConfiguration(name="kinit_path_local"),
                    conf_dir="conf_dir",
                    user="user",
                    keytab=UnknownConfiguration(name="keytab"),
                    security_enabled=False,
                    principal=UnknownConfiguration(name="principal")
                    )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': False,
                        'tries': 1,
                        'user': 'user',
                        'try_sleep': 0,
                        'environment': {'PATH': os.environ['PATH']}})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_defined_args(self, execute_mock):
    '''
    Test if defined arguments are passed to Execute
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=2,
                    keytab="keytab",
                    security_enabled=False,
                    kinit_override=False,
                    try_sleep=2,
                    logoutput=True,
                    principal="principal"
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args[0][0].arguments,
                       {'logoutput': True,
                        'tries': 2,
                        'user': 'user',
                        'try_sleep': 2,
                        'environment': {'PATH': os.environ['PATH']}})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_command_list(self, execute_mock):
    '''
    Test for "command" passed as List
    '''
    with Environment("/") as env:
      ExecuteHadoop(["command1","command2"],
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    keytab="keytab"
      )
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'hadoop --config conf_dir command1')
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command2')
      self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
                       {'logoutput': False,
                        'tries': 1,
                        'user': 'user',
                        'try_sleep': 0,
                        'environment': {'PATH': os.environ['PATH']}})
      self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
                       {'logoutput': False,
                        'tries': 1,
                        'user': 'user',
                        'try_sleep': 0,
                        'environment': {'PATH': os.environ['PATH']}})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_command_tuple(self, execute_mock):
    '''
    Test for "command" passed as Tuple
    '''
    with Environment("/") as env:
      ExecuteHadoop(("command1","command2","command3"),
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    keytab="keytab"
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args[0][0].command,
                       'hadoop --config conf_dir command1 command2 command3')

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured(self, execute_mock):
    '''
    Test security_enabled=True behaviour
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=False,
                    try_sleep=0,
                    logoutput=True
      )
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(str(execute_mock.call_args_list[0][0][0]),
                       "Execute['path -kt keytab user']")
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'path -kt keytab user')
      self.assertEqual(execute_mock.call_args_list[0][0][0].arguments,
                       {'path': ['/bin'], 'user': 'user'})
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command')
      self.assertEqual(execute_mock.call_args_list[1][0][0].arguments,
                       {'logoutput': True,
                        'tries': 1,
                        'user': 'user',
                        'try_sleep': 0,
                        'environment': {'PATH': os.environ['PATH']}})

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured_kinit_override(self, execute_mock):
    '''
    Test security_enabled=True and kinit_override=True behaviour
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=True,
                    try_sleep=0,
                    logoutput=True
      )
      self.assertEqual(execute_mock.call_count, 1)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'hadoop --config conf_dir command')

  @patch("resource_management.core.providers.system.ExecuteProvider")
  def test_run_secured_principal(self, execute_mock):
    '''
    Test with "principal" argument
    '''
    with Environment("/") as env:
      ExecuteHadoop("command",
                    action="run",
                    kinit_path_local="path",
                    conf_dir="conf_dir",
                    user="user",
                    tries=1,
                    keytab="keytab",
                    security_enabled=True,
                    kinit_override=False,
                    try_sleep=0,
                    logoutput=True,
                    principal="principal")
      self.assertEqual(execute_mock.call_count, 2)
      self.assertEqual(execute_mock.call_args_list[0][0][0].command,
                       'path -kt keytab principal')
      self.assertEqual(execute_mock.call_args_list[1][0][0].command,
                       'hadoop --config conf_dir command')<|endoftext|>