<commit_before>#!/usr/bin/env python
# -----------------------------------------------------------------
# cgreen-mocker.py
#
# Create Cgreen mocks from extern declarations of functions,
# typically in a header file.
#
# Usage:
#   cgreen-mocker.py { <cpp_directive> } <headerfile>
#
# <cpp_directive>: any 'cpp' directive but most useful is e.g.
#                  "-I <directory>" to ensure cpp finds files.
#
# <headerfile>: file with function declarations that you want
#               to mock
#
# Simplistically adapted from pycparser example: func_defs.py
#
# Since it uses pycparser it will only handle C functions and you will
# probably need the pycparsers "fake_libc_include" to avoid parsing
# the whole world of libc headers. To use it, make a soft link with
# the name 'pycparser' in the directory you are running this from, or
# in the directory of 'cgreen-mocker' itself, to the top directory of
# the pycparser source, and cgreen-mocker will pick it up
# automatically. Or you can point to it using a command line
# 'cpp_directive' arg.
#
# Thanks to @gardenia for the pointer to pycparser!
#
#    https://github.com/eliben/pycparser
#
# (C) 2016, Thomas Nilefalk
#
# Using pycparser for printing out all the functions defined in a
# C file.
#
# PyCParser - Copyright (C) 2008-2015, Eli Bendersky
# License: BSD
# -----------------------------------------------------------------
from __future__ import print_function
from pycparser.plyparser import ParseError
from pycparser import c_parser, c_ast, parse_file, c_generator
from functools import reduce
import sys
import os

# This is not required if you've installed pycparser into
# your site-packages/ with setup.py
sys.path.extend(['.', '..'])

# A visitor for FuncDef nodes that prints the
# Cgreen mock equivalent of the function
class FuncDefVisitor(c_ast.NodeVisitor):
    def __init__(self, filename):
        self._types = {}
        self.filename = filename

    def visit_FuncDecl(self, node):
        if node.coord.file == self.filename:
            # Only consider definitions that are in the processed file
            generator = c_generator.CGenerator()
            try:
                print(generator.visit(node), end="")
                print(" { ")
                self.should_return(node)
                print("mock(%s);" % ", ".join(arg_list(node.args)))
                print("}")
                print()
            except Exception as e:
                print("ERROR: {} - Unexpected AST @ {}:{}:{}:".format(e, node.coord.file,
                                                                      node.coord.line, node.coord.column))
                node.show()
                return

    def visit_Typedef(self, node):
        self._types[node.name] = {
            'is_pointer': isinstance(node.type, c_ast.PtrDecl),
        }
        if self._types[node.name]['is_pointer']:
            self._types[node.name]['to_class'] = node.type.type.type.names
        else:
            self._types[node.name]['to_class'] = None

    def should_return(self, node):
        generator = c_generator.CGenerator()
        type = node.type
        if is_double_decl(node):
            print("  return unbox_double(", end="")
        elif not is_void_decl(node):
            print("  return %s(" %
                  ("*" if self.is_return_struct_by_value(node) else ""), end="")
            print(generator.visit(node.type), end="")
            if isinstance(node.type, c_ast.PtrDecl) or self.is_return_struct_by_value(node):
                print(" *", end="")
            print(") ", end="")
        else:
            print("  ", end="")

    def is_return_struct_by_value(self, node):
        type = node.type
        return not isinstance(type, c_ast.PtrDecl) and type.type.names[0] in self._types and not self._types[type.type.names[0]]['is_pointer']

    def is_return_by_value_pointer(self, node):
        type = node.type
        return not isinstance(type, c_ast.PtrDecl) and self._types[type.type.names[0]]['is_pointer']

def arg_list(args):
    if args != None and len(args.params) > 0:
        return [el for el in map(parameter_name_or_box_double,
                                 filter(lambda x: not is_ellipsis_param(x),
                                        args.params))
                if el is not None]
    else:
        return []

def parameter_name_or_box_double(node):
    if is_double_decl(node):
        return "box_double({})".format(node.name)
    else:
        return node.name

def is_void_decl(node):
    type = node.type
    return isinstance(type, c_ast.TypeDecl) and type.type.names == ['void']

def is_double_decl(node):
    type = node.type
    return isinstance(type, c_ast.TypeDecl) and type.type.names == ['double']

def is_ellipsis_param(node):
    return isinstance(node, c_ast.EllipsisParam)

def show_func_defs(args):
    # Note that cpp is used. Provide a path to your own cpp or
    # make sure one exists in PATH.

    pycparser_path = None
    # Try to find a fake_libc
    # In current directory?
    if verbose:
        print("Looking for fake_lib in current directory...")
    if os.path.isdir('pycparser'):
        pycparser_path = r'./pycparser'
    else:
        # In the directory of this script?
        path = os.path.abspath(__file__)
        if verbose:
            print("Looking for fake_lib in directory of script ({0}...".format(path))
        if os.path.isdir(os.path.dirname(os.path.join(os.path.abspath(__file__),
                                                      'pycparser'))):
            pycparser_path = os.path.dirname(os.path.join(os.path.abspath(__file__),
                                                          'pycparser'))

    if pycparser_path:
        pycparser_lib = reduce(
            os.path.join, [pycparser_path, 'utils', 'fake_libc_include'])
        if verbose:
            print("/* Generated with cgreen-mocker and pycparser's fake_libc from %s */" % (pycparser_path))
    elif verbose:
        print("Not found")

    try:
        options = [
            '-I'+pycparser_lib ] if pycparser_path else []
        if add_gnuisms:
            # And add some common GNUisms
            options = options + [
                r'-D__gnuc_va_list(c)=',
                r'-D__attribute__(x)=',
                r'-D__extension__=',
                r'-D__restrict=',
                r'-D__inline='
            ]
        if verbose:
            print("Parsing with options = {0}".format(options))
        cpp_args=list(filter(None, options))
        ast = parse_file(args[-1], use_cpp=True,
                         cpp_args=cpp_args + args[0:-1])
    except ParseError as e:
        print("ERROR: {} - C99 parse error".format(e))
        return

    print('/* -*- c -*-*/')    # Suggest c-mode for Emacs
    print('#include "%s"' % args[len(args)-1])
    print('#include <cgreen/mocks.h>')
    print()
    v = FuncDefVisitor(args[-1])
    v.visit(ast)

def usage():
    print("""
Usage:
    cgreen-mocker.py { <cpp_directive> } <headerfile>

    <cpp_directive>: any 'cpp' directive but most useful are e.g.
                     "-I <directory>" to ensure cpp finds files and
                     "-D <define>" to create an inline define

    <headerfile>:    file with function declarations that you want
                     to mock

    Cgreen-mocker takes a header file and generates cgreen mocks for
    all functions in it. It will print the generated mocks to standard
    output so you can inspect it, or pipe it to a file that can be
    compiled and linked with your tests.

    The mocker will only handle functions that are declared in the
    header file you provide. This is based on the presumtion that the
    header file represents functions in a unit. Aggregating functions
    from multiple units into a single header for convenience is not
    supported. Also the mocker cannot handle data declarations (yet?).

    If your header does not name some arguments you will not be able
    to use those arguments in 'expect when' statements, of course.

    A tip is to ensure that the header file does not include other
    header files that have external function definitions since they
    will also be mocked, which is probably not what you want.

    If cgreen-mocker encounters parse errors and they look like
    gnu-isms you should get a copy of the source for pycparser (on
    which cgreen-mocker is built). In it you will find a
    'fake_libc_include' which help. Create a symbolic link named
    'pycparser' that links to the root of pycparser source and
    cgreen-mocker will find it itself.

    You can find pycparser at https://github.com/eliben/pycparser

""")

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        usage()
        exit(-1)
    if '-v' in sys.argv:
        verbose = True
        sys.argv.remove('-v')
    else:
        verbose = False
    if '-gnu' in sys.argv:
        add_gnuisms = True
        sys.argv.remove('-gnu')
    else:
        add_gnuisms = False
    show_func_defs(sys.argv[1:])
<commit_msg>Document cgreen-mocker only generates mocks for funcs in the file given<commit_after>#!/usr/bin/env python
# -----------------------------------------------------------------
# cgreen-mocker.py
#
# Create Cgreen mocks from extern declarations of functions,
# typically in a header file.
#
# Usage:
#   cgreen-mocker.py { <cpp_directive> } <headerfile>
#
# <cpp_directive>: any 'cpp' directive but most useful is e.g.
#                  "-I <directory>" to ensure cpp finds files.
#
# <headerfile>: file with function declarations that you want
#               to mock
#
# Simplistically adapted from pycparser example: func_defs.py
#
# Since it uses pycparser it will only handle C functions and you will
# probably need the pycparsers "fake_libc_include" to avoid parsing
# the whole world of libc headers. To use it, make a soft link with
# the name 'pycparser' in the directory you are running this from, or
# in the directory of 'cgreen-mocker' itself, to the top directory of
# the pycparser source, and cgreen-mocker will pick it up
# automatically. Or you can point to it using a command line
# 'cpp_directive' arg.
#
# Thanks to @gardenia for the pointer to pycparser!
#
#    https://github.com/eliben/pycparser
#
# (C) 2016, Thomas Nilefalk
#
# Using pycparser for printing out all the functions defined in a
# C file.
#
# PyCParser - Copyright (C) 2008-2015, Eli Bendersky
# License: BSD
# -----------------------------------------------------------------
from __future__ import print_function
from pycparser.plyparser import ParseError
from pycparser import c_parser, c_ast, parse_file, c_generator
from functools import reduce
import sys
import os

# This is not required if you've installed pycparser into
# your site-packages/ with setup.py
sys.path.extend(['.', '..'])

# A visitor for FuncDef nodes that prints the
# Cgreen mock equivalent of the function
class FuncDefVisitor(c_ast.NodeVisitor):
    def __init__(self, filename):
        self._types = {}
        self.filename = filename

    def visit_FuncDecl(self, node):
        if node.coord.file == self.filename:
            # Only consider definitions that are in the processed file
            generator = c_generator.CGenerator()
            try:
                print(generator.visit(node), end="")
                print(" { ")
                self.should_return(node)
                print("mock(%s);" % ", ".join(arg_list(node.args)))
                print("}")
                print()
            except Exception as e:
                print("ERROR: {} - Unexpected AST @ {}:{}:{}:".format(e, node.coord.file,
                                                                      node.coord.line, node.coord.column))
                node.show()
                return

    def visit_Typedef(self, node):
        self._types[node.name] = {
            'is_pointer': isinstance(node.type, c_ast.PtrDecl),
        }
        if self._types[node.name]['is_pointer']:
            self._types[node.name]['to_class'] = node.type.type.type.names
        else:
            self._types[node.name]['to_class'] = None

    def should_return(self, node):
        generator = c_generator.CGenerator()
        type = node.type
        if is_double_decl(node):
            print("  return unbox_double(", end="")
        elif not is_void_decl(node):
            print("  return %s(" %
                  ("*" if self.is_return_struct_by_value(node) else ""), end="")
            print(generator.visit(node.type), end="")
            if isinstance(node.type, c_ast.PtrDecl) or self.is_return_struct_by_value(node):
                print(" *", end="")
            print(") ", end="")
        else:
            print("  ", end="")

    def is_return_struct_by_value(self, node):
        type = node.type
        return not isinstance(type, c_ast.PtrDecl) and type.type.names[0] in self._types and not self._types[type.type.names[0]]['is_pointer']

    def is_return_by_value_pointer(self, node):
        type = node.type
        return not isinstance(type, c_ast.PtrDecl) and self._types[type.type.names[0]]['is_pointer']

def arg_list(args):
    if args != None and len(args.params) > 0:
        return [el for el in map(parameter_name_or_box_double,
                                 filter(lambda x: not is_ellipsis_param(x),
                                        args.params))
                if el is not None]
    else:
        return []

def parameter_name_or_box_double(node):
    if is_double_decl(node):
        return "box_double({})".format(node.name)
    else:
        return node.name

def is_void_decl(node):
    type = node.type
    return isinstance(type, c_ast.TypeDecl) and type.type.names == ['void']

def is_double_decl(node):
    type = node.type
    return isinstance(type, c_ast.TypeDecl) and type.type.names == ['double']

def is_ellipsis_param(node):
    return isinstance(node, c_ast.EllipsisParam)

def show_func_defs(args):
    # Note that cpp is used. Provide a path to your own cpp or
    # make sure one exists in PATH.

    pycparser_path = None
    # Try to find a fake_libc
    # In current directory?
    if verbose:
        print("Looking for fake_lib in current directory...")
    if os.path.isdir('pycparser'):
        pycparser_path = r'./pycparser'
    else:
        # In the directory of this script?
        path = os.path.abspath(__file__)
        if verbose:
            print("Looking for fake_lib in directory of script ({0}...".format(path))
        if os.path.isdir(os.path.dirname(os.path.join(os.path.abspath(__file__),
                                                      'pycparser'))):
            pycparser_path = os.path.dirname(os.path.join(os.path.abspath(__file__),
                                                          'pycparser'))

    if pycparser_path:
        pycparser_lib = reduce(
            os.path.join, [pycparser_path, 'utils', 'fake_libc_include'])
        if verbose:
            print("/* Generated with cgreen-mocker and pycparser's fake_libc from %s */" % (pycparser_path))
    elif verbose:
        print("Not found")

    try:
        options = [
            '-I'+pycparser_lib ] if pycparser_path else []
        if add_gnuisms:
            # And add some common GNUisms
            options = options + [
                r'-D__gnuc_va_list(c)=',
                r'-D__attribute__(x)=',
                r'-D__extension__=',
                r'-D__restrict=',
                r'-D__inline='
            ]
        if verbose:
            print("Parsing with options = {0}".format(options))
        cpp_args=list(filter(None, options))
        ast = parse_file(args[-1], use_cpp=True,
                         cpp_args=cpp_args + args[0:-1])
    except ParseError as e:
        print("ERROR: {} - C99 parse error".format(e))
        return

    print('/* -*- c -*-*/')    # Suggest c-mode for Emacs
    print('#include "%s"' % args[len(args)-1])
    print('#include <cgreen/mocks.h>')
    print()
    v = FuncDefVisitor(args[-1])
    v.visit(ast)

def usage():
    print("""
Usage:
    cgreen-mocker.py { <cpp_directive> } <headerfile>

    <cpp_directive>: any 'cpp' directive but most useful are e.g.
                     "-I <directory>" to ensure cpp finds files and
                     "-D <define>" to create an inline define

    <headerfile>:    file with function declarations that you want
                     to mock

    Cgreen-mocker takes a header file and generates cgreen mocks for
    all functions in it. It will print the generated mocks to standard
    output so you can inspect it, or pipe it to a file that can be
    compiled and linked with your tests.

    The mocker will only handle functions that are declared in the
    header file you provide. This is based on the presumtion that the
    header file represents functions in a unit. Aggregating functions
    from multiple units into a single header for convenience is not
    supported. Also the mocker cannot handle data declarations (yet?).

    If your header does not name some arguments you will not be able
    to use those arguments in 'expect when' statements, of course.

    Cgreen-mocker will only generate mocks for the external functions
    in the file you give as an argument, not those in included files.

    If cgreen-mocker encounters parse errors and they look like
    gnu-isms you should get a copy of the source for pycparser (on
    which cgreen-mocker is built). In it you will find a
    'fake_libc_include' which help. Create a symbolic link named
    'pycparser' that links to the root of pycparser source and
    cgreen-mocker will find it itself.

    You can find pycparser at https://github.com/eliben/pycparser

""")

if __name__ == "__main__":
    if len(sys.argv) <= 1:
        usage()
        exit(-1)
    if '-v' in sys.argv:
        verbose = True
        sys.argv.remove('-v')
    else:
        verbose = False
    if '-gnu' in sys.argv:
        add_gnuisms = True
        sys.argv.remove('-gnu')
    else:
        add_gnuisms = False
    show_func_defs(sys.argv[1:])
<|endoftext|>


<commit_before>"""
 SlipStream Client
 =====
 Copyright (C) 2013 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import logging
import os
import sys
import time
from ConfigParser import SafeConfigParser
import urllib2
import subprocess
import tempfile
import pkgutil

import uuid as uuidModule
import slipstream.exceptions.Exceptions as Exceptions

timeformat = '%Y-%m-%d %H:%M:%S'

VERBOSE_LEVEL_QUIET = 0
VERBOSE_LEVEL_NORMAL = 1
VERBOSE_LEVEL_DETAILED = 2

PRINT_TO_STDERR_ONLY = False

TMPDIR = os.path.join(tempfile.gettempdir(), 'slipstream')
REPORTSDIR = os.environ.get('SLIPSTREAM_REPORT_DIR', os.path.join(os.sep, TMPDIR, 'reports'))
WINDOWS_REPORTSDIR = '%TMP%\\slipstream\\reports'

RUN_URL_PATH = '/run'
MODULE_URL_PATH = '/module'
USER_URL_PATH = '/user'

CONFIGPARAM_CONNECTOR_MODULE_NAME = 'cloudconnector'

SUPPORTED_PLATFORMS_BY_DISTRO = {'debian_based': ('ubuntu',),
                                 'redhat_based': ('fedora', 'redhat', 'centos')}
SUPPORTED_PLATFORMS = [y for x in SUPPORTED_PLATFORMS_BY_DISTRO.values() for y in x]

ENV_NEED_TO_ADD_SSHPUBKEY = 'SLIPSTREAM_NEED_TO_ADD_SSHPUBKEY'

def get_cloudconnector_modulenames(base_package='slipstream.cloudconnectors'):
    module_names = []
    pkgwalk = pkgutil.walk_packages(path=loadModule(base_package).__path__,
                                    prefix=base_package + '.')
    for _, pkgname, ispkg in pkgwalk:
        if ispkg:
            mdlwalk = pkgutil.iter_modules(path=loadModule(pkgname).__path__,
                                           prefix=pkgname + '.')
            for _, name, ispkg in mdlwalk:
                if not ispkg:
                    if hasattr(loadModule(name), 'getConnector'):
                        module_names.append(name)
    return module_names

def get_cloudconnector_modulename_by_cloudname(cloudname):
    for module_name in get_cloudconnector_modulenames():
        connector_class = loadModule(module_name).getConnectorClass()
        if getattr(connector_class, 'cloudName') == cloudname:
            return module_name
    raise Exceptions.NotFoundError('Failed to find cloud connector module for cloud %s.' % \
                                   cloudname)

def needToAddSshPubkey():
    return (os.environ.get(ENV_NEED_TO_ADD_SSHPUBKEY, '').lower() == 'true') and \
           not isWindows()

def configureLogger():
    filename = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'slipstream.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(levelname)s %(message)s',
                        filename=filename)

def isWindows():
    return sys.platform == 'win32'

def execute(commandAndArgsList, **kwargs):
    wait = not kwargs.get('noWait', False)

    if kwargs.has_key('noWait'):
        del kwargs['noWait']

    withStderr = kwargs.get('withStderr', False)
    withStdOutErr = kwargs.get('withOutput', False)
    # Getting stderr takes precedence on getting stdout&stderr.
    if withStderr:
        kwargs['stderr'] = subprocess.PIPE
        withStdOutErr = False
    if kwargs.has_key('withStderr'):
        del kwargs['withStderr']

    if withStdOutErr:
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        kwargs['close_fds'] = True
    if kwargs.has_key('withOutput'):
        del kwargs['withOutput']

    if isWindows():
        commandAndArgsList.insert(0, '-File')
        commandAndArgsList.insert(0, 'Bypass')
        commandAndArgsList.insert(0, '-ExecutionPolicy')
        commandAndArgsList.insert(0, 'powershell')

    if isinstance(commandAndArgsList, list):
        _cmd = ' '.join(commandAndArgsList)
    else:
        _cmd = commandAndArgsList

    printDetail('Calling: %s' % _cmd, kwargs)

    if isinstance(commandAndArgsList, list) and kwargs.get('shell', False) == True:
        commandAndArgsList = ' '.join(commandAndArgsList)

    process = subprocess.Popen(commandAndArgsList, **kwargs)

    if wait:
        process.wait()

    if withStderr:
        return process.returncode, process.stderr.read()
    elif withStdOutErr:
        return process.returncode, process.stdout.read()
    else:
        return process.returncode

def removeLogger(handler):
    logger = logging.getLogger()
    logger.removeHandler(handler)

def redirectStd2Logger():
    configureLogger()
    sys.stderr = StdOutWithLogger('stderr')
    sys.stdout = StdOutWithLogger('stdout')

def resetStdFromLogger():
    sys.stdout = sys.stdout._std
    sys.stderr = sys.stderr._std

def redirectStd2File(filename):
    sys.stderr = StdOutWithFile(filename)
    sys.stdout = StdOutWithFile(filename)

def resetStdFromFile():
    sys.stdout = sys.stdout._stdout
    sys.stderr = sys.stderr._stderr

def whoami():
    return sys._getframe(1).f_code.co_name

class StdOutWithFile(object):
    def __init__(self, filename):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        self.fh = file(filename, 'w')

    def __del__(self):
        self.flush()
        self.fh.close()

    def writelines(self, msgs):
        # Test if msgs is a list or not
        try:
            msgs.append('')
        except AttributeError:
            # It's probably a string, so we write it directly
            self.write(msgs + '\n')
            return
        for msg in msgs:
            self.write(msg)
        self.flush()

    def write(self, _string):
        _string = unicode(_string).encode('utf-8')
        self._stdout.write(_string)
        self.fh.write(_string)
        self.flush()

    def flush(self):
        self._stdout.flush()
        self.fh.flush()

class StdOutWithLogger:
    def __init__(self, std):
        if std == 'stdout':
            self._std = sys.stdout
            self.logType = 'out'
        elif std == 'stderr':
            self._std = sys.stderr
            self.logType = 'err'
        else:
            raise ValueError('Unknown std type: %s' % std)

    def writelines(self, msgs):
        # Test if msgs is a list or not
        try:
            msgs.append('')
        except AttributeError:
            # It's probably a string, so we write it directly
            self.write(msgs + '\n')
            return
        for msg in msgs:
            self.write(msg)
        return

    def write(self, string):
        _string = unicode(string).encode('utf-8')
        self._std.write(_string)
        if string == '.':
            return
        if self.logType == 'out':
            logging.info(_string)
        else:
            logging.error(_string)
        return

    def flush(self):
        self._std.flush()

def getHomeDirectory():
    if (sys.platform == "win32"):
        if (os.environ.has_key("HOME")):
            return os.environ["HOME"]
        elif (os.environ.has_key("USERPROFILE")):
            return os.environ["USERPROFILE"]
        else:
            return "C:\\"
    else:
        if (os.environ.has_key("HOME")):
            return os.environ["HOME"]
        else:
            # No home directory set
            return ""

def getConfigFileName():
    ''' Look for the configuration file in the following order:
        1- local directory
        2- installation location
        3- calling module location
    '''
    filename = 'slipstream.client.conf'
    configFilename = os.path.join(os.getcwd(), filename)
    if os.path.exists(configFilename):
        return configFilename
    configFilename = os.path.join(getInstallationLocation(), filename)
    if not os.path.exists(configFilename):
        configFilename = os.path.join(os.path.dirname(sys.argv[0]), filename)
    if not os.path.exists(configFilename):
        raise Exceptions.ConfigurationError('Failed to find the configuration file: ' + configFilename)
    return configFilename

def getInstallationLocation():
    ''' Look for the installation location in the following order:
        1- SLIPSTREAM_HOME env var if set
        2- Default target directory, if exists (/opt/slipstream/src)
        3- Base module: __file__/../../.., since the util module is namespaced
    '''
    slipstreamDefaultDirName = os.path.join(os.sep, 'opt', 'slipstream', 'client', 'src')
    # Relative to the src dir.  We do this to avoid importing a module, since util
    # should have a minimum of dependencies
    slipstreamDefaultRelativeDirName = os.path.join(os.path.dirname(__file__), '..', '..', '..')
    if os.environ.has_key('SLIPSTREAM_HOME'):
        slipstreamHome = os.environ['SLIPSTREAM_HOME']
    elif os.path.exists(slipstreamDefaultDirName):
        slipstreamHome = slipstreamDefaultDirName
    else:
        slipstreamHome = slipstreamDefaultRelativeDirName
    return slipstreamHome

def uuid():
    '''Generates a unique ID.'''
    return str(uuidModule.uuid4())

def printDetail(message, verboseLevel=1, verboseThreshold=1):
    if verboseLevel >= verboseThreshold:
        printAndFlush('\n    %s\n' % message)

def _printDetail(message, kwargs={}):
    verboseLevel = _extractVerboseLevel(kwargs)
    verboseThreshold = _extractVerboseThreshold(kwargs)
    printDetail(message, verboseLevel, verboseThreshold)

def _extractVerboseLevel(kwargs):
    return _extractAndDeleteKey('verboseLevel', 0, kwargs)

def _extractVerboseThreshold(kwargs):
    return _extractAndDeleteKey('verboseThreshold', 2, kwargs)

def _extractAndDeleteKey(key, default, dict):
    value = default
    if key in dict:
        value = dict[key]
        del dict[key]
    return value

def printAction(message):
    length = len(message)
    padding = 4 * '='
    line = (length + 2 * len(padding) + 2) * '='
    _message = padding + ' %s ' % message + padding
    printAndFlush('\n%s\n%s\n%s\n' % (line, _message, line))

def printStep(message):
    printAndFlush('\n==== %s\n' % message)

def printAndFlush(message):
    if PRINT_TO_STDERR_ONLY:
        output = sys.stderr
    else:
        output = sys.stdout
    output.flush()
    print >> output, message,
    output.flush()

def printError(message):
    sys.stdout.flush()
    sys.stderr.flush()
    print >> sys.stderr, 'ERROR: %s' % message
    sys.stdout.flush()
    sys.stderr.flush()

def assignAttributes(obj, dictionary):
    for key, value in dictionary.items():
        setattr(obj, key, value)

def loadModule(moduleName):
    namespace = ''
    name = moduleName
    if name.find('.') != -1:
        # There's a namespace so we take it into account
        namespace = '.'.join(name.split('.')[:-1])

    return __import__(name, fromlist=namespace)

def parseConfigFile(filename, preserve_case=True):
    parser = SafeConfigParser()
    if preserve_case:
        parser.optionxform = str
    parser.read(filename)
    return parser

def toTimeInIso8601(_time):
    "Convert int or float to time in iso8601 format."
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(_time))

def toTimeInIso8601NoColon(_time):
    return toTimeInIso8601(_time).replace(':', '')

def filePutContent(filename, data):
    _printDetail('Creating file %s with content: \n%s\n' % (filename, data))
    fd = open(filename, 'wb')
    fd.write(data)
    fd.close()

def filePutContentInTempFile(data):
    _, filename = tempfile.mkstemp()
    filePutContent(filename, data)
    return filename

def fileAppendContent(filename, data):
    fd = open(filename, 'a')
    fd.write(data)
    fd.close()

def fileGetContent(filename):
    fd = open(filename, 'rb')
    content = fd.read()
    fd.close()
    return content

def importETree():
    try:
        from lxml import etree
    except ImportError:
        try:
            # Python 2.5
            import xml.etree.ElementTree as etree
        except ImportError:
            try:
                # Python 2.5
                import xml.etree.cElementTree as etree
            except ImportError:
                try:
                    # normal cElementTree install
                    import cElementTree as etree
                except ImportError:
                    try:
                        # normal ElementTree install
                        import elementtree.ElementTree as etree
                    except ImportError:
                        raise Exception("Failed to import ElementTree from any known place")
    return etree

def removeASCIIEscape(data):
    if hasattr(data, 'replace'):
        return data.replace('\x1b', '')
    else:
        return data

def _waitMachineNetworkUpOrAbort(host, instanceId, timeout=60):
    host_coords = "(id=%s, ip=%s)" % (host, instanceId)
    printStep("Waiting for machine network to start: %s" % host_coords)
    if not waitUntilPingOrTimeout(host, timeout):
        msg = 'Unable to ping VM in %i seconds: %s' % (timeout, host_coords)
        raise Exceptions.ExecutionException(msg)

def waitUntilPingOrTimeout(host, timeout, ticks=True, stdout=None, stderr=None):
    if not stdout:
        stdout = open('/dev/null', 'w')
    if not stderr:
        stderr = open('/dev/null', 'w')

    start = time.time()
    hostUp = False
    while not hostUp:
        if ticks:
            sys.stdout.flush()
            sys.stdout.write('.')
        hostUp = ping(host, stdout=stdout, stderr=stderr)
        sleep(1)

        if time.time() - start > timeout:
            if ticks:
                sys.stdout.flush()
                sys.stdout.write('\n')
            return False

    if ticks:
        sys.stdout.flush()
        sys.stdout.write('\n')
    return hostUp

def ping(host, timeout=5, number=1, **kwargs):
    p = subprocess.Popen(['ping', '-q', '-c', str(number), host], **kwargs)
    p.wait()
    success = (p.returncode == 0)
    return success

def sleep(seconds):
    time.sleep(seconds)

def _getSecureHostPortFromUrl(endpoint):
    scheme, netloc, _, _, _, _ = urllib2.urlparse.urlparse(endpoint)
    if scheme == 'http' or not scheme:
        secure = False
    elif scheme == 'https':
        secure = True
    else:
        raise ValueError('Unknown scheme %s' % scheme)
    if ":" in netloc:
        host, port = netloc.split(':')
    else:
        host = netloc
        port = None
    return secure, host, port

def getPackagesInstallCommand(platform, packages):
    if platform.lower() not in SUPPORTED_PLATFORMS:
        raise ValueError("Unsupported platform '%s' while installing packages. Supported: %s" %
                         (platform, ', '.join(SUPPORTED_PLATFORMS)))

    if platform.lower() in SUPPORTED_PLATFORMS_BY_DISTRO['debian_based']:
        cmd = 'apt-get -y install'
    elif platform.lower() in SUPPORTED_PLATFORMS_BY_DISTRO['redhat_based']:
        cmd = 'yum -y install'
    cmd += ' ' + packages
    return cmd

def appendSshPubkeyToAuthorizedKeys(pubkey):
    dot_ssh_path = os.path.expanduser('~') + '/.ssh'
    try:
        os.mkdir(dot_ssh_path)
    except:
        pass
    fileAppendContent(dot_ssh_path + '/authorized_keys',
                      '\n' + pubkey)
<commit_msg>Added some methods to disable standard outputs.<commit_after>"""
 SlipStream Client
 =====
 Copyright (C) 2013 SixSq Sarl (sixsq.com)
 =====
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import logging
import os
import sys
import time
from ConfigParser import SafeConfigParser
import urllib2
import subprocess
import tempfile
import pkgutil
import cStringIO
import contextlib

import uuid as uuidModule
import slipstream.exceptions.Exceptions as Exceptions

timeformat = '%Y-%m-%d %H:%M:%S'

VERBOSE_LEVEL_QUIET = 0
VERBOSE_LEVEL_NORMAL = 1
VERBOSE_LEVEL_DETAILED = 2

PRINT_TO_STDERR_ONLY = False

TMPDIR = os.path.join(tempfile.gettempdir(), 'slipstream')
REPORTSDIR = os.environ.get('SLIPSTREAM_REPORT_DIR', os.path.join(os.sep, TMPDIR, 'reports'))
WINDOWS_REPORTSDIR = '%TMP%\\slipstream\\reports'

RUN_URL_PATH = '/run'
MODULE_URL_PATH = '/module'
USER_URL_PATH = '/user'

CONFIGPARAM_CONNECTOR_MODULE_NAME = 'cloudconnector'

SUPPORTED_PLATFORMS_BY_DISTRO = {'debian_based': ('ubuntu',),
                                 'redhat_based': ('fedora', 'redhat', 'centos')}
SUPPORTED_PLATFORMS = [y for x in SUPPORTED_PLATFORMS_BY_DISTRO.values() for y in x]

ENV_NEED_TO_ADD_SSHPUBKEY = 'SLIPSTREAM_NEED_TO_ADD_SSHPUBKEY'

def get_cloudconnector_modulenames(base_package='slipstream.cloudconnectors'):
    module_names = []
    pkgwalk = pkgutil.walk_packages(path=loadModule(base_package).__path__,
                                    prefix=base_package + '.')
    for _, pkgname, ispkg in pkgwalk:
        if ispkg:
            mdlwalk = pkgutil.iter_modules(path=loadModule(pkgname).__path__,
                                           prefix=pkgname + '.')
            for _, name, ispkg in mdlwalk:
                if not ispkg:
                    if hasattr(loadModule(name), 'getConnector'):
                        module_names.append(name)
    return module_names

def get_cloudconnector_modulename_by_cloudname(cloudname):
    for module_name in get_cloudconnector_modulenames():
        connector_class = loadModule(module_name).getConnectorClass()
        if getattr(connector_class, 'cloudName') == cloudname:
            return module_name
    raise Exceptions.NotFoundError('Failed to find cloud connector module for cloud %s.' % \
                                   cloudname)

def needToAddSshPubkey():
    return (os.environ.get(ENV_NEED_TO_ADD_SSHPUBKEY, '').lower() == 'true') and \
           not isWindows()

def configureLogger():
    filename = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'slipstream.log')
    logging.basicConfig(level=logging.DEBUG,
                        format='%(asctime)s %(levelname)s %(message)s',
                        filename=filename)

def isWindows():
    return sys.platform == 'win32'

def execute(commandAndArgsList, **kwargs):
    wait = not kwargs.get('noWait', False)

    if kwargs.has_key('noWait'):
        del kwargs['noWait']

    withStderr = kwargs.get('withStderr', False)
    withStdOutErr = kwargs.get('withOutput', False)
    # Getting stderr takes precedence on getting stdout&stderr.
    if withStderr:
        kwargs['stderr'] = subprocess.PIPE
        withStdOutErr = False
    if kwargs.has_key('withStderr'):
        del kwargs['withStderr']

    if withStdOutErr:
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.STDOUT
        kwargs['close_fds'] = True
    if kwargs.has_key('withOutput'):
        del kwargs['withOutput']

    if isWindows():
        commandAndArgsList.insert(0, '-File')
        commandAndArgsList.insert(0, 'Bypass')
        commandAndArgsList.insert(0, '-ExecutionPolicy')
        commandAndArgsList.insert(0, 'powershell')

    if isinstance(commandAndArgsList, list):
        _cmd = ' '.join(commandAndArgsList)
    else:
        _cmd = commandAndArgsList

    printDetail('Calling: %s' % _cmd, kwargs)

    if isinstance(commandAndArgsList, list) and kwargs.get('shell', False) == True:
        commandAndArgsList = ' '.join(commandAndArgsList)

    process = subprocess.Popen(commandAndArgsList, **kwargs)

    if wait:
        process.wait()

    if withStderr:
        return process.returncode, process.stderr.read()
    elif withStdOutErr:
        return process.returncode, process.stdout.read()
    else:
        return process.returncode

def removeLogger(handler):
    logger = logging.getLogger()
    logger.removeHandler(handler)

def redirectStd2Logger():
    configureLogger()
    sys.stderr = StdOutWithLogger('stderr')
    sys.stdout = StdOutWithLogger('stdout')

def resetStdFromLogger():
    sys.stdout = sys.stdout._std
    sys.stderr = sys.stderr._std

def redirectStd2File(filename):
    sys.stderr = StdOutWithFile(filename)
    sys.stdout = StdOutWithFile(filename)

def resetStdFromFile():
    sys.stdout = sys.stdout._stdout
    sys.stderr = sys.stderr._stderr

def whoami():
    return sys._getframe(1).f_code.co_name

class StdOutWithFile(object):
    def __init__(self, filename):
        self._stdout = sys.stdout
        self._stderr = sys.stderr
        self.fh = file(filename, 'w')

    def __del__(self):
        self.flush()
        self.fh.close()

    def writelines(self, msgs):
        # Test if msgs is a list or not
        try:
            msgs.append('')
        except AttributeError:
            # It's probably a string, so we write it directly
            self.write(msgs + '\n')
            return
        for msg in msgs:
            self.write(msg)
        self.flush()

    def write(self, _string):
        _string = unicode(_string).encode('utf-8')
        self._stdout.write(_string)
        self.fh.write(_string)
        self.flush()

    def flush(self):
        self._stdout.flush()
        self.fh.flush()

class StdOutWithLogger:
    def __init__(self, std):
        if std == 'stdout':
            self._std = sys.stdout
            self.logType = 'out'
        elif std == 'stderr':
            self._std = sys.stderr
            self.logType = 'err'
        else:
            raise ValueError('Unknown std type: %s' % std)

    def writelines(self, msgs):
        # Test if msgs is a list or not
        try:
            msgs.append('')
        except AttributeError:
            # It's probably a string, so we write it directly
            self.write(msgs + '\n')
            return
        for msg in msgs:
            self.write(msg)
        return

    def write(self, string):
        _string = unicode(string).encode('utf-8')
        self._std.write(_string)
        if string == '.':
            return
        if self.logType == 'out':
            logging.info(_string)
        else:
            logging.error(_string)
        return

    def flush(self):
        self._std.flush()

def getHomeDirectory():
    if (sys.platform == "win32"):
        if (os.environ.has_key("HOME")):
            return os.environ["HOME"]
        elif (os.environ.has_key("USERPROFILE")):
            return os.environ["USERPROFILE"]
        else:
            return "C:\\"
    else:
        if (os.environ.has_key("HOME")):
            return os.environ["HOME"]
        else:
            # No home directory set
            return ""

def getConfigFileName():
    ''' Look for the configuration file in the following order:
        1- local directory
        2- installation location
        3- calling module location
    '''
    filename = 'slipstream.client.conf'
    configFilename = os.path.join(os.getcwd(), filename)
    if os.path.exists(configFilename):
        return configFilename
    configFilename = os.path.join(getInstallationLocation(), filename)
    if not os.path.exists(configFilename):
        configFilename = os.path.join(os.path.dirname(sys.argv[0]), filename)
    if not os.path.exists(configFilename):
        raise Exceptions.ConfigurationError('Failed to find the configuration file: ' + configFilename)
    return configFilename

def getInstallationLocation():
    ''' Look for the installation location in the following order:
        1- SLIPSTREAM_HOME env var if set
        2- Default target directory, if exists (/opt/slipstream/src)
        3- Base module: __file__/../../.., since the util module is namespaced
    '''
    slipstreamDefaultDirName = os.path.join(os.sep, 'opt', 'slipstream', 'client', 'src')
    # Relative to the src dir.  We do this to avoid importing a module, since util
    # should have a minimum of dependencies
    slipstreamDefaultRelativeDirName = os.path.join(os.path.dirname(__file__), '..', '..', '..')
    if os.environ.has_key('SLIPSTREAM_HOME'):
        slipstreamHome = os.environ['SLIPSTREAM_HOME']
    elif os.path.exists(slipstreamDefaultDirName):
        slipstreamHome = slipstreamDefaultDirName
    else:
        slipstreamHome = slipstreamDefaultRelativeDirName
    return slipstreamHome

def uuid():
    '''Generates a unique ID.'''
    return str(uuidModule.uuid4())

def printDetail(message, verboseLevel=1, verboseThreshold=1):
    if verboseLevel >= verboseThreshold:
        printAndFlush('\n    %s\n' % message)

def _printDetail(message, kwargs={}):
    verboseLevel = _extractVerboseLevel(kwargs)
    verboseThreshold = _extractVerboseThreshold(kwargs)
    printDetail(message, verboseLevel, verboseThreshold)

def _extractVerboseLevel(kwargs):
    return _extractAndDeleteKey('verboseLevel', 0, kwargs)

def _extractVerboseThreshold(kwargs):
    return _extractAndDeleteKey('verboseThreshold', 2, kwargs)

def _extractAndDeleteKey(key, default, dict):
    value = default
    if key in dict:
        value = dict[key]
        del dict[key]
    return value

def printAction(message):
    length = len(message)
    padding = 4 * '='
    line = (length + 2 * len(padding) + 2) * '='
    _message = padding + ' %s ' % message + padding
    printAndFlush('\n%s\n%s\n%s\n' % (line, _message, line))

def printStep(message):
    printAndFlush('\n==== %s\n' % message)

def printAndFlush(message):
    if PRINT_TO_STDERR_ONLY:
        output = sys.stderr
    else:
        output = sys.stdout
    output.flush()
    print >> output, message,
    output.flush()

def printError(message):
    sys.stdout.flush()
    sys.stderr.flush()
    print >> sys.stderr, 'ERROR: %s' % message
    sys.stdout.flush()
    sys.stderr.flush()

def assignAttributes(obj, dictionary):
    for key, value in dictionary.items():
        setattr(obj, key, value)

def loadModule(moduleName):
    namespace = ''
    name = moduleName
    if name.find('.') != -1:
        # There's a namespace so we take it into account
        namespace = '.'.join(name.split('.')[:-1])

    return __import__(name, fromlist=namespace)

def parseConfigFile(filename, preserve_case=True):
    parser = SafeConfigParser()
    if preserve_case:
        parser.optionxform = str
    parser.read(filename)
    return parser

def toTimeInIso8601(_time):
    "Convert int or float to time in iso8601 format."
    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(_time))

def toTimeInIso8601NoColon(_time):
    return toTimeInIso8601(_time).replace(':', '')

def filePutContent(filename, data):
    _printDetail('Creating file %s with content: \n%s\n' % (filename, data))
    fd = open(filename, 'wb')
    fd.write(data)
    fd.close()

def filePutContentInTempFile(data):
    _, filename = tempfile.mkstemp()
    filePutContent(filename, data)
    return filename

def fileAppendContent(filename, data):
    fd = open(filename, 'a')
    fd.write(data)
    fd.close()

def fileGetContent(filename):
    fd = open(filename, 'rb')
    content = fd.read()
    fd.close()
    return content

def importETree():
    try:
        from lxml import etree
    except ImportError:
        try:
            # Python 2.5
            import xml.etree.ElementTree as etree
        except ImportError:
            try:
                # Python 2.5
                import xml.etree.cElementTree as etree
            except ImportError:
                try:
                    # normal cElementTree install
                    import cElementTree as etree
                except ImportError:
                    try:
                        # normal ElementTree install
                        import elementtree.ElementTree as etree
                    except ImportError:
                        raise Exception("Failed to import ElementTree from any known place")
    return etree

def removeASCIIEscape(data):
    if hasattr(data, 'replace'):
        return data.replace('\x1b', '')
    else:
        return data

def _waitMachineNetworkUpOrAbort(host, instanceId, timeout=60):
    host_coords = "(id=%s, ip=%s)" % (host, instanceId)
    printStep("Waiting for machine network to start: %s" % host_coords)
    if not waitUntilPingOrTimeout(host, timeout):
        msg = 'Unable to ping VM in %i seconds: %s' % (timeout, host_coords)
        raise Exceptions.ExecutionException(msg)

def waitUntilPingOrTimeout(host, timeout, ticks=True, stdout=None, stderr=None):
    if not stdout:
        stdout = open('/dev/null', 'w')
    if not stderr:
        stderr = open('/dev/null', 'w')

    start = time.time()
    hostUp = False
    while not hostUp:
        if ticks:
            sys.stdout.flush()
            sys.stdout.write('.')
        hostUp = ping(host, stdout=stdout, stderr=stderr)
        sleep(1)

        if time.time() - start > timeout:
            if ticks:
                sys.stdout.flush()
                sys.stdout.write('\n')
            return False

    if ticks:
        sys.stdout.flush()
        sys.stdout.write('\n')
    return hostUp

def ping(host, timeout=5, number=1, **kwargs):
    p = subprocess.Popen(['ping', '-q', '-c', str(number), host], **kwargs)
    p.wait()
    success = (p.returncode == 0)
    return success

def sleep(seconds):
    time.sleep(seconds)

def _getSecureHostPortFromUrl(endpoint):
    scheme, netloc, _, _, _, _ = urllib2.urlparse.urlparse(endpoint)
    if scheme == 'http' or not scheme:
        secure = False
    elif scheme == 'https':
        secure = True
    else:
        raise ValueError('Unknown scheme %s' % scheme)
    if ":" in netloc:
        host, port = netloc.split(':')
    else:
        host = netloc
        port = None
    return secure, host, port

def getPackagesInstallCommand(platform, packages):
    if platform.lower() not in SUPPORTED_PLATFORMS:
        raise ValueError("Unsupported platform '%s' while installing packages. Supported: %s" %
                         (platform, ', '.join(SUPPORTED_PLATFORMS)))

    if platform.lower() in SUPPORTED_PLATFORMS_BY_DISTRO['debian_based']:
        cmd = 'apt-get -y install'
    elif platform.lower() in SUPPORTED_PLATFORMS_BY_DISTRO['redhat_based']:
        cmd = 'yum -y install'
    cmd += ' ' + packages
    return cmd

def appendSshPubkeyToAuthorizedKeys(pubkey):
    dot_ssh_path = os.path.expanduser('~') + '/.ssh'
    try:
        os.mkdir(dot_ssh_path)
    except:
        pass
    fileAppendContent(dot_ssh_path + '/authorized_keys',
                      '\n' + pubkey)

class NullFile(object):
    def write(self, x): pass
    def flush(self): pass

@contextlib.contextmanager
def nostdout():
    save_stdout = sys.stdout
    sys.stdout = NullFile()
    yield
    sys.stdout = save_stdout

@contextlib.contextmanager
def nostderr():
    save_stderr = sys.stderr
    sys.stderr = NullFile()
    yield
    sys.stderr = save_stderr

@contextlib.contextmanager
def nostdouterr():
    save_stdout = sys.stdout
    save_stderr = sys.stderr
    sys.stdout = NullFile()
    sys.stderr = NullFile()
    yield
    sys.stdout = save_stdout
    sys.stderr = save_stderr

<|endoftext|>


<commit_before>from collections import namedtuple
from itertools import chain, imap

from couchdbkit.exceptions import DocTypeError
from couchdbkit.resource import ResourceNotFound
from corehq.util.quickcache import quickcache
from django.http import Http404

from corehq.apps.es import AppES
from dimagi.utils.couch.database import iter_docs

AppBuildVersion = namedtuple('AppBuildVersion', ['app_id', 'build_id', 'version', 'comment'])

@quickcache(['domain'], timeout=1 * 60 * 60)
def domain_has_apps(domain):
    from .models import Application
    results = Application.get_db().view('app_manager/applications_brief',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
        limit=1,
    ).all()
    return len(results) > 0

def get_latest_released_app_doc(domain, app_id):
    """Get the latest starred build for the application"""
    from .models import Application
    key = ['^ReleasedApplications', domain, app_id]
    app = Application.get_db().view(
        'app_manager/applications',
        startkey=key + [{}],
        endkey=key,
        descending=True,
        include_docs=True,
        limit=1,
    ).first()
    return app['doc'] if app else None

def _get_latest_build_view(domain, app_id, include_docs):
    from .models import Application
    return Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain, app_id, {}],
        endkey=[domain, app_id],
        descending=True,
        include_docs=include_docs,
        limit=1,
    ).first()

def get_latest_build_doc(domain, app_id):
    """Get the latest saved build of the application, regardless of star."""
    res = _get_latest_build_view(domain, app_id, include_docs=True)
    return res['doc'] if res else None

def get_latest_build_id(domain, app_id):
    """Get id of the latest build of the application, regardless of star."""
    res = _get_latest_build_view(domain, app_id, include_docs=False)
    return res['id'] if res else None

def get_build_doc_by_version(domain, app_id, version):
    from .models import Application
    res = Application.get_db().view(
        'app_manager/saved_app',
        key=[domain, app_id, version],
        include_docs=True,
        reduce=False,
        limit=1,
    ).first()
    return res['doc'] if res else None

def wrap_app(app_doc, wrap_cls=None):
    """Will raise DocTypeError if it can't figure out the correct class"""
    from corehq.apps.app_manager.util import get_correct_app_class
    cls = wrap_cls or get_correct_app_class(app_doc)
    return cls.wrap(app_doc)

def get_current_app_doc(domain, app_id):
    from .models import Application
    app = Application.get_db().get(app_id)
    if app.get('domain', None) != domain:
        raise ResourceNotFound()
    return app

def get_current_app(domain, app_id):
    return wrap_app(get_current_app_doc(domain, app_id))

def get_app(domain, app_id, wrap_cls=None, latest=False, target=None):
    """
    Utility for getting an app, making sure it's in the domain specified, and
    wrapping it in the right class (Application or RemoteApp).

    'target' is only used if latest=True.  It should be set to one of:
       'build', 'release', or 'save'

    Here are some common usages and the simpler dbaccessor alternatives:
        current_app = get_app(domain, app_id)
                    = get_current_app_doc(domain, app_id)
        latest_released_build = get_app(domain, app_id, latest=True)
                              = get_latest_released_app_doc(domain, app_id)
        latest_build = get_app(domain, app_id, latest=True, target='build')
                     = get_latest_build_doc(domain, app_id)
    Use wrap_app() if you need the wrapped object.
    """
    from .models import Application
    if not app_id:
        raise Http404()
    try:
        app = Application.get_db().get(app_id)
    except ResourceNotFound:
        raise Http404()

    if latest:
        if not domain:
            domain = app['domain']

        if app.get('copy_of'):
            # The id passed in corresponds to a build
            app_id = app.get('copy_of')

        if target == 'build':
            app = get_latest_build_doc(domain, app_id) or app
        elif target == 'save':
            # If the app_id passed in was the working copy, just use that app.
            # If it's a build, get the working copy.
            if app.get('copy_of'):
                app = get_current_app_doc(domain, app_id)
        else:
            app = get_latest_released_app_doc(domain, app_id) or app

    if domain and app['domain'] != domain:
        raise Http404()
    try:
        return wrap_app(app, wrap_cls=wrap_cls)
    except DocTypeError:
        raise Http404()

def get_apps_in_domain(domain, include_remote=True):
    from .models import Application
    from corehq.apps.app_manager.util import get_correct_app_class
    docs = [row['doc'] for row in Application.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}],
        include_docs=True
    )]
    apps = [get_correct_app_class(doc).wrap(doc) for doc in docs]
    if not include_remote:
        apps = [app for app in apps if not app.is_remote_app()]
    return apps

def get_brief_apps_in_domain(domain, include_remote=True):
    from .models import Application
    from corehq.apps.app_manager.util import get_correct_app_class
    docs = [row['value'] for row in Application.get_db().view(
        'app_manager/applications_brief',
        startkey=[domain],
        endkey=[domain, {}]
    )]
    apps = [get_correct_app_class(doc).wrap(doc) for doc in docs]
    if not include_remote:
        apps = [app for app in apps if not app.is_remote_app()]
    return apps

def get_app_ids_in_domain(domain):
    from .models import Application
    return [row['id'] for row in Application.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}]
    )]

def get_built_app_ids(domain):
    """
    Returns the app ids of all apps in the domain that have at least one build.
    """
    from .models import Application
    result = Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
    )
    app_ids = [data.get('value', {}).get('copy_of') for data in result]
    app_ids = list(set(app_ids))
    return [app_id for app_id in app_ids if app_id]

def get_built_app_ids_for_app_id(domain, app_id, version=None):
    """
    Returns all the built apps for an application id. If version is specified returns all apps after that
    version.
    """
    from .models import Application
    key = [domain, app_id]
    skip = 1 if version else 0
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=key + [version],
        endkey=key + [{}],
        reduce=False,
        include_docs=False,
        skip=skip
    ).all()
    return [result['id'] for result in results]

def get_built_app_ids_with_submissions_for_app_id(domain, app_id, version=None):
    """
    Returns all the built apps for an application id that have submissions.
    If version is specified returns all apps after that version.
    """
    from .models import Application
    key = [domain, app_id]
    skip = 1 if version else 0
    results = Application.get_db().view(
        'apps_with_submissions/view',
        startkey=key + [version],
        endkey=key + [{}],
        reduce=False,
        include_docs=False,
        skip=skip
    ).all()
    return [result['id'] for result in results]

def get_built_app_ids_with_submissions_for_app_ids_and_versions(domain, app_ids_and_versions=None):
    """
    Returns all the built app_ids for a domain that has submissions.
    If version is specified returns all apps after that version.
    :domain:
    :app_ids_and_versions: A dictionary mapping an app_id to build version
    """
    app_ids_and_versions = app_ids_and_versions or {}
    app_ids = get_app_ids_in_domain(domain)
    results = []
    for app_id in app_ids:
        results.extend(
            get_built_app_ids_with_submissions_for_app_id(domain, app_id, app_ids_and_versions.get(app_id))
        )
    return results

def get_latest_app_ids_and_versions(domain, app_id=None):
    """
    Returns all the latest app_ids and versions in a dictionary.
    :param domain: The domain to get the app from
    :param app_id: The app_id to get the latest version from. If not specified gets latest versions of all
        apps in the domain
    :returns: {app_id: latest_version}
    """
    from .models import Application
    key = [domain]

    results = Application.get_db().view(
        'app_manager/applications_brief',
        startkey=key + [{}],
        endkey=key,
        descending=True,
        reduce=False,
        include_docs=True,
    ).all()

    latest_ids_and_versions = {}
    if app_id:
        results = filter(lambda r: r['value']['_id'] == app_id, results)
    for result in results:
        app_id = result['value']['_id']
        version = result['value']['version']

        # Since we have sorted, we know the first instance is the latest version
        if app_id not in latest_ids_and_versions:
            latest_ids_and_versions[app_id] = version

    return latest_ids_and_versions

def get_all_apps(domain):
    """
    Returns an iterable over all the apps ever built and current Applications.
    Used for subscription management when apps use subscription only features
    that shouldn't be present in built apps as well as app definitions.
    """
    def _saved_apps():
        from .models import Application
        from corehq.apps.app_manager.util import get_correct_app_class
        saved_app_ids = Application.get_db().view(
            'app_manager/saved_app',
            startkey=[domain],
            endkey=[domain, {}],
            include_docs=False,
            wrapper=lambda row: row['id'],
        )
        correct_wrap = lambda app_doc: get_correct_app_class(app_doc).wrap(app_doc)
        return imap(correct_wrap, iter_docs(Application.get_db(), saved_app_ids))

    return chain(get_apps_in_domain(domain), _saved_apps())

def get_all_app_ids(domain):
    """
    Returns a list of all the app_ids ever built and current Applications.
    """
    from .models import Application
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
    ).all()
    return [result['id'] for result in results]

def get_all_built_app_ids_and_versions(domain, app_id=None):
    """
    Returns a list of all the app_ids ever built and their version.
    [[AppBuildVersion(app_id, build_id, version, comment)], ...]
    If app_id is provided, limit to bulds for that app.
    """
    from .models import Application
    startkey = [domain]
    endkey = [domain, {}]
    if app_id:
        startkey = [domain, app_id]
        endkey = [domain, app_id, {}]
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=startkey,
        endkey=endkey,
        include_docs=True,
    ).all()
    return map(lambda result: AppBuildVersion(
        app_id=result['key'][1],
        build_id=result['id'],
        version=result['key'][2],
        comment=result['value']['build_comment'],
    ), results)

def get_case_types_from_apps(domain):
    """
    Get the case types of modules in applications in the domain.
    :returns: A set of case_types
    """
    q = (AppES()
         .domain(domain)
         .is_build(False)
         .size(0)
         .terms_aggregation('modules.case_type.exact', 'case_types'))
    return set(q.run().aggregations.case_types.keys)

def get_case_sharing_apps_in_domain(domain, exclude_app_id=None):
    apps = get_apps_in_domain(domain, include_remote=False)
    return [a for a in apps if a.case_sharing and exclude_app_id != a.id]
<commit_msg>exclude blank case type<commit_after>from collections import namedtuple
from itertools import chain, imap

from couchdbkit.exceptions import DocTypeError
from couchdbkit.resource import ResourceNotFound
from corehq.util.quickcache import quickcache
from django.http import Http404

from corehq.apps.es import AppES
from dimagi.utils.couch.database import iter_docs

AppBuildVersion = namedtuple('AppBuildVersion', ['app_id', 'build_id', 'version', 'comment'])

@quickcache(['domain'], timeout=1 * 60 * 60)
def domain_has_apps(domain):
    from .models import Application
    results = Application.get_db().view('app_manager/applications_brief',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
        limit=1,
    ).all()
    return len(results) > 0

def get_latest_released_app_doc(domain, app_id):
    """Get the latest starred build for the application"""
    from .models import Application
    key = ['^ReleasedApplications', domain, app_id]
    app = Application.get_db().view(
        'app_manager/applications',
        startkey=key + [{}],
        endkey=key,
        descending=True,
        include_docs=True,
        limit=1,
    ).first()
    return app['doc'] if app else None

def _get_latest_build_view(domain, app_id, include_docs):
    from .models import Application
    return Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain, app_id, {}],
        endkey=[domain, app_id],
        descending=True,
        include_docs=include_docs,
        limit=1,
    ).first()

def get_latest_build_doc(domain, app_id):
    """Get the latest saved build of the application, regardless of star."""
    res = _get_latest_build_view(domain, app_id, include_docs=True)
    return res['doc'] if res else None

def get_latest_build_id(domain, app_id):
    """Get id of the latest build of the application, regardless of star."""
    res = _get_latest_build_view(domain, app_id, include_docs=False)
    return res['id'] if res else None

def get_build_doc_by_version(domain, app_id, version):
    from .models import Application
    res = Application.get_db().view(
        'app_manager/saved_app',
        key=[domain, app_id, version],
        include_docs=True,
        reduce=False,
        limit=1,
    ).first()
    return res['doc'] if res else None

def wrap_app(app_doc, wrap_cls=None):
    """Will raise DocTypeError if it can't figure out the correct class"""
    from corehq.apps.app_manager.util import get_correct_app_class
    cls = wrap_cls or get_correct_app_class(app_doc)
    return cls.wrap(app_doc)

def get_current_app_doc(domain, app_id):
    from .models import Application
    app = Application.get_db().get(app_id)
    if app.get('domain', None) != domain:
        raise ResourceNotFound()
    return app

def get_current_app(domain, app_id):
    return wrap_app(get_current_app_doc(domain, app_id))

def get_app(domain, app_id, wrap_cls=None, latest=False, target=None):
    """
    Utility for getting an app, making sure it's in the domain specified, and
    wrapping it in the right class (Application or RemoteApp).

    'target' is only used if latest=True.  It should be set to one of:
       'build', 'release', or 'save'

    Here are some common usages and the simpler dbaccessor alternatives:
        current_app = get_app(domain, app_id)
                    = get_current_app_doc(domain, app_id)
        latest_released_build = get_app(domain, app_id, latest=True)
                              = get_latest_released_app_doc(domain, app_id)
        latest_build = get_app(domain, app_id, latest=True, target='build')
                     = get_latest_build_doc(domain, app_id)
    Use wrap_app() if you need the wrapped object.
    """
    from .models import Application
    if not app_id:
        raise Http404()
    try:
        app = Application.get_db().get(app_id)
    except ResourceNotFound:
        raise Http404()

    if latest:
        if not domain:
            domain = app['domain']

        if app.get('copy_of'):
            # The id passed in corresponds to a build
            app_id = app.get('copy_of')

        if target == 'build':
            app = get_latest_build_doc(domain, app_id) or app
        elif target == 'save':
            # If the app_id passed in was the working copy, just use that app.
            # If it's a build, get the working copy.
            if app.get('copy_of'):
                app = get_current_app_doc(domain, app_id)
        else:
            app = get_latest_released_app_doc(domain, app_id) or app

    if domain and app['domain'] != domain:
        raise Http404()
    try:
        return wrap_app(app, wrap_cls=wrap_cls)
    except DocTypeError:
        raise Http404()

def get_apps_in_domain(domain, include_remote=True):
    from .models import Application
    from corehq.apps.app_manager.util import get_correct_app_class
    docs = [row['doc'] for row in Application.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}],
        include_docs=True
    )]
    apps = [get_correct_app_class(doc).wrap(doc) for doc in docs]
    if not include_remote:
        apps = [app for app in apps if not app.is_remote_app()]
    return apps

def get_brief_apps_in_domain(domain, include_remote=True):
    from .models import Application
    from corehq.apps.app_manager.util import get_correct_app_class
    docs = [row['value'] for row in Application.get_db().view(
        'app_manager/applications_brief',
        startkey=[domain],
        endkey=[domain, {}]
    )]
    apps = [get_correct_app_class(doc).wrap(doc) for doc in docs]
    if not include_remote:
        apps = [app for app in apps if not app.is_remote_app()]
    return apps

def get_app_ids_in_domain(domain):
    from .models import Application
    return [row['id'] for row in Application.get_db().view(
        'app_manager/applications',
        startkey=[domain, None],
        endkey=[domain, None, {}]
    )]

def get_built_app_ids(domain):
    """
    Returns the app ids of all apps in the domain that have at least one build.
    """
    from .models import Application
    result = Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
    )
    app_ids = [data.get('value', {}).get('copy_of') for data in result]
    app_ids = list(set(app_ids))
    return [app_id for app_id in app_ids if app_id]

def get_built_app_ids_for_app_id(domain, app_id, version=None):
    """
    Returns all the built apps for an application id. If version is specified returns all apps after that
    version.
    """
    from .models import Application
    key = [domain, app_id]
    skip = 1 if version else 0
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=key + [version],
        endkey=key + [{}],
        reduce=False,
        include_docs=False,
        skip=skip
    ).all()
    return [result['id'] for result in results]

def get_built_app_ids_with_submissions_for_app_id(domain, app_id, version=None):
    """
    Returns all the built apps for an application id that have submissions.
    If version is specified returns all apps after that version.
    """
    from .models import Application
    key = [domain, app_id]
    skip = 1 if version else 0
    results = Application.get_db().view(
        'apps_with_submissions/view',
        startkey=key + [version],
        endkey=key + [{}],
        reduce=False,
        include_docs=False,
        skip=skip
    ).all()
    return [result['id'] for result in results]

def get_built_app_ids_with_submissions_for_app_ids_and_versions(domain, app_ids_and_versions=None):
    """
    Returns all the built app_ids for a domain that has submissions.
    If version is specified returns all apps after that version.
    :domain:
    :app_ids_and_versions: A dictionary mapping an app_id to build version
    """
    app_ids_and_versions = app_ids_and_versions or {}
    app_ids = get_app_ids_in_domain(domain)
    results = []
    for app_id in app_ids:
        results.extend(
            get_built_app_ids_with_submissions_for_app_id(domain, app_id, app_ids_and_versions.get(app_id))
        )
    return results

def get_latest_app_ids_and_versions(domain, app_id=None):
    """
    Returns all the latest app_ids and versions in a dictionary.
    :param domain: The domain to get the app from
    :param app_id: The app_id to get the latest version from. If not specified gets latest versions of all
        apps in the domain
    :returns: {app_id: latest_version}
    """
    from .models import Application
    key = [domain]

    results = Application.get_db().view(
        'app_manager/applications_brief',
        startkey=key + [{}],
        endkey=key,
        descending=True,
        reduce=False,
        include_docs=True,
    ).all()

    latest_ids_and_versions = {}
    if app_id:
        results = filter(lambda r: r['value']['_id'] == app_id, results)
    for result in results:
        app_id = result['value']['_id']
        version = result['value']['version']

        # Since we have sorted, we know the first instance is the latest version
        if app_id not in latest_ids_and_versions:
            latest_ids_and_versions[app_id] = version

    return latest_ids_and_versions

def get_all_apps(domain):
    """
    Returns an iterable over all the apps ever built and current Applications.
    Used for subscription management when apps use subscription only features
    that shouldn't be present in built apps as well as app definitions.
    """
    def _saved_apps():
        from .models import Application
        from corehq.apps.app_manager.util import get_correct_app_class
        saved_app_ids = Application.get_db().view(
            'app_manager/saved_app',
            startkey=[domain],
            endkey=[domain, {}],
            include_docs=False,
            wrapper=lambda row: row['id'],
        )
        correct_wrap = lambda app_doc: get_correct_app_class(app_doc).wrap(app_doc)
        return imap(correct_wrap, iter_docs(Application.get_db(), saved_app_ids))

    return chain(get_apps_in_domain(domain), _saved_apps())

def get_all_app_ids(domain):
    """
    Returns a list of all the app_ids ever built and current Applications.
    """
    from .models import Application
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=[domain],
        endkey=[domain, {}],
        include_docs=False,
    ).all()
    return [result['id'] for result in results]

def get_all_built_app_ids_and_versions(domain, app_id=None):
    """
    Returns a list of all the app_ids ever built and their version.
    [[AppBuildVersion(app_id, build_id, version, comment)], ...]
    If app_id is provided, limit to bulds for that app.
    """
    from .models import Application
    startkey = [domain]
    endkey = [domain, {}]
    if app_id:
        startkey = [domain, app_id]
        endkey = [domain, app_id, {}]
    results = Application.get_db().view(
        'app_manager/saved_app',
        startkey=startkey,
        endkey=endkey,
        include_docs=True,
    ).all()
    return map(lambda result: AppBuildVersion(
        app_id=result['key'][1],
        build_id=result['id'],
        version=result['key'][2],
        comment=result['value']['build_comment'],
    ), results)

def get_case_types_from_apps(domain):
    """
    Get the case types of modules in applications in the domain.
    :returns: A set of case_types
    """
    q = (AppES()
         .domain(domain)
         .is_build(False)
         .size(0)
         .terms_aggregation('modules.case_type.exact', 'case_types'))
    return set(q.run().aggregations.case_types.keys) - {''}

def get_case_sharing_apps_in_domain(domain, exclude_app_id=None):
    apps = get_apps_in_domain(domain, include_remote=False)
    return [a for a in apps if a.case_sharing and exclude_app_id != a.id]
<|endoftext|>