<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2021 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Page compiler plugin for pandoc.

You will need, of course, to install pandoc
"""

import io
import os
import subprocess
from typing import List, Union, Dict
from pathlib import Path

from nikola.plugin_categories import PageCompiler
from nikola.utils import req_missing, makedirs, write_metadata

class CompilePandoc(PageCompiler):
    """Compile markups into HTML using pandoc."""

    name = "pandoc"
    friendly_name = "pandoc"

    def set_site(self, site):
        """Set Nikola site."""
        self.config_dependencies = [str(site.config['PANDOC_OPTIONS'])]
        super().set_site(site)

    def _get_pandoc_options(self, source: str) -> List[str]:
        config_options: Union[List[str], Dict[str, List[str]]] = self.site.config['PANDOC_OPTIONS']
        type_ = type(config_options)
        if type_ is list:
            pandoc_options = config_options
        elif type_ is dict:
            ext = Path(source).suffix
            try:
                pandoc_options = config_options[ext]
            except KeyError:
                raise ValueError('Extension {} is not defined in PANDOC_OPTIONS: {}'.format(ext, config_options))
        else:
            raise ValueError('PANDOC_OPTIONS expected to be of type Union[List[str], Dict[str, List[str]]]: {}'.format(config_options))
        self.logger.info('compiling {} with args "{}"'.format(source, ' '.join(pandoc_options)))
        return pandoc_options

    def compile(self, source, dest, is_two_file=True, post=None, lang=None):
        """Compile the source file into HTML and save as dest."""
        makedirs(os.path.dirname(dest))
        try:
            subprocess.check_call(['pandoc', '-o', dest, source] + self._get_pandoc_options(source))
            with open(dest, 'r', encoding='utf-8-sig') as inf:
                output, shortcode_deps = self.site.apply_shortcodes(inf.read())
            with open(dest, 'w', encoding='utf-8') as outf:
                outf.write(output)
            if post is None:
                if shortcode_deps:
                    self.logger.error(
                        "Cannot save dependencies for post {0} (post unknown)",
                        source)
            else:
                post._depfile[dest] += shortcode_deps
        except OSError as e:
            if e.strreror == 'No such file or directory':
                req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False)

    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
        """Compile into HTML strings."""
        raise ValueError("Pandoc compiler does not support compile_string due to multiple output formats")

    def create_post(self, path, **kw):
        """Create a new post."""
        content = kw.pop('content', None)
        onefile = kw.pop('onefile', False)
        # is_page is not used by create_post as of now.
        kw.pop('is_page', False)
        metadata = {}
        metadata.update(self.default_metadata)
        metadata.update(kw)
        makedirs(os.path.dirname(path))
        if not content.endswith('\n'):
            content += '\n'
        with io.open(path, "w+", encoding="utf8") as fd:
            if onefile:
                fd.write(write_metadata(metadata, comment_wrap=True, site=self.site, compiler=self))
            fd.write(content)
<commit_msg>pandoc.py: improve error handling and logging<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2021 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Page compiler plugin for pandoc.

You will need, of course, to install pandoc
"""

import io
import os
import subprocess
from typing import List, Union, Dict
from pathlib import Path

from nikola.plugin_categories import PageCompiler
from nikola.utils import req_missing, makedirs, write_metadata

class CompilePandoc(PageCompiler):
    """Compile markups into HTML using pandoc."""

    name = "pandoc"
    friendly_name = "pandoc"

    def set_site(self, site):
        """Set Nikola site."""
        self.config_dependencies = [str(site.config['PANDOC_OPTIONS'])]
        super().set_site(site)

    def _get_pandoc_options(self, source: str) -> List[str]:
        config_options: Union[List[str], Dict[str, List[str]]] = self.site.config['PANDOC_OPTIONS']
        type_ = type(config_options)
        if type_ is list:
            pandoc_options = config_options
        elif type_ is dict:
            ext = Path(source).suffix
            try:
                pandoc_options = config_options[ext]
            except KeyError:
                self.logger.warn('Setting PANDOC_OPTIONS to [], because extension {} is not defined in PANDOC_OPTIONS: {}.'.format(ext, config_options))
                pandoc_options = []
        else:
            self.logger.warn('Setting PANDOC_OPTIONS to [], because PANDOC_OPTIONS is expected to be of type Union[List[str], Dict[str, List[str]]] but this is not: {}'.format(config_options))
            pandoc_options = []
        return pandoc_options

    def compile(self, source, dest, is_two_file=True, post=None, lang=None):
        """Compile the source file into HTML and save as dest."""
        makedirs(os.path.dirname(dest))
        try:
            subprocess.check_call(['pandoc', '-o', dest, source] + self._get_pandoc_options(source))
            with open(dest, 'r', encoding='utf-8-sig') as inf:
                output, shortcode_deps = self.site.apply_shortcodes(inf.read())
            with open(dest, 'w', encoding='utf-8') as outf:
                outf.write(output)
            if post is None:
                if shortcode_deps:
                    self.logger.error(
                        "Cannot save dependencies for post {0} (post unknown)",
                        source)
            else:
                post._depfile[dest] += shortcode_deps
        except OSError as e:
            if e.strreror == 'No such file or directory':
                req_missing(['pandoc'], 'build this site (compile with pandoc)', python=False)

    def compile_string(self, data, source_path=None, is_two_file=True, post=None, lang=None):
        """Compile into HTML strings."""
        raise ValueError("Pandoc compiler does not support compile_string due to multiple output formats")

    def create_post(self, path, **kw):
        """Create a new post."""
        content = kw.pop('content', None)
        onefile = kw.pop('onefile', False)
        # is_page is not used by create_post as of now.
        kw.pop('is_page', False)
        metadata = {}
        metadata.update(self.default_metadata)
        metadata.update(kw)
        makedirs(os.path.dirname(path))
        if not content.endswith('\n'):
            content += '\n'
        with io.open(path, "w+", encoding="utf8") as fd:
            if onefile:
                fd.write(write_metadata(metadata, comment_wrap=True, site=self.site, compiler=self))
            fd.write(content)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# All Rights Reserved
#

import logging
import os, sys
import math
import md5
import re
import urllib
import shutil
import tempfile

from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from conary.lib import util

from jobmaster.util import logCall

SWAP_SIZE = 268435456 # 256 MB in bytes
TAGSCRIPT_GROWTH = 20971520 # 20MB in bytes
CYLINDERSIZE = 516096
SECTORS         = 63
HEADS           = 16

###########
# all image building related functions are not class members to
# enforce prevention of side effects
###########

def md5sum(s):
    m = md5.new()
    m.update(s)
    return m.hexdigest()

def roundUpSize(size):
    # 13% accounts for reserved block and inode consumption
    size = int(math.ceil((size + TAGSCRIPT_GROWTH + SWAP_SIZE) / 0.87))
    # now round up to next cylinder size
    return size + ((CYLINDERSIZE - (size % CYLINDERSIZE)) % CYLINDERSIZE)

def createDir(d):
    if not os.path.exists(d):
        createDir(os.path.split(d)[0])
        os.mkdir(d)

def mkBlankFile(fn, size, sparse = True):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    if sparse:
        f.seek(size - 1)
        f.write(chr(0))
    else:
        for i in range(size / 512):
            f.write(512 * chr(0))
        f.write((size % 512) * chr(0))
    f.close()

def createFile(fn, contents):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    f.write(contents)
    f.close()

def writeConaryRc(d, mirrorUrl = ''):
    # write the conaryrc file
    conaryrcFile = open(os.path.join(d, 'etc', 'conaryrc'), "w")
    if mirrorUrl:
        type, url = urllib.splittype(mirrorUrl)
        relativeLink = ''
        if not type:
            type = 'http'
        if not url.startswith('//'):
            url = '//' + url
        if not urllib.splithost(url)[1]:
            relativeLink = '/conaryrc'
        mirrorUrl = type + ':' + url + relativeLink
        print >> conaryrcFile, 'includeConfigFile ' + mirrorUrl
    print >> conaryrcFile, "pinTroves kernel.*"
    print >> conaryrcFile, "includeConfigFile /etc/conary/config.d/*"
    conaryrcFile.close()

def createTemporaryRoot(fakeRoot):
    for d in ('etc', 'etc/sysconfig', 'etc/sysconfig/network-scripts',
              'boot/grub', 'tmp', 'proc', 'sys', 'root', 'var'):
        util.mkdirChain(os.path.join(fakeRoot, d))

def fsOddsNEnds(d):
    createFile(os.path.join(d, 'etc', 'fstab'),
               '\n'.join(('LABEL=/ / ext3 defaults 1 1',
                          'none /dev/pts devpts gid=5,mode=620 0 0',
                          'none /dev/shm tmpfs defaults 0 0',
                          'none /proc proc defaults 0 0',
                          'none /sys sysfs defaults 0 0',
                          '/var/swap swap swap defaults 0 0\n')))
    #create a swap file
    mkBlankFile(os.path.join(d, 'var', 'swap'), SWAP_SIZE, sparse = False)
    logCall('/sbin/mkswap %s >/dev/null 2>&1' % \
                  os.path.join(d, 'var', 'swap'))

    util.copytree(os.path.join(d, 'usr', 'share', 'grub', '*', '*'), \
                      os.path.join(d, 'boot', 'grub'))

    #copy the files needed by grub and set up the links
    grubContents = \
        '\n'.join(('default=0',
                   'timeout=0',
                   'title rBuilder Job Slave (template)',
                   '    root (hd0,0)',
                   '    kernel /boot/vmlinuz-template ro root=LABEL=/ quiet',
                   '    initrd /boot/initrd-template.img\n'))
    createFile(os.path.join(d, 'boot', 'grub', 'grub.conf'), grubContents)
    os.symlink('grub.conf', os.path.join(d, 'boot', 'grub', 'menu.1st'))

    #Add the other miscellaneous files needed
    createFile(os.path.join(d, 'etc', 'hosts'),
               '127.0.0.1       localhost.localdomain   localhost\n')
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network'),
               '\n'.join(('NETWORKING=yes',
                          'HOSTNAME=localhost.localdomain\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network-scripts',
                            'ifcfg-eth0.template'),
               '\n'.join(('DEVICE=eth0',
                          'BOOTPROTO=static',
                          'IPADDR=%(ipaddr)s',
                          'GATEWAY=%(masterip)s',
                          'ONBOOT=yes',
                          'TYPE=Ethernet\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'keyboard'),
               '\n'.join(('KEYBOARDTYPE="pc"',
                          'KEYTABLE="us"\n')))
    writeConaryRc(d)

def getRunningKernel():
    p = os.popen('uname -r')
    data = p.read()
    p.close()

    m = re.match('[\d.-]*', data)
    ver = m.group()[:-1]
    p = os.popen('conary q kernel:runtime --full-versions --flavors | grep %s' % ver)
    return p.read().strip()

class ImageCache(object):
    def __init__(self, cachePath):
        self.cachePath = cachePath
        util.mkdirChain(self.cachePath)

        self.conarycfg = conarycfg.ConaryConfiguration(True)

        self.cc = conaryclient.ConaryClient(self.conarycfg)
        self.nc = self.cc.getRepos()

        self.tmpPath = os.path.join(os.path.split(cachePath)[0], 'tmp')

    def deleteAllImages(self):
        # this is for clearing the cache, eg. needed if entitlements changed
        for image in os.listdir(self.cachePath):
            os.unlink(os.path.join(self.cachePath, image))

    def haveImage(self, troveSpec):
        return os.path.exists(self.imagePath(troveSpec))

    def imagePath(self, troveSpec):
        hash = md5sum(troveSpec)
        return os.path.join(self.cachePath, hash)

    def getImage(self, troveSpec):
        hash = md5sum(troveSpec)
        if hash in os.listdir(self.cachePath):
            logging.info("Found image in cache for %s" % troveSpec)
            return os.path.join(self.cachePath, hash)
        else:
            logging.info("Image not cached, creating image for %s" % troveSpec)
            return self.makeImage(troveSpec, hash)

    def makeImage(self, troveSpec, hash):
        n, v, f = cmdline.parseTroveSpec(troveSpec)
        NVF = self.nc.findTrove(None, (n, v, f), self.conarycfg.flavor)[0]

        trv = self.nc.getTrove(NVF[0], NVF[1], NVF[2], withFiles = False)

        size = trv.getSize()
        size = roundUpSize(size)

        fd, fn = tempfile.mkstemp(dir = self.tmpPath)
        os.close(fd)

        fd, tagScript = tempfile.mkstemp(prefix = "tagscript",
                                         dir = self.tmpPath)
        os.close(fd)

        fd, kernelTagScript = tempfile.mkstemp(prefix = "kernel-tagscript",
                                               dir = self.tmpPath)
        os.close(fd)

        mntDir = tempfile.mkdtemp(dir = self.tmpPath)
        try:
            mkBlankFile(fn, size)

            # run mke2fs on blank image
            logCall('mkfs -t ext2 -F -L / %s %d' % \
                          (fn, size / 1024))
            logCall('tune2fs -m 0 -i 0 -c 0 %s' % fn)

            logCall('mount -o loop %s %s' % (fn, mntDir))

            createTemporaryRoot(mntDir)
            logCall('mount -t proc none %s' % os.path.join(mntDir, 'proc'))
            logCall('mount -t sysfs none %s' % os.path.join(mntDir, 'sys'))

            logCall(("conary update '%s' --root %s --replace-files " \
                           "--tag-script=%s") % \
                          (troveSpec, mntDir, tagScript))

            shutil.move(tagScript, os.path.join(mntDir, 'root',
                'conary-tag-script.in'))

            kernelSpec = getRunningKernel()
            logCall(("conary update '%s' --root %s --resolve " \
                       "--keep-required --tag-script=%s" ) \
                          % (kernelSpec, mntDir, kernelTagScript))

            shutil.move(kernelTagScript, os.path.join(mntDir, 'root',
                                        'conary-tag-script-kernel'))
            fsOddsNEnds(mntDir)

            outScript = os.path.join(mntDir, 'root', 'conary-tag-script')
            inScript = outScript + '.in'
            logCall('echo "/sbin/ldconfig" > %s; cat %s | sed "s|/sbin/ldconfig||g" | grep -vx "" >> %s' % (outScript, inScript, outScript))
            os.unlink(os.path.join(mntDir, 'root', 'conary-tag-script.in'))

            for tagScript in ('conary-tag-script', 'conary-tag-script-kernel'):
                tagPath = util.joinPaths(os.path.sep, 'root', tagScript)
                if os.path.exists(util.joinPaths(mntDir, tagPath)):
                    util.execute("chroot %s bash -c 'sh -x %s > %s 2>&1'" % \
                                     (mntDir, tagPath, tagPath + '.output'))

            # FIXME: long term this code would be needed for remote slaves
            #os.system("conary update --sync-to-parents kernel:runtime "
            #          "--root %s" % mntDir)

            safeEnv = {"LD_PRELOAD": "/usr/lib/jobmaster/chrootsafe_wrapper.so"}
            logCall("chroot %s /usr/sbin/authconfig --kickstart --enablemd5 --enableshadow --disablecache" % mntDir, env = safeEnv)
            logCall("chroot %s /usr/sbin/usermod -p '' root" % mntDir)
            logCall('grubby --remove-kernel=/boot/vmlinuz-template --config-file=%s' % os.path.join(mntDir, 'boot', 'grub', 'grub.conf'))
        finally:
            logCall('umount %s' % os.path.join(mntDir, 'proc'))
            logCall('umount %s' % os.path.join(mntDir, 'sys'))
            logCall('sync')
            logCall('umount %s' % mntDir)
            logCall('sync')
            util.rmtree(mntDir, ignore_errors = True)
        shutil.move(fn, os.path.join(self.cachePath, hash))
        return os.path.join(self.cachePath, hash)
<commit_msg>hack to solve RBL-1797 until the preload wrapper is working<commit_after>#!/usr/bin/python
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# All Rights Reserved
#

import logging
import os, sys
import math
import md5
import re
import urllib
import shutil
import tempfile

from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from conary.lib import util

from jobmaster.util import logCall

SWAP_SIZE = 268435456 # 256 MB in bytes
TAGSCRIPT_GROWTH = 20971520 # 20MB in bytes
CYLINDERSIZE = 516096
SECTORS         = 63
HEADS           = 16

###########
# all image building related functions are not class members to
# enforce prevention of side effects
###########

def md5sum(s):
    m = md5.new()
    m.update(s)
    return m.hexdigest()

def roundUpSize(size):
    # 13% accounts for reserved block and inode consumption
    size = int(math.ceil((size + TAGSCRIPT_GROWTH + SWAP_SIZE) / 0.87))
    # now round up to next cylinder size
    return size + ((CYLINDERSIZE - (size % CYLINDERSIZE)) % CYLINDERSIZE)

def createDir(d):
    if not os.path.exists(d):
        createDir(os.path.split(d)[0])
        os.mkdir(d)

def mkBlankFile(fn, size, sparse = True):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    if sparse:
        f.seek(size - 1)
        f.write(chr(0))
    else:
        for i in range(size / 512):
            f.write(512 * chr(0))
        f.write((size % 512) * chr(0))
    f.close()

def createFile(fn, contents):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    f.write(contents)
    f.close()

def writeConaryRc(d, mirrorUrl = ''):
    # write the conaryrc file
    conaryrcFile = open(os.path.join(d, 'etc', 'conaryrc'), "w")
    if mirrorUrl:
        type, url = urllib.splittype(mirrorUrl)
        relativeLink = ''
        if not type:
            type = 'http'
        if not url.startswith('//'):
            url = '//' + url
        if not urllib.splithost(url)[1]:
            relativeLink = '/conaryrc'
        mirrorUrl = type + ':' + url + relativeLink
        print >> conaryrcFile, 'includeConfigFile ' + mirrorUrl
    print >> conaryrcFile, "pinTroves kernel.*"
    print >> conaryrcFile, "includeConfigFile /etc/conary/config.d/*"
    conaryrcFile.close()

def createTemporaryRoot(fakeRoot):
    for d in ('etc', 'etc/sysconfig', 'etc/sysconfig/network-scripts',
              'boot/grub', 'tmp', 'proc', 'sys', 'root', 'var'):
        util.mkdirChain(os.path.join(fakeRoot, d))

def fsOddsNEnds(d):
    createFile(os.path.join(d, 'etc', 'fstab'),
               '\n'.join(('LABEL=/ / ext3 defaults 1 1',
                          'none /dev/pts devpts gid=5,mode=620 0 0',
                          'none /dev/shm tmpfs defaults 0 0',
                          'none /proc proc defaults 0 0',
                          'none /sys sysfs defaults 0 0',
                          '/var/swap swap swap defaults 0 0\n')))
    #create a swap file
    mkBlankFile(os.path.join(d, 'var', 'swap'), SWAP_SIZE, sparse = False)
    logCall('/sbin/mkswap %s >/dev/null 2>&1' % \
                  os.path.join(d, 'var', 'swap'))

    util.copytree(os.path.join(d, 'usr', 'share', 'grub', '*', '*'), \
                      os.path.join(d, 'boot', 'grub'))

    #copy the files needed by grub and set up the links
    grubContents = \
        '\n'.join(('default=0',
                   'timeout=0',
                   'title rBuilder Job Slave (template)',
                   '    root (hd0,0)',
                   '    kernel /boot/vmlinuz-template ro root=LABEL=/ quiet',
                   '    initrd /boot/initrd-template.img\n'))
    createFile(os.path.join(d, 'boot', 'grub', 'grub.conf'), grubContents)
    os.symlink('grub.conf', os.path.join(d, 'boot', 'grub', 'menu.1st'))

    #Add the other miscellaneous files needed
    createFile(os.path.join(d, 'etc', 'hosts'),
               '127.0.0.1       localhost.localdomain   localhost\n')
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network'),
               '\n'.join(('NETWORKING=yes',
                          'HOSTNAME=localhost.localdomain\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network-scripts',
                            'ifcfg-eth0.template'),
               '\n'.join(('DEVICE=eth0',
                          'BOOTPROTO=static',
                          'IPADDR=%(ipaddr)s',
                          'GATEWAY=%(masterip)s',
                          'ONBOOT=yes',
                          'TYPE=Ethernet\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'keyboard'),
               '\n'.join(('KEYBOARDTYPE="pc"',
                          'KEYTABLE="us"\n')))
    writeConaryRc(d)

def getRunningKernel():
    p = os.popen('uname -r')
    data = p.read()
    p.close()

    m = re.match('[\d.-]*', data)
    ver = m.group()[:-1]
    p = os.popen('conary q kernel:runtime --full-versions --flavors | grep %s' % ver)
    return p.read().strip()

class ImageCache(object):
    def __init__(self, cachePath):
        self.cachePath = cachePath
        util.mkdirChain(self.cachePath)

        self.conarycfg = conarycfg.ConaryConfiguration(True)

        self.cc = conaryclient.ConaryClient(self.conarycfg)
        self.nc = self.cc.getRepos()

        self.tmpPath = os.path.join(os.path.split(cachePath)[0], 'tmp')

    def deleteAllImages(self):
        # this is for clearing the cache, eg. needed if entitlements changed
        for image in os.listdir(self.cachePath):
            os.unlink(os.path.join(self.cachePath, image))

    def haveImage(self, troveSpec):
        return os.path.exists(self.imagePath(troveSpec))

    def imagePath(self, troveSpec):
        hash = md5sum(troveSpec)
        return os.path.join(self.cachePath, hash)

    def getImage(self, troveSpec):
        hash = md5sum(troveSpec)
        if hash in os.listdir(self.cachePath):
            logging.info("Found image in cache for %s" % troveSpec)
            return os.path.join(self.cachePath, hash)
        else:
            logging.info("Image not cached, creating image for %s" % troveSpec)
            return self.makeImage(troveSpec, hash)

    def makeImage(self, troveSpec, hash):
        n, v, f = cmdline.parseTroveSpec(troveSpec)
        NVF = self.nc.findTrove(None, (n, v, f), self.conarycfg.flavor)[0]

        trv = self.nc.getTrove(NVF[0], NVF[1], NVF[2], withFiles = False)

        size = trv.getSize()
        size = roundUpSize(size)

        fd, fn = tempfile.mkstemp(dir = self.tmpPath)
        os.close(fd)

        fd, tagScript = tempfile.mkstemp(prefix = "tagscript",
                                         dir = self.tmpPath)
        os.close(fd)

        fd, kernelTagScript = tempfile.mkstemp(prefix = "kernel-tagscript",
                                               dir = self.tmpPath)
        os.close(fd)

        mntDir = tempfile.mkdtemp(dir = self.tmpPath)
        try:
            mkBlankFile(fn, size)

            # run mke2fs on blank image
            logCall('mkfs -t ext2 -F -L / %s %d' % \
                          (fn, size / 1024))
            logCall('tune2fs -m 0 -i 0 -c 0 %s' % fn)

            logCall('mount -o loop %s %s' % (fn, mntDir))

            createTemporaryRoot(mntDir)
            logCall('mount -t proc none %s' % os.path.join(mntDir, 'proc'))
            logCall('mount -t sysfs none %s' % os.path.join(mntDir, 'sys'))

            logCall(("conary update '%s' --root %s --replace-files " \
                           "--tag-script=%s") % \
                          (troveSpec, mntDir, tagScript))

            shutil.move(tagScript, os.path.join(mntDir, 'root',
                'conary-tag-script.in'))

            kernelSpec = getRunningKernel()
            logCall(("conary update '%s' --root %s --resolve " \
                       "--keep-required --tag-script=%s" ) \
                          % (kernelSpec, mntDir, kernelTagScript))

            shutil.move(kernelTagScript, os.path.join(mntDir, 'root',
                                        'conary-tag-script-kernel'))
            fsOddsNEnds(mntDir)

            outScript = os.path.join(mntDir, 'root', 'conary-tag-script')
            inScript = outScript + '.in'
            logCall('echo "/sbin/ldconfig" > %s; cat %s | sed "s|/sbin/ldconfig||g" | grep -vx "" >> %s' % (outScript, inScript, outScript))
            os.unlink(os.path.join(mntDir, 'root', 'conary-tag-script.in'))

            for tagScript in ('conary-tag-script', 'conary-tag-script-kernel'):
                tagPath = util.joinPaths(os.path.sep, 'root', tagScript)
                if os.path.exists(util.joinPaths(mntDir, tagPath)):
                    util.execute("chroot %s bash -c 'sh -x %s > %s 2>&1'" % \
                                     (mntDir, tagPath, tagPath + '.output'))

            # FIXME: long term this code would be needed for remote slaves
            #os.system("conary update --sync-to-parents kernel:runtime "
            #          "--root %s" % mntDir)

            # the preload wrapper isn't working yet, work around until we know why
            #safeEnv = {"LD_PRELOAD": "/usr/lib/jobmaster/chrootsafe_wrapper.so"}

            # authconfig can whack the domainname in certain circumstances
            oldDomainname = os.popen('domainname').read().strip() # save old domainname
            logCall("chroot %s /usr/sbin/authconfig --kickstart --enablemd5 --enableshadow --disablecache" % mntDir)
            logCall("domainname %s" % oldDomainname) # restore it

            logCall("chroot %s /usr/sbin/usermod -p '' root" % mntDir)
            logCall('grubby --remove-kernel=/boot/vmlinuz-template --config-file=%s' % os.path.join(mntDir, 'boot', 'grub', 'grub.conf'))
        finally:
            logCall('umount %s' % os.path.join(mntDir, 'proc'))
            logCall('umount %s' % os.path.join(mntDir, 'sys'))
            logCall('sync')
            logCall('umount %s' % mntDir)
            logCall('sync')
            util.rmtree(mntDir, ignore_errors = True)
        shutil.move(fn, os.path.join(self.cachePath, hash))
        return os.path.join(self.cachePath, hash)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2013 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Jinja template handlers"""

import os
import json
from collections import deque
try:
    import jinja2
    from jinja2 import meta
except ImportError:
    jinja2 = None  # NOQA

from nikola.plugin_categories import TemplateSystem

class JinjaTemplates(TemplateSystem):
    """Wrapper for Jinja2 templates."""

    name = "jinja"
    lookup = None
    dependency_cache = {}

    def __init__(self):
        """ initialize Jinja2 wrapper with extended set of filters"""
        if jinja2 is None:
            return
        self.lookup = jinja2.Environment()
        self.lookup.filters['tojson'] = json.dumps

    def set_directories(self, directories, cache_folder):
        """Createa  template lookup."""
        if jinja2 is None:
            raise Exception('To use this theme you need to install the '
                            '"Jinja2" package.')
        self.lookup.loader = jinja2.FileSystemLoader(directories,
                                                     encoding='utf-8')

    def render_template(self, template_name, output_name, context):
        """Render the template into output_name using context."""
        if jinja2 is None:
            raise Exception('To use this theme you need to install the '
                            '"Jinja2" package.')
        template = self.lookup.get_template(template_name)
        output = template.render(**context)
        if output_name is not None:
            try:
                os.makedirs(os.path.dirname(output_name))
            except:
                pass
            with open(output_name, 'w+') as output:
                output.write(output.encode('utf8'))
        return output

    def template_deps(self, template_name):
        # Cache the lists of dependencies for each template name.
        if self.dependency_cache.get(template_name) is None:
            # Use a breadth-first search to find all templates this one
            # depends on.
            queue = deque([template_name])
            visited_templates = set([template_name])
            deps = []
            while len(queue) > 0:
                curr = queue.popleft()
                source, filename = self.lookup.loader.get_source(self.lookup,
                                                                 curr)[:2]
                deps.append(filename)
                ast = self.lookup.parse(source)
                dep_names = meta.find_referenced_templates(ast)
                for dep_name in dep_names:
                    if (dep_name not in visited_templates
                            and dep_name is not None):
                        visited_templates.add(dep_name)
                        queue.append(dep_name)
            self.dependency_cache[template_name] = deps
        return self.dependency_cache[template_name]
<commit_msg>expose enumerate for easier jinja conversions<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2013 Roberto Alsina and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Jinja template handlers"""

import os
import json
from collections import deque
try:
    import jinja2
    from jinja2 import meta
except ImportError:
    jinja2 = None  # NOQA

from nikola.plugin_categories import TemplateSystem

class JinjaTemplates(TemplateSystem):
    """Wrapper for Jinja2 templates."""

    name = "jinja"
    lookup = None
    dependency_cache = {}

    def __init__(self):
        """ initialize Jinja2 wrapper with extended set of filters"""
        if jinja2 is None:
            return
        self.lookup = jinja2.Environment()
        self.lookup.filters['tojson'] = json.dumps
        self.lookup.globals['enumerate'] = enumerate

    def set_directories(self, directories, cache_folder):
        """Createa  template lookup."""
        if jinja2 is None:
            raise Exception('To use this theme you need to install the '
                            '"Jinja2" package.')
        self.lookup.loader = jinja2.FileSystemLoader(directories,
                                                     encoding='utf-8')

    def render_template(self, template_name, output_name, context):
        """Render the template into output_name using context."""
        if jinja2 is None:
            raise Exception('To use this theme you need to install the '
                            '"Jinja2" package.')
        template = self.lookup.get_template(template_name)
        output = template.render(**context)
        if output_name is not None:
            try:
                os.makedirs(os.path.dirname(output_name))
            except:
                pass
            with open(output_name, 'w+') as output:
                output.write(output.encode('utf8'))
        return output

    def template_deps(self, template_name):
        # Cache the lists of dependencies for each template name.
        if self.dependency_cache.get(template_name) is None:
            # Use a breadth-first search to find all templates this one
            # depends on.
            queue = deque([template_name])
            visited_templates = set([template_name])
            deps = []
            while len(queue) > 0:
                curr = queue.popleft()
                source, filename = self.lookup.loader.get_source(self.lookup,
                                                                 curr)[:2]
                deps.append(filename)
                ast = self.lookup.parse(source)
                dep_names = meta.find_referenced_templates(ast)
                for dep_name in dep_names:
                    if (dep_name not in visited_templates
                            and dep_name is not None):
                        visited_templates.add(dep_name)
                        queue.append(dep_name)
            self.dependency_cache[template_name] = deps
        return self.dependency_cache[template_name]
<|endoftext|>


<commit_before># Copyright (c) 2015, Somia Reality Oy
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import absolute_import

from . import _AbstractObjectMessage, declare_messagetype, log
from .. import is_float, is_object

def _is_time(x):
	return is_float(x) and x > 0

@declare_messagetype("ninchat.com/metadata")
class MetadataMessage(_AbstractObjectMessage):
	"""Handler for ninchat.com/metadata messages.
	"""
	_specs = {
		"data": (is_object, True),
		"time": (_is_time,  False),
	}
<commit_msg>ninchat.com/metadata stringification<commit_after># Copyright (c) 2015, Somia Reality Oy
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from __future__ import absolute_import

from . import _AbstractObjectMessage, declare_messagetype, log
from .. import is_float, is_object, is_string

def _is_time(x):
	return is_float(x) and x > 0

@declare_messagetype("ninchat.com/metadata")
class MetadataMessage(_AbstractObjectMessage):
	"""Handler for ninchat.com/metadata messages.
	"""
	_specs = {
		"data": (is_object, True),
		"time": (_is_time,  False),
	}

	def _decode(self):
		data = _AbstractObjectMessage._decode(self)
		if data is None:
			return

		for v in data["data"].values():
			if not is_string(v):
				log.warning("%s values must be strings", self.type)
				return None

		return data

	def stringify(self):
		entries = []

		if self.validate():
			for i in self._data["data"].items():
				entries.append("%s: %s" % i)

		return "\n".join(entries)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

from positive_alert_test_case import PositiveAlertTestCase
from negative_alert_test_case import NegativeAlertTestCase
from alert_test_suite import AlertTestSuite

class TestSessionOpenedUser(AlertTestSuite):
    alert_filename = "session_opened_sensitive_user"

    # This event is the default positive event that will cause the
    # alert to trigger
    default_event = {
        "_type": "event",
        "_source": {
            "category": "syslog",
            "summary": 'pam_unix(sshd:session): session opened for user user1 by (uid=0)',
            "details": {
                "program": "sshd",
                "hostname": "exhostname",
            }
        }
    }

    # This alert is the expected result from running this task
    default_alert = {
        "category": "session",
        "severity": "WARNING",
        "tags": ['pam', 'syslog'],
        "summary": "Session opened by a sensitive user outside of the expected window - sample hosts: exhostname [total 10 hosts]"
    }

    test_cases = []

    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with default event and default alert expected",
            events=AlertTestSuite.create_events(default_event, 10),
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1})
        event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1})
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events a minute earlier",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user2 by (uid=0)'
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user2'",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user1 by (uid=0)'
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user1'",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    randomhostsalert = AlertTestSuite.copy(default_alert)
    randomhostsalert['summary'] = "Session opened by a sensitive user outside of the expected window - sample hosts:"
    for event in events:
        randomhostname = 'host' + str(events.index(event))
        event['_source']['details']['hostname'] = randomhostname
        randomhostsalert['summary'] += ' {0}'.format(randomhostname)
    randomhostsalert['summary'] += " [total 10 hosts]"
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user1'",
            events=events,
            expected_alert=randomhostsalert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user3 by (uid=0)'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with wrong username",
            events=events
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.create_timestamp_from_now_lambda(hour=3, minute=45, second=30)
        event['_source']['receivedtimestamp'] = AlertTestSuite.create_timestamp_from_now_lambda(hour=3, minute=45, second=30)
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with events inside whitelisted time",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'login failed'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with events with a summary of 'login failed'",
            events=events
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['category'] = 'badcategory'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with wrong category",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['details']['program'] = 'ssh'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with wrong program",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session closed for user user1 by (uid=0)'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with 'session closed' summary",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15})
        event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15})
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with old timestamp",
            events=events,
        )
    )
<commit_msg>Adding modified unit test<commit_after>#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Copyright (c) 2017 Mozilla Corporation

from positive_alert_test_case import PositiveAlertTestCase
from negative_alert_test_case import NegativeAlertTestCase
from alert_test_suite import AlertTestSuite

class TestSessionOpenedUser(AlertTestSuite):
    alert_filename = "session_opened_sensitive_user"

    # This event is the default positive event that will cause the
    # alert to trigger
    default_event = {
        "_type": "event",
        "_source": {
            "category": "syslog",
            "hostname": "exhostname",
            "summary": 'pam_unix(sshd:session): session opened for user user1 by (uid=0)',
            "details": {
                "program": "sshd",
            }
        }
    }

    # This alert is the expected result from running this task
    default_alert = {
        "category": "session",
        "severity": "WARNING",
        "tags": ['pam', 'syslog'],
        "summary": "Session opened by a sensitive user outside of the expected window - sample hosts: exhostname [total 10 hosts]"
    }

    test_cases = []

    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with default event and default alert expected",
            events=AlertTestSuite.create_events(default_event, 10),
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1})
        event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda(date_timedelta={'minutes': 1})
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events a minute earlier",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user2 by (uid=0)'
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user2'",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user1 by (uid=0)'
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user1'",
            events=events,
            expected_alert=default_alert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    randomhostsalert = AlertTestSuite.copy(default_alert)
    randomhostsalert['summary'] = "Session opened by a sensitive user outside of the expected window - sample hosts:"
    for event in events:
        randomhostname = 'host' + str(events.index(event))
        event['_source']['hostname'] = randomhostname
        randomhostsalert['summary'] += ' {0}'.format(randomhostname)
    randomhostsalert['summary'] += " [total 10 hosts]"
    test_cases.append(
        PositiveAlertTestCase(
            description="Positive test with events with a user 'user1'",
            events=events,
            expected_alert=randomhostsalert
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session opened for user user3 by (uid=0)'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with wrong username",
            events=events
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.create_timestamp_from_now_lambda(hour=3, minute=45, second=30)
        event['_source']['receivedtimestamp'] = AlertTestSuite.create_timestamp_from_now_lambda(hour=3, minute=45, second=30)
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with events inside whitelisted time",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'login failed'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test with events with a summary of 'login failed'",
            events=events
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['category'] = 'badcategory'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with wrong category",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['details']['program'] = 'ssh'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with wrong program",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['summary'] = 'pam_unix(sshd:session): session closed for user user1 by (uid=0)'
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with events with 'session closed' summary",
            events=events,
        )
    )

    events = AlertTestSuite.create_events(default_event, 10)
    for event in events:
        event['_source']['utctimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15})
        event['_source']['receivedtimestamp'] = AlertTestSuite.subtract_from_timestamp_lambda({'minutes': 15})
    test_cases.append(
        NegativeAlertTestCase(
            description="Negative test case with old timestamp",
            events=events,
        )
    )
<|endoftext|>


<commit_before>import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import pandas as pd
import pprint

import dash_table
from index import app

df = pd.read_csv("./datasets/gapminder.csv")
df_small = pd.read_csv("./datasets/gapminder-small.csv")

CONTROLS = [
    html.Label("Dataset"),
    dcc.RadioItems(
        id="dataset",
        options=[{"label": i, "value": i} for i in ["1700 Rows", "13 Rows"]],
        value="13 Rows",
    ),
    html.Label("Editable"),
    dcc.RadioItems(
        id="editable",
        options=[{"label": str(i), "value": i} for i in [True, False]],
        value=True,
    ),
    html.Label("Sorting"),
    dcc.RadioItems(
        id="sorting",
        options=[{"label": str(i), "value": i} for i in ["fe", True, False]],
        value=True,
    ),
    html.Label("Sorting Type"),
    dcc.RadioItems(
        id="sorting_type",
        options=[{"label": str(i), "value": i} for i in ["single", "multi"]],
        value="single",
    ),
    html.Label("Sorting Treat Empty String As None"),
    dcc.RadioItems(
        id="sorting_treat_empty_string_as_none",
        options=[{"label": str(i), "value": i} for i in [True, False]],
        value=True,
    ),
    html.Label("Row Selectable"),
    dcc.RadioItems(
        id="row_selectable",
        options=[{"label": str(i), "value": i} for i in ["single", "multi"]],
        value="single",
    ),
    html.Label("Virtualization"),
    dcc.RadioItems(
        id="virtualization",
        options=[{"label": str(i), "value": i} for i in ["fe", True, False]],
        value="fe",
    ),
    html.Label("Number of Fixed Rows"),
    html.Div(dcc.Input(id="n_fixed_rows", type="number", value="0")),
    html.Label("Number of Fixed Columns"),
    html.Div(dcc.Input(id="n_fixed_columns", type="number", value="0")),
]

def layout():
    return html.Div(
        className="row",
        children=[
            html.Div(className="four columns", children=html.Div(id="table-container")),
            html.Div(
                className="four columns",
                children=html.Div(
                    [
                        html.Div(
                            CONTROLS,
                            style={"maxHeight": "100vh", "overflowY": "scroll"},
                        )
                    ]
                ),
            ),
            html.Div(
                className="four columns",
                children=html.Div(
                    id="output", style={"maxHeight": "100vh", "overflowY": "scroll"}
                ),
            ),
        ],
    )

@app.callback(
    Output("table-container", "children"),
    [
        Input(prop, "value")
        for prop in [
            "editable",  # 0
            "sorting",  # 1
            "sorting_type",  # 2
            "sorting_treat_empty_string_as_none",  # 3
            "row_selectable",  # 4
            "virtualization",  # 5
            "n_fixed_rows",  # 6
            "n_fixed_columns",  # 7
            "dataset",
        ]
    ],
)
def update_table(*args):
    if args[8] == "1700 Rows":
        rows = df.to_dict("rows")
    else:
        rows = df_small.to_dict("rows")
    return dash_table.Table(
        id=__name__,
        columns=[{"name": i, "id": i} for i in df.columns],
        dataframe=rows,
        editable=args[0],
        sorting=args[1],
        sorting_type=args[2],
        sorting_treat_empty_string_as_none=args[3],
        row_selectable=args[4],
        virtualization=args[5],
        n_fixed_rows=args[6],
        n_fixed_columns=args[7],
    )

AVAILABLE_PROPERTIES = dash_table.Table(id="req").available_properties

@app.callback(
    Output("output", "children"),
    [Input(__name__, prop) for prop in AVAILABLE_PROPERTIES],
)
def display_propeties(*args):
    return html.Div(
        [
            html.Details(
                open=True,
                children=[
                    html.Summary(html.Code(prop)),
                    html.Pre(
                        str(pprint.pformat(args[i])),
                        style={"maxHeight": 200, "overflowY": "scroll"},
                    ),
                ],
            )
            for (i, prop) in enumerate(AVAILABLE_PROPERTIES)
        ]
    )
<commit_msg>:x: add row 'unselectable' option<commit_after>import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import pandas as pd
import pprint

import dash_table
from index import app

df = pd.read_csv("./datasets/gapminder.csv")
df_small = pd.read_csv("./datasets/gapminder-small.csv")

CONTROLS = [
    html.Label("Dataset"),
    dcc.RadioItems(
        id="dataset",
        options=[{"label": i, "value": i} for i in ["1700 Rows", "13 Rows"]],
        value="13 Rows",
    ),
    html.Label("Editable"),
    dcc.RadioItems(
        id="editable",
        options=[{"label": str(i), "value": i} for i in [True, False]],
        value=True,
    ),
    html.Label("Sorting"),
    dcc.RadioItems(
        id="sorting",
        options=[{"label": str(i), "value": i} for i in ["fe", True, False]],
        value=True,
    ),
    html.Label("Sorting Type"),
    dcc.RadioItems(
        id="sorting_type",
        options=[{"label": str(i), "value": i} for i in ["single", "multi"]],
        value="single",
    ),
    html.Label("Sorting Treat Empty String As None"),
    dcc.RadioItems(
        id="sorting_treat_empty_string_as_none",
        options=[{"label": str(i), "value": i} for i in [True, False]],
        value=True,
    ),
    html.Label("Row Selectable"),
    dcc.RadioItems(
        id="row_selectable",
        options=[{"label": str(i), "value": i} for i in ["single", "multi", False]],
        value="single",
    ),
    html.Label("Virtualization"),
    dcc.RadioItems(
        id="virtualization",
        options=[{"label": str(i), "value": i} for i in ["fe", True, False]],
        value="fe",
    ),
    html.Label("Number of Fixed Rows"),
    html.Div(dcc.Input(id="n_fixed_rows", type="number", value="0")),
    html.Label("Number of Fixed Columns"),
    html.Div(dcc.Input(id="n_fixed_columns", type="number", value="0")),
]

def layout():
    return html.Div(
        className="row",
        children=[
            html.Div(className="four columns", children=html.Div(id="table-container")),
            html.Div(
                className="four columns",
                children=html.Div(
                    [
                        html.Div(
                            CONTROLS,
                            style={"maxHeight": "100vh", "overflowY": "scroll"},
                        )
                    ]
                ),
            ),
            html.Div(
                className="four columns",
                children=html.Div(
                    id="output", style={"maxHeight": "100vh", "overflowY": "scroll"}
                ),
            ),
        ],
    )

@app.callback(
    Output("table-container", "children"),
    [
        Input(prop, "value")
        for prop in [
            "editable",  # 0
            "sorting",  # 1
            "sorting_type",  # 2
            "sorting_treat_empty_string_as_none",  # 3
            "row_selectable",  # 4
            "virtualization",  # 5
            "n_fixed_rows",  # 6
            "n_fixed_columns",  # 7
            "dataset",
        ]
    ],
)
def update_table(*args):
    if args[8] == "1700 Rows":
        rows = df.to_dict("rows")
    else:
        rows = df_small.to_dict("rows")
    return dash_table.Table(
        id=__name__,
        columns=[{"name": i, "id": i} for i in df.columns],
        dataframe=rows,
        editable=args[0],
        sorting=args[1],
        sorting_type=args[2],
        sorting_treat_empty_string_as_none=args[3],
        row_selectable=args[4],
        virtualization=args[5],
        n_fixed_rows=args[6],
        n_fixed_columns=args[7],
    )

AVAILABLE_PROPERTIES = dash_table.Table(id="req").available_properties

@app.callback(
    Output("output", "children"),
    [Input(__name__, prop) for prop in AVAILABLE_PROPERTIES],
)
def display_propeties(*args):
    return html.Div(
        [
            html.Details(
                open=True,
                children=[
                    html.Summary(html.Code(prop)),
                    html.Pre(
                        str(pprint.pformat(args[i])),
                        style={"maxHeight": 200, "overflowY": "scroll"},
                    ),
                ],
            )
            for (i, prop) in enumerate(AVAILABLE_PROPERTIES)
        ]
    )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Tests for the Openstack Cloud Provider
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging
import os
import shutil
from time import sleep

# Import Salt Testing libs
from tests.support.case import ShellCase
from tests.support.helpers import generate_random_name, expensiveTest
from tests.support.paths import FILES
from tests.support.runtests import RUNTIME_VARS

# Import Salt Libs
from salt.config import cloud_config, cloud_providers_config
from salt.ext.six.moves import range
from salt.utils import yaml

TIMEOUT = 500

log = logging.getLogger(__name__)

class CloudTest(ShellCase):
    PROVIDER = ''
    REQUIRED_PROVIDER_CONFIG_ITEMS = tuple()
    TMP_PROVIDER_DIR = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'cloud.providers.d')
    __RE_RUN_DELAY = 15
    __RE_TRIES = 3

    def clean_cloud_dir(self, tmp_dir):
        '''
        Clean the cloud.providers.d tmp directory
        '''
        # make sure old provider configs are deleted
        for i in os.listdir(tmp_dir):
            os.remove(os.path.join(tmp_dir, i))

    def query_instances(self):
        '''
        Standardize the data returned from a salt-cloud --query
        '''
        return set(x.strip(': ') for x in self.run_cloud('--query') if x.lstrip().lower().startswith('cloud-test-'))

    def _instance_exists(self, instance_name=None, query=None):
        '''
        :param instance_name: The name of the instance to check for in salt-cloud.
        For example this is may used when a test temporarily renames an instance
        :param query: The result of a salt-cloud --query run outside of this function
        '''
        # salt-cloud -a show_instance myinstance
        if not instance_name:
            instance_name = self.instance_name
        if not query:
            query = self.query_instances()

        log.debug('Checking for "{}" in {}'.format(instance_name, query))
        if isinstance(query, set):
            return instance_name in query
        return any(instance_name == q.strip(': ') for q in query)

    def assertInstanceExists(self, creation_ret=None, instance_name=None):
        '''
        :param instance_name: Override the checked instance name, otherwise the class default will be used.
        :param creation_ret: The return value from the run_cloud() function that created the instance
        '''
        if not instance_name:
            instance_name = self.instance_name

        # If it exists but doesn't show up in the creation_ret, there was probably an error during creation
        if creation_ret:
            self.assertIn(instance_name, [i.strip(': ') for i in creation_ret],
                          'An error occured during instance creation:  |\n\t{}\n\t|'.format(
                              '\n\t'.join(creation_ret)
                          ))
        else:
            # Verify that the instance exists via query
            query = self.query_instances()
            for tries in range(self.__RE_TRIES):
                if self._instance_exists(instance_name, query):
                    log.debug(
                        'Instance "{}" reported after {} seconds'.format(instance_name, tries * self.__RE_RUN_DELAY))
                    break
                else:
                    sleep(self.__RE_RUN_DELAY)
                    query = self.query_instances()

            # Assert that the last query was successful
            self.assertTrue(self._instance_exists(instance_name, query),
                            'Instance "{}" was not created successfully: |\n\t{}\n\t|`'.format(
                                instance_name, '\n\t'.join(creation_ret if creation_ret else query)))

            log.debug('Instance exists and was created: "{}"'.format(instance_name))

    def _destroy_instance(self):
        shutdown_delay = 30
        log.debug('Deleting instance "{}"'.format(self.instance_name))
        delete = self.run_cloud('-d {0} --assume-yes'.format(self.instance_name), timeout=TIMEOUT)
        # example response: ['gce-config:', '----------', '    gce:', '----------', 'cloud-test-dq4e6c:', 'True', '']
        delete_str = yaml.safe_dump(delete)
        log.debug('Deletion status: |\n{}'.format(delete_str))

        if any([x in delete_str for x in (
            'True',
            'was successfully deleted'
        )]):
            destroyed = True
            log.debug('Instance "{}" was successfully deleted'.format(self.instance_name))
        elif any([x in delete_str for x in (
            'shutting-down',
            '.delete',
        )]):
            sleep(shutdown_delay)
            destroyed = True
            log.debug('Instance "{}" is cleaning up'.format(self.instance_name))
        else:
            sleep(shutdown_delay)
            # It's not clear from the delete string that deletion was successful, ask salt-cloud
            query = self.query_instances()
            destroyed = self.instance_name in query
            delete_str += ' :: ' * bool(delete_str) + ', '.join(query)

        return destroyed, delete_str

    def assertDestroyInstance(self):
        success, delete_str = self._destroy_instance()
        self.assertTrue(success, 'Instance "{}" was not deleted: {}'.format(self.instance_name, delete_str))

    @property
    def instance_name(self):
        if not hasattr(self, '_instance_name'):
            # Create the cloud instance name to be used throughout the tests
            subclass = self.__class__.__name__.strip('Test')
            # Use the first three letters of the subclass, fill with '-' if too short
            self._instance_name = generate_random_name('cloud-test-{:-<3}-'.format(subclass[:3])).lower()
        return self._instance_name

    @property
    def providers(self):
        if not hasattr(self, '_providers'):
            self._providers = self.run_cloud('--list-providers')
        return self._providers

    @property
    def provider_config(self):
        if not hasattr(self, '_provider_config'):
            self._provider_config = cloud_providers_config(
                os.path.join(
                    self.config_dir,
                    'cloud.providers.d',
                    self.PROVIDER + '.conf'
                )
            )
        return self._provider_config[self.profile_str][self.PROVIDER]

    @property
    def config(self):
        if not hasattr(self, '_config'):
            self._config = cloud_config(
                os.path.join(
                    self.config_dir,
                    'cloud.profiles.d',
                    self.PROVIDER + '.conf'
                )
            )
        return self._config

    @property
    def profile_str(self):
        return self.PROVIDER + '-config'

    @expensiveTest
    def setUp(self):
        '''
        Sets up the test requirements.  In child classes, define PROVIDER and REQUIRED_CONFIG_ITEMS or this will fail
        '''
        super(CloudTest, self).setUp()

        if not self.PROVIDER:
            self.fail('A PROVIDER must be defined for this test')

        # check if appropriate cloud provider and profile files are present
        if self.profile_str + ':' not in self.providers:
            self.skipTest(
                'Configuration file for {0} was not found. Check {0}.conf files '
                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
                    .format(self.PROVIDER)
            )

        missing_conf_item = []
        for att in self.REQUIRED_PROVIDER_CONFIG_ITEMS:
            if not self.provider_config.get(att):
                missing_conf_item.append(att)

        if missing_conf_item:
            self.skipTest('Conf items are missing that must be provided to run these tests:  {}'
                          .format(', '.join(missing_conf_item)) +
                          '\nCheck tests/integration/files/conf/cloud.providers.d/{0}.conf'.format(self.PROVIDER))

    def tearDown(self):
        '''
        Clean up after tests, If the instance still exists for any reason, delete it.
        Instances should be destroyed before the tearDown, _destroy_instance() should be called exactly
        one time in a test for each instance created.  This is a failSafe and something went wrong
        if the tearDown is where an instance is destroyed.
        '''
        # Make sure that the instance for sure gets deleted, but fail the test if it happens in the tearDown
        destroyed = False
        if self._instance_exists():
            for _ in range(3):
                sleep(30)
                success, result_str = self._destroy_instance()
                if success:
                    self.fail('The instance "{}" was deleted during the tearDown, not the test.'.format(
                        self.instance_name))
                if not self._instance_exists():
                    destroyed = True
                    break

            if not destroyed:
                # Destroying instances in the tearDown is a contingency, not the way things should work by default.
                self.fail('The Instance "{}" was not deleted after multiple attempts'.format(self.instance_name))

    @classmethod
    def tearDownClass(cls):
        cls.clean_cloud_dir(cls, cls.TMP_PROVIDER_DIR)

    @classmethod
    def setUpClass(cls):
        # clean up before setup
        cls.clean_cloud_dir(cls, cls.TMP_PROVIDER_DIR)

        # add the provider config for only the cloud we are testing
        provider_file = cls.PROVIDER + '.conf'
        shutil.copyfile(os.path.join(os.path.join(FILES, 'conf', 'cloud.providers.d'), provider_file),
                        os.path.join(os.path.join(cls.TMP_PROVIDER_DIR, provider_file)))
<commit_msg>Correctly check for GCE and EC2 deletion<commit_after># -*- coding: utf-8 -*-
'''
Tests for the Openstack Cloud Provider
'''

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging
import os
import shutil
from time import sleep

# Import Salt Testing libs
from tests.support.case import ShellCase
from tests.support.helpers import generate_random_name, expensiveTest
from tests.support.paths import FILES
from tests.support.runtests import RUNTIME_VARS

# Import Salt Libs
from salt.config import cloud_config, cloud_providers_config
from salt.ext.six.moves import range
from salt.utils import yaml

TIMEOUT = 500

log = logging.getLogger(__name__)

class CloudTest(ShellCase):
    PROVIDER = ''
    REQUIRED_PROVIDER_CONFIG_ITEMS = tuple()
    TMP_PROVIDER_DIR = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'cloud.providers.d')
    __RE_RUN_DELAY = 15
    __RE_TRIES = 3

    def clean_cloud_dir(self, tmp_dir):
        '''
        Clean the cloud.providers.d tmp directory
        '''
        # make sure old provider configs are deleted
        for i in os.listdir(tmp_dir):
            os.remove(os.path.join(tmp_dir, i))

    def query_instances(self):
        '''
        Standardize the data returned from a salt-cloud --query
        '''
        return set(x.strip(': ') for x in self.run_cloud('--query') if x.lstrip().lower().startswith('cloud-test-'))

    def _instance_exists(self, instance_name=None, query=None):
        '''
        :param instance_name: The name of the instance to check for in salt-cloud.
        For example this is may used when a test temporarily renames an instance
        :param query: The result of a salt-cloud --query run outside of this function
        '''
        # salt-cloud -a show_instance myinstance
        if not instance_name:
            instance_name = self.instance_name
        if not query:
            query = self.query_instances()

        log.debug('Checking for "{}" in {}'.format(instance_name, query))
        if isinstance(query, set):
            return instance_name in query
        return any(instance_name == q.strip(': ') for q in query)

    def assertInstanceExists(self, creation_ret=None, instance_name=None):
        '''
        :param instance_name: Override the checked instance name, otherwise the class default will be used.
        :param creation_ret: The return value from the run_cloud() function that created the instance
        '''
        if not instance_name:
            instance_name = self.instance_name

        # If it exists but doesn't show up in the creation_ret, there was probably an error during creation
        if creation_ret:
            self.assertIn(instance_name, [i.strip(': ') for i in creation_ret],
                          'An error occured during instance creation:  |\n\t{}\n\t|'.format(
                              '\n\t'.join(creation_ret)
                          ))
        else:
            # Verify that the instance exists via query
            query = self.query_instances()
            for tries in range(self.__RE_TRIES):
                if self._instance_exists(instance_name, query):
                    log.debug(
                        'Instance "{}" reported after {} seconds'.format(instance_name, tries * self.__RE_RUN_DELAY))
                    break
                else:
                    sleep(self.__RE_RUN_DELAY)
                    query = self.query_instances()

            # Assert that the last query was successful
            self.assertTrue(self._instance_exists(instance_name, query),
                            'Instance "{}" was not created successfully: |\n\t{}\n\t|`'.format(
                                instance_name, '\n\t'.join(creation_ret if creation_ret else query)))

            log.debug('Instance exists and was created: "{}"'.format(instance_name))

    def _destroy_instance(self):
        shutdown_delay = 30
        log.debug('Deleting instance "{}"'.format(self.instance_name))
        delete = self.run_cloud('-d {0} --assume-yes'.format(self.instance_name), timeout=TIMEOUT)
        # example response: ['gce-config:', '----------', '    gce:', '----------', 'cloud-test-dq4e6c:', 'True', '']
        delete_str = yaml.safe_dump(delete).strip()
        log.debug('Deletion status: |\n{}'.format(delete_str))

        if any([x in delete_str for x in (
            'True',
            'was successfully deleted'
        )]):
            destroyed = True
            log.debug('Instance "{}" was successfully deleted'.format(self.instance_name))
        elif any([x in delete_str for x in (
            'shutting-down',
            '.delete',
            self.instance_name + '-DEL'
        )]):
            destroyed = True
            log.debug('Instance "{}" is cleaning up'.format(self.instance_name))
        else:
            # It's not clear from the delete string that deletion was successful, ask salt-cloud after a delay
            sleep(shutdown_delay)
            query = self.query_instances()
            destroyed = self.instance_name not in query
            delete_str += ' :: ' * bool(delete_str) + ', '.join(query)

        return destroyed, delete_str

    def assertDestroyInstance(self):
        success, delete_str = self._destroy_instance()
        self.assertTrue(success, 'Instance "{}" was not deleted: {}'.format(self.instance_name, delete_str))

    @property
    def instance_name(self):
        if not hasattr(self, '_instance_name'):
            # Create the cloud instance name to be used throughout the tests
            subclass = self.__class__.__name__.strip('Test')
            # Use the first three letters of the subclass, fill with '-' if too short
            self._instance_name = generate_random_name('cloud-test-{:-<3}-'.format(subclass[:3])).lower()
        return self._instance_name

    @property
    def providers(self):
        if not hasattr(self, '_providers'):
            self._providers = self.run_cloud('--list-providers')
        return self._providers

    @property
    def provider_config(self):
        if not hasattr(self, '_provider_config'):
            self._provider_config = cloud_providers_config(
                os.path.join(
                    self.config_dir,
                    'cloud.providers.d',
                    self.PROVIDER + '.conf'
                )
            )
        return self._provider_config[self.profile_str][self.PROVIDER]

    @property
    def config(self):
        if not hasattr(self, '_config'):
            self._config = cloud_config(
                os.path.join(
                    self.config_dir,
                    'cloud.profiles.d',
                    self.PROVIDER + '.conf'
                )
            )
        return self._config

    @property
    def profile_str(self):
        return self.PROVIDER + '-config'

    @expensiveTest
    def setUp(self):
        '''
        Sets up the test requirements.  In child classes, define PROVIDER and REQUIRED_CONFIG_ITEMS or this will fail
        '''
        super(CloudTest, self).setUp()

        if not self.PROVIDER:
            self.fail('A PROVIDER must be defined for this test')

        # check if appropriate cloud provider and profile files are present
        if self.profile_str + ':' not in self.providers:
            self.skipTest(
                'Configuration file for {0} was not found. Check {0}.conf files '
                'in tests/integration/files/conf/cloud.*.d/ to run these tests.'
                    .format(self.PROVIDER)
            )

        missing_conf_item = []
        for att in self.REQUIRED_PROVIDER_CONFIG_ITEMS:
            if not self.provider_config.get(att):
                missing_conf_item.append(att)

        if missing_conf_item:
            self.skipTest('Conf items are missing that must be provided to run these tests:  {}'
                          .format(', '.join(missing_conf_item)) +
                          '\nCheck tests/integration/files/conf/cloud.providers.d/{0}.conf'.format(self.PROVIDER))

    def tearDown(self):
        '''
        Clean up after tests, If the instance still exists for any reason, delete it.
        Instances should be destroyed before the tearDown, _destroy_instance() should be called exactly
        one time in a test for each instance created.  This is a failSafe and something went wrong
        if the tearDown is where an instance is destroyed.
        '''
        # Make sure that the instance for sure gets deleted, but fail the test if it happens in the tearDown
        destroyed = False
        if self._instance_exists():
            for _ in range(3):
                sleep(30)
                success, result_str = self._destroy_instance()
                if success:
                    self.fail('The instance "{}" was deleted during the tearDown, not the test.'.format(
                        self.instance_name))
                if not self._instance_exists():
                    destroyed = True
                    break

            if not destroyed:
                # Destroying instances in the tearDown is a contingency, not the way things should work by default.
                self.fail('The Instance "{}" was not deleted after multiple attempts'.format(self.instance_name))

    @classmethod
    def tearDownClass(cls):
        cls.clean_cloud_dir(cls, cls.TMP_PROVIDER_DIR)

    @classmethod
    def setUpClass(cls):
        # clean up before setup
        cls.clean_cloud_dir(cls, cls.TMP_PROVIDER_DIR)

        # add the provider config for only the cloud we are testing
        provider_file = cls.PROVIDER + '.conf'
        shutil.copyfile(os.path.join(os.path.join(FILES, 'conf', 'cloud.providers.d'), provider_file),
                        os.path.join(os.path.join(cls.TMP_PROVIDER_DIR, provider_file)))
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 31 23:25:33 2017

@author: asadm2
"""

from __future__ import division, print_function, absolute_import
from glob import glob
from astropy.io import fits
from astropy.wcs import WCS
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
import subprocess

eco_repo_path = '/fs1/masad/Research/Repositories/ECO_Globular_Clusters/'

goodObj = '../../../data/interim/goodObj.txt'
good_Obj = pd.read_csv(goodObj,header=None)
good_Obj.columns = ['ECOID']

ECOnew  = '../../../data/interim/ECO_formatted.txt'
ECO_new  = pd.read_csv(ECOnew, delimiter='\t')

ECOphotcat = '../../../data/interim/ECOphot_final.txt'
ECO_phot_cat  = pd.read_csv(ECOphotcat, delim_whitespace=True,header=None,\
                            skiprows=1,names=['ECOID','RA','DEC','VLG','umag',\
                                              'gmag','rmag','imag','zmag',\
                                              'Jmag','Hmag','Kmag'])

#good_Obj_new = good_Obj.ECOID[:4].append(good_Obj.ECOID[5:11].append(good_Obj\
#                             .ECOID[104:105]))

good_Obj_new = ['ECO12028']

sdssr_calc = []
sdssr_cat = []
for obj in good_Obj_new:
    print(obj)
    dir_path = os.getcwd()
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/interim/'+obj)
    elif os.path.basename(dir_path) != obj:
        os.chdir('../interim/'+obj)

    comb_coadd = glob(obj+'comb_coadd.fits')[0]
    f814_coadd = glob(obj+'_acs_wfc_f814w_coadd.fits')[0]

    print('Starting source extractor')
    subprocess.call(['sex',comb_coadd+","+f814_coadd,'-ANALYSIS_THRESH','1.5',\
    '-BACK_SIZE','128','-DEBLEND_MINCONT','0.0025', '-DETECT_THRESH','1.5',\
    '-DETECT_MINAREA','9','-SEEING_FWHM','0.1',\
    '-CATALOG_NAME',obj+'acs_wfc_f814w.cat'])
    print('Finished running source extractor')

    hdu_f814w_coadd = fits.open(f814_coadd)
    prihdr = hdu_f814w_coadd[0].header
    photflam814 = prihdr['PHOTFLAM']
    photzpt814 = prihdr['PHOTZPT']
    print('Calculating zeropoint')
    zpt814 = -2.5*np.log10(photflam814) + photzpt814

    ra = np.unique(ECO_new.RA.loc[ECO_new.ECOID==obj])[0]
    dec = np.unique(ECO_new.DEC.loc[ECO_new.ECOID==obj])[0]

    print('Converting to pixel coordinates')
    wcs = WCS(prihdr)
    xx,yy = wcs.all_world2pix(ra,dec,1)
    print(xx,yy)

#remember to copy already uncommented default.param file and remove other one
#remember to copy all default files to last obj in contents_new

#glob comb_coadd and store as image1
#glob f814 coadd and store as image2
#use subprocess to pass parameters
#open filter image and get necessary keywords for finding zeropoint
#get RA and DEC of galaxy from ECO catalog
#open filter image and get WCS info to find world2pix transformation for RA and
#DEC of galaxy
#open catalog that sextractor generated and get petro mag in f814 at
#calculated x and y pixel values
#Apply zeropoint to magnitude
#Use conversion to change to r band
#Get r band mag from photometry catalog
#Plot both points for each galaxy

#need to use WCS

    print('Reading test.cat')
    f814w_cat = pd.read_csv(obj+'acs_wfc_f814w.cat',header=None,\
                            delim_whitespace=True,skiprows=7,\
                            names=['petro_mag','petro_magerr','petro_radius',\
                                  'x_image','y_image','a_image','class_star'])

    print('Getting petro mag from test.cat')
    print(f814w_cat.x_image==xx)
    print(f814w_cat.y_image==yy)
    f814mag = f814w_cat.petro_mag.loc[(f814w_cat.x_image==xx)&(f814w_cat.y_image==yy)].values[0]

    print('Calculating rmag')
    f814mag += zpt814
    sdss_r = f814mag - 1

    print('Retrieving rmag from catalog')
    sdss_r_cat = ECO_phot_cat.rmag.loc[ECO_phot_cat.ECOID==obj].values[0]

    sdssr_calc.append(sdss_r)
    sdssr_cat.append(sdss_r_cat)

print('Plotting')
x = np.linspace(0,len(good_Obj_new.values)+1,len(good_Obj_new.values))
my_xticks = good_Obj_new.values
fig1 = plt.figure(figsize=(10,8))
plt.xticks(x, my_xticks,rotation=90)
plt.scatter(x,sdssr_calc, c='r',label='calculated rmag')
plt.scatter(x,sdssr_cat, c='g',label='catalog rmag')
plt.xlabel('ECOID')
plt.ylabel('rmag')
plt.gca().invert_yaxis()
plt.title('Comparison of calculated rmag and rmag from ECO photometric table')
plt.savefig('calcrmag_catrmag.png')

    <commit_msg>Adding print statements,removing -CATALOG_NAME parameter<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 31 23:25:33 2017

@author: asadm2
"""

from __future__ import division, print_function, absolute_import
from glob import glob
from astropy.io import fits
from astropy.wcs import WCS
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
import subprocess

eco_repo_path = '/fs1/masad/Research/Repositories/ECO_Globular_Clusters/'

goodObj = '../../../data/interim/goodObj.txt'
good_Obj = pd.read_csv(goodObj,header=None)
good_Obj.columns = ['ECOID']

ECOnew  = '../../../data/interim/ECO_formatted.txt'
ECO_new  = pd.read_csv(ECOnew, delimiter='\t')

ECOphotcat = '../../../data/interim/ECOphot_final.txt'
ECO_phot_cat  = pd.read_csv(ECOphotcat, delim_whitespace=True,header=None,\
                            skiprows=1,names=['ECOID','RA','DEC','VLG','umag',\
                                              'gmag','rmag','imag','zmag',\
                                              'Jmag','Hmag','Kmag'])

#good_Obj_new = good_Obj.ECOID[:4].append(good_Obj.ECOID[5:11].append(good_Obj\
#                             .ECOID[104:105]))

good_Obj_new = ['ECO12028']

sdssr_calc = []
sdssr_cat = []
for obj in good_Obj_new:
    print(obj)
    dir_path = os.getcwd()
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/interim/'+obj)
    elif os.path.basename(dir_path) != obj:
        os.chdir('../interim/'+obj)

    comb_coadd = glob(obj+'comb_coadd.fits')[0]
    f814_coadd = glob(obj+'_acs_wfc_f814w_coadd.fits')[0]

    print('Starting source extractor')
    subprocess.call(['sex',comb_coadd+","+f814_coadd,'-ANALYSIS_THRESH','1.5',\
    '-BACK_SIZE','128','-DEBLEND_MINCONT','0.0025', '-DETECT_THRESH','1.5',\
    '-DETECT_MINAREA','9','-SEEING_FWHM','0.1',\
    '-CATALOG_NAME',obj+'acs_wfc_f814w.cat'])
    print('Finished running source extractor')

    hdu_f814w_coadd = fits.open(f814_coadd)
    prihdr = hdu_f814w_coadd[0].header
    photflam814 = prihdr['PHOTFLAM']
    photzpt814 = prihdr['PHOTZPT']
    print('Calculating zeropoint')
    zpt814 = -2.5*np.log10(photflam814) + photzpt814

    ra = np.unique(ECO_new.RA.loc[ECO_new.ECOID==obj])[0]
    dec = np.unique(ECO_new.DEC.loc[ECO_new.ECOID==obj])[0]

    print('Converting to pixel coordinates')
    wcs = WCS(prihdr)
    print(wcs)
    xx,yy = wcs.all_world2pix(ra,dec,1)
    print(xx,yy)

#remember to copy already uncommented default.param file and remove other one
#remember to copy all default files to last obj in contents_new

#glob comb_coadd and store as image1
#glob f814 coadd and store as image2
#use subprocess to pass parameters
#open filter image and get necessary keywords for finding zeropoint
#get RA and DEC of galaxy from ECO catalog
#open filter image and get WCS info to find world2pix transformation for RA and
#DEC of galaxy
#open catalog that sextractor generated and get petro mag in f814 at
#calculated x and y pixel values
#Apply zeropoint to magnitude
#Use conversion to change to r band
#Get r band mag from photometry catalog
#Plot both points for each galaxy

#need to use WCS

    print('Reading test.cat')
    f814w_cat = pd.read_csv(obj+'acs_wfc_f814w.cat',header=None,\
                            delim_whitespace=True,skiprows=7,\
                            names=['petro_mag','petro_magerr','petro_radius',\
                                  'x_image','y_image','a_image','class_star'])

    print('Getting petro mag from test.cat')
    print(f814w_cat.x_image==xx)
    print(f814w_cat.y_image==yy)
    f814mag = f814w_cat.petro_mag.loc[(f814w_cat.x_image==xx)&(f814w_cat.y_image==yy)].values[0]

    print('Calculating rmag')
    f814mag += zpt814
    sdss_r = f814mag - 1

    print('Retrieving rmag from catalog')
    sdss_r_cat = ECO_phot_cat.rmag.loc[ECO_phot_cat.ECOID==obj].values[0]

    sdssr_calc.append(sdss_r)
    sdssr_cat.append(sdss_r_cat)
    hdu_f814w_coadd.close()

print('Plotting')
x = np.linspace(0,len(good_Obj_new.values)+1,len(good_Obj_new.values))
my_xticks = good_Obj_new.values
fig1 = plt.figure(figsize=(10,8))
plt.xticks(x, my_xticks,rotation=90)
plt.scatter(x,sdssr_calc, c='r',label='calculated rmag')
plt.scatter(x,sdssr_cat, c='g',label='catalog rmag')
plt.xlabel('ECOID')
plt.ylabel('rmag')
plt.gca().invert_yaxis()
plt.title('Comparison of calculated rmag and rmag from ECO photometric table')
plt.savefig('calcrmag_catrmag.png')

    <|endoftext|>