<commit_before>from lxml import etree

from epubcreator import epubbase_names

class EbookData:
    def __init__(self):
        # Lista de Section.
        self.sections = []

        # Lista de Image.
        self.images = []

        # Un objeto Toc.
        self.toc = Toc()

    def addSection(self, section):
        self.sections.append(section)

    def addImage(self, imageName, imageContent):
        self.images.append(Image(imageName, imageContent))

class Section:
    _DOCTYPE = ('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"'
                ' "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">')

    _TEXT = 0
    _TAIL = 1

    def __init__(self, sectionNumber):
        self.name = self._generateSectionName(sectionNumber)

        self._html = etree.Element("html", xmlns="http://www.w3.org/1999/xhtml")

        head = etree.Element("head")
        head.append(etree.Element("title"))
        head.append(etree.Element("link", href="../Styles/style.css", rel="stylesheet", type="text/css"))

        self._html.append(head)

        body = etree.Element("body")

        self._html.append(body)

        # Una pila que contiene los elementos abiertos anidados. La necesito para saber en qué
        # elemento debo escribir el texto.
        self._openedElements = [body]

        # El último elemento en ser abierto o cerrado. Representa el elemento en el cual debo
        # escribir el texto cuando sea necesario.
        self._lastElement = body

        # Tener el último elemento en ser abierto o cerrado no me basta, sino que necesito saber
        # en qué posición debo escribir: text o tail, ya que lxml hace esta diferencia.
        self._textWritePos = Section._TEXT

        # El texto propiamente dicho de cada nodo. Solamente lo vuelco al nodo (ya sea en su
        # atributo text o tail) al momento de abrir o cerrar otro nodo.
        self._textBuffer = []

    def appendText(self, text):
        self._textBuffer.append(text)

    def openTag(self, tag, **attributes):
        self._writeTextBuffer()

        e = etree.Element(tag, **attributes)

        self._openedElements[-1].append(e)
        self._openedElements.append(e)

        self._lastElement = e
        self._textWritePos = Section._TEXT

    def _writeTextBuffer(self):
        text = "".join(self._textBuffer)

        if self._textWritePos == Section._TEXT:
            self._lastElement.text = text
        else:
            self._lastElement.tail = text

        self._textBuffer = []

    def closeTag(self, tag):
        self._writeTextBuffer()

        e = self._openedElements.pop()

        if e.tag != tag:
            raise Exception("Tag mismatch")

        self._lastElement = e
        self._textWritePos = Section._TAIL

    def openHeading(self, level, headingId=None):
        tag = "h{0}".format(level)
        if headingId:
            self.openTag(tag, id=headingId)
        else:
            self.openTag(tag)

    def closeHeading(self, level):
        tag = "h{0}".format(level)
        self.closeTag(tag)

    def appendImg(self, imageName):
        self.openTag("img", alt="", src="../Images/{0}".format(imageName))
        self.closeTag("img")

    def toHtml(self):
        return etree.tostring(self._html, xml_declaration=True, pretty_print=True, encoding="utf-8", doctype=Section._DOCTYPE).decode()

    def toRawText(self):
        text = self._html.xpath("//text()")
        return "".join(text)

    def _generateOpenTag(self, tag, attributes=None):
        if attributes is None:
            return "<{0}>".format(tag)
        else:
            attr = " ".join(['{0}="{1}"'.format(k, v) for k, v in attributes.items()])
            return "<{0} {1}>".format(tag, attr)

    def _generateCloseTag(self, tag):
        return "</{0}>".format(tag)

    def _generateSectionName(self, sectionNumber):
        raise NotImplemented

class TextSection(Section):
    def __init__(self, sectionNumber):
        super().__init__(sectionNumber)

    def insertNoteReference(self, noteNumber):
        self.openTag("a", id="rf{0}".format(noteNumber), href="../Text/notas.xhtml#nt{0}".format(noteNumber))
        self.openTag("sup")
        self.appendText("[{0}]".format(str(noteNumber)))
        self.closeTag("sup")
        self.closeTag("a")

    def toRawText(self):
        # Debo ignorar las referencias a las notas al pie.
        text = self._html.xpath("//text()[not(parent::sup[parent::a])]")
        return "".join(text)

    def _generateSectionName(self, sectionNumber):
        return epubbase_names.generateTextSectionName(sectionNumber)

class NotesSection(Section):
    def __init__(self, sectionNumber=0):
        super().__init__(sectionNumber)

        self._footnotesCount = 0
        self._currentFootnoteSection = ""
        self._hasCurrentFootnoteContent = True

        self.openHeading(1)
        self.appendText("Notas")
        self.closeHeading(1)

    def openTag(self, tag, **attributes):
        if not self._hasCurrentFootnoteContent:
            attributes["id"] = "nt{0}".format(self._footnotesCount)

            super().openTag(tag, **attributes)
            super().openTag("sup")
            self.appendText("[{0}]".format(self._footnotesCount))
            super().closeTag("sup")

            self._hasCurrentFootnoteContent = True
        else:
            super().openTag(tag, **attributes)

    def openNote(self, footnoteSection):
        super().openTag("div", **{"class": "nota"})

        self._currentFootnoteSection = footnoteSection
        self._hasCurrentFootnoteContent = False
        self._footnotesCount += 1

    def closeNote(self):
        # Debe haber un espacio antes del link de retorno.
        # No hay posibilidad de que se genere una excepción acá, porque sé que siempre al menos el
        # elemento "sup" va a existir.
        # Como sé que _lastElement contiene el último elemento en ser abierto o cerrado, entonces el elemento
        # "a" debe ir dentro de éste. No sólo eso, sino que además, obteniendo el tail de su último elemento
        # hijo, tengo la posición exacta para agregar el espacio necesario que separa el link de retorno.
        lastChild = list(self._lastElement)[-1]

        # Cuidado, porque tail puede ser None!
        if lastChild.tail:
            lastChild.tail += " "
        else:
            lastChild.tail = " "

        anchor = etree.Element("a", href="../Text/{0}#rf{1}".format(self._currentFootnoteSection, self._footnotesCount))
        anchor.text = "<<"

        self._lastElement.append(anchor)
        self.closeTag("div")

    def toRawText(self):
        # Debo obviar el título "Notas", el texto de los superíndices y el texto del link de retorno.
        text = self._html.xpath("//text()[not(parent::h1) and not(parent::sup[parent::p[@id]]) and not(parent::a)]")
        return "".join(text)

    def _generateSectionName(self, sectionNumber):
        return epubbase_names.NOTES_FILENAME

class Image:
    def __init__(self, name, content):
        self.name = name
        self.content = content

class Toc:
    def __init__(self):
        self.titles = []
        self._titlesCount = 0

    def addFirstLevelTitle(self, sectionName, titleText, generateId=True):
        self._titlesCount += 1

        title, titleId = self._createTitle(sectionName, titleText, generateId)
        self.titles.append(title)

        return title, titleId

    def addTitleToParent(self, parentTitle, sectionName, titleText, generateId=True):
        self._titlesCount += 1

        title, titleId = self._createTitle(sectionName, titleText, generateId)
        parentTitle.addTitle(title)

        return title, titleId

    def getTotalTitlesCount(self):
        return self._titlesCount

    def getFirstLevelTitlesCount(self):
        return len(self.titles)

    def _createTitle(self, sectionName, titleText, generateId=True):
        titleLocation = sectionName
        titleId = None

        if generateId:
            titleId = "heading_id_{0}".format(self._titlesCount)
            titleLocation += "#{0}".format(titleId)

        title = Title(titleLocation, titleText)
        return title, titleId

class Title:
    def __init__(self, titleLocation, titleText):
        self.titleLocation = titleLocation
        self.text = titleText
        self.childTitles = []

    def addTitle(self, title):
        self.childTitles.append(title)<commit_msg>Corregido bug en los métodos toRawText, donde se ignoraba texto en los superíndices que debía ser procesado.<commit_after>from lxml import etree

from epubcreator import epubbase_names

class EbookData:
    def __init__(self):
        # Lista de Section.
        self.sections = []

        # Lista de Image.
        self.images = []

        # Un objeto Toc.
        self.toc = Toc()

    def addSection(self, section):
        self.sections.append(section)

    def addImage(self, imageName, imageContent):
        self.images.append(Image(imageName, imageContent))

class Section:
    _DOCTYPE = ('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"'
                ' "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">')

    _TEXT = 0
    _TAIL = 1

    def __init__(self, sectionNumber):
        self.name = self._generateSectionName(sectionNumber)

        self._html = etree.Element("html", xmlns="http://www.w3.org/1999/xhtml")

        head = etree.Element("head")
        head.append(etree.Element("title"))
        head.append(etree.Element("link", href="../Styles/style.css", rel="stylesheet", type="text/css"))

        self._html.append(head)

        body = etree.Element("body")

        self._html.append(body)

        # Una pila que contiene los elementos abiertos anidados. La necesito para saber en qué
        # elemento debo escribir el texto.
        self._openedElements = [body]

        # El último elemento en ser abierto o cerrado. Representa el elemento en el cual debo
        # escribir el texto cuando sea necesario.
        self._lastElement = body

        # Tener el último elemento en ser abierto o cerrado no me basta, sino que necesito saber
        # en qué posición debo escribir: text o tail, ya que lxml hace esta diferencia.
        self._textWritePos = Section._TEXT

        # El texto propiamente dicho de cada nodo. Solamente lo vuelco al nodo (ya sea en su
        # atributo text o tail) al momento de abrir o cerrar otro nodo.
        self._textBuffer = []

    def appendText(self, text):
        self._textBuffer.append(text)

    def openTag(self, tag, **attributes):
        self._writeTextBuffer()

        e = etree.Element(tag, **attributes)

        self._openedElements[-1].append(e)
        self._openedElements.append(e)

        self._lastElement = e
        self._textWritePos = Section._TEXT

    def _writeTextBuffer(self):
        text = "".join(self._textBuffer)

        if self._textWritePos == Section._TEXT:
            self._lastElement.text = text
        else:
            self._lastElement.tail = text

        self._textBuffer = []

    def closeTag(self, tag):
        self._writeTextBuffer()

        e = self._openedElements.pop()

        if e.tag != tag:
            raise Exception("Tag mismatch")

        self._lastElement = e
        self._textWritePos = Section._TAIL

    def openHeading(self, level, headingId=None):
        tag = "h{0}".format(level)
        if headingId:
            self.openTag(tag, id=headingId)
        else:
            self.openTag(tag)

    def closeHeading(self, level):
        tag = "h{0}".format(level)
        self.closeTag(tag)

    def appendImg(self, imageName):
        self.openTag("img", alt="", src="../Images/{0}".format(imageName))
        self.closeTag("img")

    def toHtml(self):
        return etree.tostring(self._html, xml_declaration=True, pretty_print=True, encoding="utf-8", doctype=Section._DOCTYPE).decode()

    def toRawText(self):
        text = self._html.xpath("//text()")
        return "".join(text)

    def _generateOpenTag(self, tag, attributes=None):
        if attributes is None:
            return "<{0}>".format(tag)
        else:
            attr = " ".join(['{0}="{1}"'.format(k, v) for k, v in attributes.items()])
            return "<{0} {1}>".format(tag, attr)

    def _generateCloseTag(self, tag):
        return "</{0}>".format(tag)

    def _generateSectionName(self, sectionNumber):
        raise NotImplemented

class TextSection(Section):
    def __init__(self, sectionNumber):
        super().__init__(sectionNumber)

    def insertNoteReference(self, noteNumber):
        self.openTag("a", id="rf{0}".format(noteNumber), href="../Text/notas.xhtml#nt{0}".format(noteNumber))
        self.openTag("sup")
        self.appendText("[{0}]".format(str(noteNumber)))
        self.closeTag("sup")
        self.closeTag("a")

    def toRawText(self):
        # Debo ignorar las referencias a las notas al pie.
        text = self._html.xpath("//*[not(self::sup[parent::a[starts-with(@id, 'rf')]])]/text()")
        return "".join(text)

    def _generateSectionName(self, sectionNumber):
        return epubbase_names.generateTextSectionName(sectionNumber)

class NotesSection(Section):
    def __init__(self, sectionNumber=0):
        super().__init__(sectionNumber)

        self._footnotesCount = 0
        self._currentFootnoteSection = ""
        self._hasCurrentFootnoteContent = True

        self.openHeading(1)
        self.appendText("Notas")
        self.closeHeading(1)

    def openTag(self, tag, **attributes):
        if not self._hasCurrentFootnoteContent:
            attributes["id"] = "nt{0}".format(self._footnotesCount)

            super().openTag(tag, **attributes)
            super().openTag("sup")
            self.appendText("[{0}]".format(self._footnotesCount))
            super().closeTag("sup")

            self._hasCurrentFootnoteContent = True
        else:
            super().openTag(tag, **attributes)

    def openNote(self, footnoteSection):
        super().openTag("div", **{"class": "nota"})

        self._currentFootnoteSection = footnoteSection
        self._hasCurrentFootnoteContent = False
        self._footnotesCount += 1

    def closeNote(self):
        # Debe haber un espacio antes del link de retorno.
        # No hay posibilidad de que se genere una excepción acá, porque sé que siempre al menos el
        # elemento "sup" va a existir.
        # Como sé que _lastElement contiene el último elemento en ser abierto o cerrado, entonces el elemento
        # "a" debe ir dentro de éste. No sólo eso, sino que además, obteniendo el tail de su último elemento
        # hijo, tengo la posición exacta para agregar el espacio necesario que separa el link de retorno.
        lastChild = list(self._lastElement)[-1]

        # Cuidado, porque tail puede ser None!
        if lastChild.tail:
            lastChild.tail += " "
        else:
            lastChild.tail = " "

        anchor = etree.Element("a", href="../Text/{0}#rf{1}".format(self._currentFootnoteSection, self._footnotesCount))
        anchor.text = "<<"

        self._lastElement.append(anchor)
        self.closeTag("div")

    def toRawText(self):
        # Debo obviar el título "Notas", el texto del primer superíndice y el texto del link de retorno.
        text = self._html.xpath("//*[not(self::a) and "
                                "not(. = ./parent::*[starts-with(@id, 'nt')]/child::sup[1]) and "
                                ". != /html/body/child::*[1]]/text()")
        return "".join(text)

    def _generateSectionName(self, sectionNumber):
        return epubbase_names.NOTES_FILENAME

class Image:
    def __init__(self, name, content):
        self.name = name
        self.content = content

class Toc:
    def __init__(self):
        self.titles = []
        self._titlesCount = 0

    def addFirstLevelTitle(self, sectionName, titleText, generateId=True):
        self._titlesCount += 1

        title, titleId = self._createTitle(sectionName, titleText, generateId)
        self.titles.append(title)

        return title, titleId

    def addTitleToParent(self, parentTitle, sectionName, titleText, generateId=True):
        self._titlesCount += 1

        title, titleId = self._createTitle(sectionName, titleText, generateId)
        parentTitle.addTitle(title)

        return title, titleId

    def getTotalTitlesCount(self):
        return self._titlesCount

    def getFirstLevelTitlesCount(self):
        return len(self.titles)

    def _createTitle(self, sectionName, titleText, generateId=True):
        titleLocation = sectionName
        titleId = None

        if generateId:
            titleId = "heading_id_{0}".format(self._titlesCount)
            titleLocation += "#{0}".format(titleId)

        title = Title(titleLocation, titleText)
        return title, titleId

class Title:
    def __init__(self, titleLocation, titleText):
        self.titleLocation = titleLocation
        self.text = titleText
        self.childTitles = []

    def addTitle(self, title):
        self.childTitles.append(title)<|endoftext|>


<commit_before># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.

from hyperstream import MultiOutputTool, StreamInstance, StreamMetaInstance

from dateutil.parser import parse

class CsvMultiReader(MultiOutputTool):
    def __init__(self, filename_template, datetime_parser=parse, datetime_column=0, skip_rows=0):
        super(CsvMultiReader, self).__init__(
            filename_template=filename_template,
            datetime_parser=datetime_parser,
            datetime_column=datetime_column,
            skip_rows=skip_rows
        )

    def _execute(self, source, splitting_stream, interval, output_plate):

        # Let's make the assumption that the first field is the timestamp

        for pv in output_plate.values:
            count = 0

            if len(pv) > 1:
                raise NotImplementedError("Nested plates not supported for this tool")

            # Unpack the plate value
            ((meta_data_id, plate_value),) = pv

            filename = self.filename_template.format(plate_value)

            with open(filename, 'rU') as f:
                for line in f.readlines():
                    count += 1
                    if count < self.skip_rows + 1:
                        continue
                    elements = line.split(',')
                    dt = self.datetime_parser(elements[self.datetime_column])
                    if dt in interval:
                        instance = StreamInstance(dt, map(float, elements[1:]))
                        yield StreamMetaInstance(instance, (meta_data_id, plate_value))
<commit_msg>Solved problem with the datetime_column not being the first column<commit_after># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.

from hyperstream import MultiOutputTool, StreamInstance, StreamMetaInstance

from dateutil.parser import parse

class CsvMultiReader(MultiOutputTool):
    def __init__(self, filename_template, datetime_parser=parse, datetime_column=0, skip_rows=0):
        super(CsvMultiReader, self).__init__(
            filename_template=filename_template,
            datetime_parser=datetime_parser,
            datetime_column=datetime_column,
            skip_rows=skip_rows
        )

    def _execute(self, source, splitting_stream, interval, output_plate):

        # Let's make the assumption that the first field is the timestamp

        for pv in output_plate.values:
            count = 0

            if len(pv) > 1:
                raise NotImplementedError("Nested plates not supported for this tool")

            # Unpack the plate value
            ((meta_data_id, plate_value),) = pv

            filename = self.filename_template.format(plate_value)

            with open(filename, 'rU') as f:
                for line in f.readlines():
                    count += 1
                    if count < self.skip_rows + 1:
                        continue
                    elements = line.split(',')
                    dt = self.datetime_parser(elements[self.datetime_column])
                    del elements[self.datetime_column]
                    if dt in interval:
                        instance = StreamInstance(dt, map(float, elements))
                        yield StreamMetaInstance(instance, (meta_data_id, plate_value))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Installs and configures puppet
"""

import sys
import logging
import os
import platform
import time

from packstack.installer import utils
from packstack.installer import basedefs, output_messages
from packstack.installer.exceptions import ScriptRuntimeError, PuppetError

from packstack.modules.common import filtered_hosts
from packstack.modules.ospluginutils import manifestfiles
from packstack.modules.puppet import scan_logfile, validate_logfile

#------------------ oVirt installer initialization ------------------

PLUGIN_NAME = "Puppet"
PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')

PUPPET_DIR = os.environ.get('PACKSTACK_PUPPETDIR',
                            '/usr/share/openstack-puppet/')
MODULE_DIR = os.path.join(PUPPET_DIR, 'modules')

def initConfig(controller):
    group = {"GROUP_NAME": "PUPPET",
             "DESCRIPTION": "Puppet Config parameters",
             "PRE_CONDITION": lambda x: 'yes',
             "PRE_CONDITION_MATCH": "yes",
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, [])

def initSequences(controller):
    puppetpresteps = [
        {'title': 'Clean Up', 'functions': [run_cleanup]},
    ]
    controller.insertSequence("Clean Up", [], [], puppetpresteps, index=0)

    puppetsteps = [
        {'title': 'Installing Dependencies',
            'functions': [install_deps]},
        {'title': 'Copying Puppet modules and manifests',
            'functions': [copy_puppet_modules]},
        {'title': 'Applying Puppet manifests',
            'functions': [apply_puppet_manifest]},
        {'title': 'Finalizing',
            'functions': [finalize]}
    ]
    controller.addSequence("Puppet", [], [], puppetsteps)

#------------------------- helper functions -------------------------

def wait_for_puppet(currently_running, messages):
    log_len = 0
    twirl = ["-", "\\", "|", "/"]
    while currently_running:
        for hostname, finished_logfile in currently_running:
            log_file = os.path.splitext(os.path.basename(finished_logfile))[0]
            space_len = basedefs.SPACE_LEN - len(log_file)
            if len(log_file) > log_len:
                log_len = len(log_file)
            if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
                twirl = twirl[-1:] + twirl[:-1]
                sys.stdout.write(("\rTesting if puppet apply is finished: %s"
                                 % log_file).ljust(40 + log_len))
                sys.stdout.write("[ %s ]" % twirl[0])
                sys.stdout.flush()
            try:
                # Once a remote puppet run has finished, we retrieve the log
                # file and check it for errors
                local_server = utils.ScriptRunner()
                log = os.path.join(basedefs.PUPPET_MANIFEST_DIR,
                                   os.path.basename(finished_logfile))
                log = log.replace(".finished", ".log")
                local_server.append('scp -o StrictHostKeyChecking=no '
                                    '-o UserKnownHostsFile=/dev/null '
                                    'root@%s:%s %s'
                                    % (hostname, finished_logfile, log))
                # To not pollute logs we turn of logging of command execution
                local_server.execute(log=False)

                # If we got to this point the puppet apply has finished
                currently_running.remove((hostname, finished_logfile))

                # clean off the last "testing apply" msg
                if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
                    sys.stdout.write(("\r").ljust(45 + log_len))

            except ScriptRuntimeError:
                # the test raises an exception if the file doesn't exist yet
                # TO-DO: We need to start testing 'e' for unexpected exceptions
                time.sleep(3)
                continue

            # check log file for relevant notices
            messages.extend(scan_logfile(log))

            # check the log file for errors
            sys.stdout.write('\r')
            try:
                validate_logfile(log)
                state = utils.state_message('%s:' % log_file, 'DONE', 'green')
                sys.stdout.write('%s\n' % state)
                sys.stdout.flush()
            except PuppetError:
                state = utils.state_message('%s:' % log_file, 'ERROR', 'red')
                sys.stdout.write('%s\n' % state)
                sys.stdout.flush()
                raise

#-------------------------- step functions --------------------------

def run_cleanup(config, messages):
    localserver = utils.ScriptRunner()
    localserver.append("rm -rf %s/*pp" % basedefs.PUPPET_MANIFEST_DIR)
    localserver.execute()

def install_deps(config, messages):
    deps = ["puppet", "openssh-clients", "tar", "nc"]
    modules_pkg = 'openstack-puppet-modules'

    local = utils.ScriptRunner()
    local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"'
                 % modules_pkg)

    # This can fail if there are no dependencies other than those
    # filtered out by the egrep expression.
    rc, modules_deps = local.execute(can_fail=False)

    # Modules package might not be installed if we are running from source.
    # In this case we assume user knows what (s)he's doing and we don't
    # install modules dependencies
    if ('%s is not installed' % modules_pkg) not in modules_deps:
        modules_deps = [i.strip() for i in modules_deps.split() if i.strip()]
        deps.extend(modules_deps)

    for hostname in filtered_hosts(config):
        server = utils.ScriptRunner(hostname)
        for package in deps:
            server.append("rpm -q --whatprovides %s || yum install -y %s"
                          % (package, package))
        server.execute()

def copy_puppet_modules(config, messages):
    os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder',
                           'concat', 'firewall', 'glance', 'heat', 'horizon',
                           'inifile', 'keystone', 'memcached', 'mongodb',
                           'mysql', 'neutron', 'nova', 'nssdb', 'openstack',
                           'packstack', 'qpid', 'rabbitmq', 'remote', 'rsync',
                           'ssh', 'stdlib', 'swift', 'sysctl', 'tempest',
                           'vcsrepo', 'vlan', 'vswitch', 'xinetd'))

        # write puppet manifest to disk
    manifestfiles.writeManifests()

    server = utils.ScriptRunner()
    for hostname in filtered_hosts(config):
        host_dir = config['HOST_DETAILS'][hostname]['tmpdir']
        # copy Packstack manifests
        server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR)
        server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR)
        server.append("tar --dereference -cpzf - ../manifests | "
                      "ssh -o StrictHostKeyChecking=no "
                      "-o UserKnownHostsFile=/dev/null "
                      "root@%s tar -C %s -xpzf -" % (hostname, host_dir))

        # copy resources
        resources = config.get('RESOURCES', {})
        for path, localname in resources.get(hostname, []):
            server.append("scp -o StrictHostKeyChecking=no "
                          "-o UserKnownHostsFile=/dev/null "
                          "%s root@%s:%s/resources/%s" %
                          (path, hostname, host_dir, localname))

        # copy Puppet modules required by Packstack
        server.append("cd %s" % MODULE_DIR)
        server.append("tar --dereference -cpzf - %s | "
                      "ssh -o StrictHostKeyChecking=no "
                      "-o UserKnownHostsFile=/dev/null "
                      "root@%s tar -C %s -xpzf -" %
                      (os_modules, hostname,
                       os.path.join(host_dir, 'modules')))
    server.execute()

def apply_puppet_manifest(config, messages):
    if config.get("DRY_RUN"):
        return
    currently_running = []
    lastmarker = None
    loglevel = ''
    logcmd = False
    if logging.root.level <= logging.DEBUG:
        loglevel = '--debug'
        logcmd = True
    for manifest, marker in manifestfiles.getFiles():
        # if the marker has changed then we don't want to proceed until
        # all of the previous puppet runs have finished
        if lastmarker is not None and lastmarker != marker:
            wait_for_puppet(currently_running, messages)
        lastmarker = marker

        for hostname in filtered_hosts(config):
            if "%s_" % hostname not in manifest:
                continue

            host_dir = config['HOST_DETAILS'][hostname]['tmpdir']
            print "Applying %s" % manifest
            server = utils.ScriptRunner(hostname)

            man_path = os.path.join(config['HOST_DETAILS'][hostname]['tmpdir'],
                                    basedefs.PUPPET_MANIFEST_RELATIVE,
                                    manifest)

            running_logfile = "%s.running" % man_path
            finished_logfile = "%s.finished" % man_path
            currently_running.append((hostname, finished_logfile))

            server.append("touch %s" % running_logfile)
            server.append("chmod 600 %s" % running_logfile)
            server.append("export PACKSTACK_VAR_DIR=%s" % host_dir)
            cmd = ("( flock %s/ps.lock "
                   "puppet apply %s --modulepath %s/modules %s > %s "
                   "2>&1 < /dev/null ; "
                   "mv %s %s ) > /dev/null 2>&1 < /dev/null &"
                   % (host_dir, loglevel, host_dir, man_path, running_logfile,
                      running_logfile, finished_logfile))
            server.append(cmd)
            server.execute(log=logcmd)

    # wait for outstanding puppet runs befor exiting
    wait_for_puppet(currently_running, messages)

def finalize(config, messages):
    for hostname in filtered_hosts(config):
        server = utils.ScriptRunner(hostname)
        server.append("installed=$(rpm -q kernel --last | head -n1 | "
                      "sed 's/kernel-\([a-z0-9\.\_\-]*\).*/\\1/g')")
        server.append("loaded=$(uname -r | head -n1)")
        server.append('[ "$loaded" == "$installed" ]')
        try:
            rc, out = server.execute()
        except ScriptRuntimeError:
            messages.append('Because of the kernel update the host %s '
                            'requires reboot.' % hostname)
<commit_msg>Install and update packages required by packstack<commit_after># -*- coding: utf-8 -*-

"""
Installs and configures puppet
"""

import sys
import logging
import os
import platform
import time

from packstack.installer import utils
from packstack.installer import basedefs, output_messages
from packstack.installer.exceptions import ScriptRuntimeError, PuppetError

from packstack.modules.common import filtered_hosts
from packstack.modules.ospluginutils import manifestfiles
from packstack.modules.puppet import scan_logfile, validate_logfile

#------------------ oVirt installer initialization ------------------

PLUGIN_NAME = "Puppet"
PLUGIN_NAME_COLORED = utils.color_text(PLUGIN_NAME, 'blue')

PUPPET_DIR = os.environ.get('PACKSTACK_PUPPETDIR',
                            '/usr/share/openstack-puppet/')
MODULE_DIR = os.path.join(PUPPET_DIR, 'modules')

def initConfig(controller):
    group = {"GROUP_NAME": "PUPPET",
             "DESCRIPTION": "Puppet Config parameters",
             "PRE_CONDITION": lambda x: 'yes',
             "PRE_CONDITION_MATCH": "yes",
             "POST_CONDITION": False,
             "POST_CONDITION_MATCH": True}
    controller.addGroup(group, [])

def initSequences(controller):
    puppetpresteps = [
        {'title': 'Clean Up', 'functions': [run_cleanup]},
    ]
    controller.insertSequence("Clean Up", [], [], puppetpresteps, index=0)

    puppetsteps = [
        {'title': 'Installing Dependencies',
            'functions': [install_deps]},
        {'title': 'Copying Puppet modules and manifests',
            'functions': [copy_puppet_modules]},
        {'title': 'Applying Puppet manifests',
            'functions': [apply_puppet_manifest]},
        {'title': 'Finalizing',
            'functions': [finalize]}
    ]
    controller.addSequence("Puppet", [], [], puppetsteps)

#------------------------- helper functions -------------------------

def wait_for_puppet(currently_running, messages):
    log_len = 0
    twirl = ["-", "\\", "|", "/"]
    while currently_running:
        for hostname, finished_logfile in currently_running:
            log_file = os.path.splitext(os.path.basename(finished_logfile))[0]
            space_len = basedefs.SPACE_LEN - len(log_file)
            if len(log_file) > log_len:
                log_len = len(log_file)
            if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
                twirl = twirl[-1:] + twirl[:-1]
                sys.stdout.write(("\rTesting if puppet apply is finished: %s"
                                 % log_file).ljust(40 + log_len))
                sys.stdout.write("[ %s ]" % twirl[0])
                sys.stdout.flush()
            try:
                # Once a remote puppet run has finished, we retrieve the log
                # file and check it for errors
                local_server = utils.ScriptRunner()
                log = os.path.join(basedefs.PUPPET_MANIFEST_DIR,
                                   os.path.basename(finished_logfile))
                log = log.replace(".finished", ".log")
                local_server.append('scp -o StrictHostKeyChecking=no '
                                    '-o UserKnownHostsFile=/dev/null '
                                    'root@%s:%s %s'
                                    % (hostname, finished_logfile, log))
                # To not pollute logs we turn of logging of command execution
                local_server.execute(log=False)

                # If we got to this point the puppet apply has finished
                currently_running.remove((hostname, finished_logfile))

                # clean off the last "testing apply" msg
                if hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
                    sys.stdout.write(("\r").ljust(45 + log_len))

            except ScriptRuntimeError:
                # the test raises an exception if the file doesn't exist yet
                # TO-DO: We need to start testing 'e' for unexpected exceptions
                time.sleep(3)
                continue

            # check log file for relevant notices
            messages.extend(scan_logfile(log))

            # check the log file for errors
            sys.stdout.write('\r')
            try:
                validate_logfile(log)
                state = utils.state_message('%s:' % log_file, 'DONE', 'green')
                sys.stdout.write('%s\n' % state)
                sys.stdout.flush()
            except PuppetError:
                state = utils.state_message('%s:' % log_file, 'ERROR', 'red')
                sys.stdout.write('%s\n' % state)
                sys.stdout.flush()
                raise

#-------------------------- step functions --------------------------

def run_cleanup(config, messages):
    localserver = utils.ScriptRunner()
    localserver.append("rm -rf %s/*pp" % basedefs.PUPPET_MANIFEST_DIR)
    localserver.execute()

def install_deps(config, messages):
    deps = ["puppet", "openssh-clients", "tar", "nc"]
    modules_pkg = 'openstack-puppet-modules'

    local = utils.ScriptRunner()
    local.append('rpm -q --requires %s | egrep -v "^(rpmlib|\/|perl)"'
                 % modules_pkg)

    # This can fail if there are no dependencies other than those
    # filtered out by the egrep expression.
    rc, modules_deps = local.execute(can_fail=False)

    # Modules package might not be installed if we are running from source.
    # In this case we assume user knows what (s)he's doing and we don't
    # install modules dependencies
    if ('%s is not installed' % modules_pkg) not in modules_deps:
        modules_deps = [i.strip() for i in modules_deps.split() if i.strip()]
        deps.extend(modules_deps)

    for hostname in filtered_hosts(config):
        server = utils.ScriptRunner(hostname)
        packages = ' '.join(deps)
        server.append("yum install -y %s"
                          % packages)
        server.append("yum update -y %s"
                          % packages)
        server.execute()

def copy_puppet_modules(config, messages):
    os_modules = ' '.join(('apache', 'ceilometer', 'certmonger', 'cinder',
                           'concat', 'firewall', 'glance', 'heat', 'horizon',
                           'inifile', 'keystone', 'memcached', 'mongodb',
                           'mysql', 'neutron', 'nova', 'nssdb', 'openstack',
                           'packstack', 'qpid', 'rabbitmq', 'remote', 'rsync',
                           'ssh', 'stdlib', 'swift', 'sysctl', 'tempest',
                           'vcsrepo', 'vlan', 'vswitch', 'xinetd'))

        # write puppet manifest to disk
    manifestfiles.writeManifests()

    server = utils.ScriptRunner()
    for hostname in filtered_hosts(config):
        host_dir = config['HOST_DETAILS'][hostname]['tmpdir']
        # copy Packstack manifests
        server.append("cd %s/puppet" % basedefs.DIR_PROJECT_DIR)
        server.append("cd %s" % basedefs.PUPPET_MANIFEST_DIR)
        server.append("tar --dereference -cpzf - ../manifests | "
                      "ssh -o StrictHostKeyChecking=no "
                      "-o UserKnownHostsFile=/dev/null "
                      "root@%s tar -C %s -xpzf -" % (hostname, host_dir))

        # copy resources
        resources = config.get('RESOURCES', {})
        for path, localname in resources.get(hostname, []):
            server.append("scp -o StrictHostKeyChecking=no "
                          "-o UserKnownHostsFile=/dev/null "
                          "%s root@%s:%s/resources/%s" %
                          (path, hostname, host_dir, localname))

        # copy Puppet modules required by Packstack
        server.append("cd %s" % MODULE_DIR)
        server.append("tar --dereference -cpzf - %s | "
                      "ssh -o StrictHostKeyChecking=no "
                      "-o UserKnownHostsFile=/dev/null "
                      "root@%s tar -C %s -xpzf -" %
                      (os_modules, hostname,
                       os.path.join(host_dir, 'modules')))
    server.execute()

def apply_puppet_manifest(config, messages):
    if config.get("DRY_RUN"):
        return
    currently_running = []
    lastmarker = None
    loglevel = ''
    logcmd = False
    if logging.root.level <= logging.DEBUG:
        loglevel = '--debug'
        logcmd = True
    for manifest, marker in manifestfiles.getFiles():
        # if the marker has changed then we don't want to proceed until
        # all of the previous puppet runs have finished
        if lastmarker is not None and lastmarker != marker:
            wait_for_puppet(currently_running, messages)
        lastmarker = marker

        for hostname in filtered_hosts(config):
            if "%s_" % hostname not in manifest:
                continue

            host_dir = config['HOST_DETAILS'][hostname]['tmpdir']
            print "Applying %s" % manifest
            server = utils.ScriptRunner(hostname)

            man_path = os.path.join(config['HOST_DETAILS'][hostname]['tmpdir'],
                                    basedefs.PUPPET_MANIFEST_RELATIVE,
                                    manifest)

            running_logfile = "%s.running" % man_path
            finished_logfile = "%s.finished" % man_path
            currently_running.append((hostname, finished_logfile))

            server.append("touch %s" % running_logfile)
            server.append("chmod 600 %s" % running_logfile)
            server.append("export PACKSTACK_VAR_DIR=%s" % host_dir)
            cmd = ("( flock %s/ps.lock "
                   "puppet apply %s --modulepath %s/modules %s > %s "
                   "2>&1 < /dev/null ; "
                   "mv %s %s ) > /dev/null 2>&1 < /dev/null &"
                   % (host_dir, loglevel, host_dir, man_path, running_logfile,
                      running_logfile, finished_logfile))
            server.append(cmd)
            server.execute(log=logcmd)

    # wait for outstanding puppet runs befor exiting
    wait_for_puppet(currently_running, messages)

def finalize(config, messages):
    for hostname in filtered_hosts(config):
        server = utils.ScriptRunner(hostname)
        server.append("installed=$(rpm -q kernel --last | head -n1 | "
                      "sed 's/kernel-\([a-z0-9\.\_\-]*\).*/\\1/g')")
        server.append("loaded=$(uname -r | head -n1)")
        server.append('[ "$loaded" == "$installed" ]')
        try:
            rc, out = server.execute()
        except ScriptRuntimeError:
            messages.append('Because of the kernel update the host %s '
                            'requires reboot.' % hostname)
<|endoftext|>


<commit_before>import inspect

from marshmallow import Schema, SchemaOpts, fields, post_load

from marshpillow.relationship import Relationship
import inflection

# TODO: Ability to add relationships without relationship string interpretation
# TODO: Wrap model collections in a class such that __getitem__ will fullfill the relationship...
# TODO: Inherit fields and relationships from super class
# TODO: Automatically load class when relationship is fullfilled so you don't have to code in cls.load(r) in the Base class you use
class MarshpillowError(Exception):
    """ Generic MarshpillowException """

class MarshpillowInitializerError(Exception):
    """ Generic initializer exception """

# TODO: Rename "RELATIONSHIPS", "FIELDS" to something shorter
# TODO: Raw isn't loading for many fields.Nested relationships
def validate_init(fxn):
    """ Raises errors for dynamically generated __init__ definitions """

    def wrapped(*args, **kwargs):
        self = args[0]
        args = args[1:]
        fields = self.__class__.FIELDS
        field_dict = {}
        if len(args) > len(fields):
            raise TypeError("Expected {0} arguments but got {1} for {2}.__init__".format(
                    len(fields), len(args), self.__class__.__name__))
        for i, a in enumerate(args):
            f = fields[i]
            field_dict[f] = a
        for k, v in kwargs.items():
            if k in field_dict:
                raise MarshpillowInitializerError("Got multiple values for {0}".format(k))
            else:
                field_dict[k] = v
        for f in fields:
            if f not in field_dict.keys():
                raise MarshpillowInitializerError("Missing argument for {0}".format(f))
        return fxn(self, (), **field_dict)

    return wrapped

class BaseSchemaOpts(SchemaOpts):
    """ Custom Schema options """

    def __init__(self, meta):
        super().__init__(meta)
        self.allow_none = getattr(meta, 'allow_none', ())

class BaseSchema(Schema):
    OPTIONS_CLASS = BaseSchemaOpts

    def on_bind_field(self, field_name, field_obj):
        """ dynamically allow_none for fields in the "allow_none" list in SchemaMeta """
        super().on_bind_field(field_name, field_obj)
        if field_name in self.opts.allow_none:
            field_obj.allow_none = True

def add_schema(cls, *args, **kwargs):
    """ Decorator that adds a dynamically generated schema to a model """

    # add model to Base.models
    MarshpillowBase.models[cls.__name__] = cls

    # automatically generated Schema
    class ModSchema(BaseSchema):

        relationships = {}

        class Meta:
            if hasattr(cls, "ALLOW_NONE"):
                allow_none = cls.ALLOW_NONE
            additional = cls.FIELDS
            ordered = True

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._clone_fields_from_model(cls)
            self._load_one_and_many()

        def _save_relationship(self, relation):
            self.__class__.relationships[relation.name] = relation
            if relation.attr1 not in self.fields:
                self.fields[relation.attr1] = fields.Raw()

        def _attach_model_to_relation(self, relation):
            if type(relation.mod2) is str:
                relation.mod2 = cls.get_model_by_name(relation.mod2)

        def _load_one_and_many(self):
            """ create nested fields from ONE and MANY """
            """
                Types of relationships:
                    ["Address"]
                    SmartRelation(mod1, attr1, etc.)
            """
            if hasattr(cls, "RELATIONSHIPS"):
                for relation in cls.RELATIONSHIPS:
                    model = None
                    attribute_name = None
                    many = False
                    if type(relation) is str:
                        attribute_name = relation
                        model = cls.get_model_by_name(attribute_name)
                    elif issubclass(relation.__class__, Relationship):
                        self._attach_model_to_relation(relation)
                        self._save_relationship(relation)
                        if relation.many:
                            many = True
                        model = relation.mod2
                        attribute_name = relation.name
                    self.fields[inflection.underscore(attribute_name)] = fields.Nested(model.Schema, many=many)

        def _clone_fields_from_model(self, model):
            """ Clone fields defined in model to model.Schema """
            for field_name, field in model.model_fields():
                self.fields[field_name] = field

        @post_load
        def make(self, data):
            try:
                model = cls(**data)
                return model
            except MarshpillowInitializerError as e:
                pass

    # end of ModeSchema

    cls.Schema = ModSchema
    return cls

class MarshpillowBase(object):
    """ Basic model for api items """

    Schema = None
    models = {}
    unmarshall = "_unmarshall"

    @validate_init
    def __init__(self, *args, **kwargs):
        vars(self).update(kwargs)
        self.raw = None
        self.__class__.check_for_schema()
        self._unmarshall = False

    @classmethod
    def check_for_schema(cls):
        if not hasattr(cls, "Schema") or cls.Schema is None:
            raise MarshpillowError("Schema not found. @add_schema may not have been added to class definition.")

    def __getattribute__(self, name):
        schema_cls = object.__getattribute__(self, "Schema")
        unmarshal = object.__getattribute__(self, "_unmarshall")
        x = object.__getattribute__(self, name)
        if name in schema_cls.relationships:
            if unmarshal:
                r = schema_cls.relationships[name]
                if type(x) is r.mod2:
                    return x
                else:
                    new_x = self.fullfill_relationship(name)
                    if new_x is not None and new_x != []:
                        x = new_x
        return x

    def __getattr__(self, name, saveattr=False):
        # print("Getting {} from {}".format(name, self.__class__.__name__))
        schema_cls = object.__getattribute__(self, "Schema")

        if name in schema_cls.relationships:
            # print("fullfilling relationship...")
            v = self.fullfill_relationship(name)
            if saveattr:
                setattr(self, name, v)
            return v
        v = object.__getattribute__(self, name)
        return v

    def _get_relationship(self, name):
        return self.Schema.relationships[name]

    def _has_relationship(self, name):
        schema_cls = object.__getattribute__(self, "Schema")
        return name in schema_cls.relationships

    @classmethod
    def model_fields(cls):
        members = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
        return [m for m in members if issubclass(m[1].__class__, fields.Field)]

    # TODO: Propogate attributes
    def fullfill_relationship(self, relationship_name):
        """
        Fullfills a relationship using "using", "ref", "fxn."

        Sample
            Promise("sample_type", <SampleType>, "sample_type_id", "find")
        """
        relationship = self._get_relationship(relationship_name)
        self._lock_unmarshalling()
        x = relationship.fullfill(self)
        self._unlock_unmarshalling()
        return x

    # def _parse_model_from_name(self, name):
    #     model_name = inflection.camelize(name)
    #     model = Base.models[model_name]
    #     model_id = getattr(self, name+"_id")
    #     m = model.find(model_id)
    #     setattr(self, name, m)
    #     return m

    @classmethod
    def to_model(cls, result, additional_opts=None):
        opts = {}
        opts.update(cls.get_schema_opts())
        if additional_opts is not None:
            opts.update(additional_opts)
        schema = cls.Schema(**opts)
        model, errors = schema.load(result)
        return model

    @classmethod
    def get_model_by_name(cls, name):
        """ Converts a snake_case model name to the model object """
        model_name = inflection.camelize(name)  # class name of the model to use
        model = cls.models[model_name]
        return model

    @classmethod
    def get_schema_opts(cls):
        if hasattr(cls, "SCHEMA_OPTS"):
            return cls.SCHEMA_OPTS
        else:
            return {}

    @classmethod
    def json_to_model(cls, data):
        m = cls.to_model(data)
        m.raw = data
        cls._unlock_unmarshalling(m)
        return m

    @classmethod
    def json_to_models(cls, data):
        m = cls.to_model(data, {"many": True})
        for r, model in zip(data, m):
            model.raw = r
            cls._unlock_unmarshalling(model)
        return m

    def _lock_unmarshalling(self):
        object.__setattr__(self, MarshpillowBase.unmarshall, False)

    def _unlock_unmarshalling(self):
        object.__setattr__(self, MarshpillowBase.unmarshall, True)

    # def fullfill(self, name, relationship):
    #     if relationship.reference is None:
    #         # get from _id
    #         x = name+"_id"
    #         if hasattr(self, x):
    #             model_id = getattr(self, x)

    @classmethod
    def find(cls, id):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("find", cls.__name__))

    @classmethod
    def where(cls, *args, **kwargs):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("where", cls.__name__))

    @classmethod
    def all(cls):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("all", cls.__name__))

    @classmethod
    def find_by_name(cls, name):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("find_by_name", cls.__name__))

    def _propogate_attributes(self):
        """ Propogates attributes forward for vanilla Marshmallow objects and fullfilled relationship """

    # TODO: forward propogate properties if there is a relationship...from
    @classmethod
    def load(cls, data):
        """ Special load that will unmarshall dict objects or a list of dict objects """
        cls.check_for_schema()
        if type(data) is list:
            return cls.json_to_models(data)
        elif type(data) is dict:
            return cls.json_to_model(data)
        else:
            raise MarshpillowError("Data not recognized. Supply a dict or list: \"{0}\"".format(data))

    # TODO: Force unmarshalling of all or some of the relationships...
    def force(self):
        raise NotImplementedError("Force is not yet implemented")
<commit_msg>removed print statements<commit_after>import inspect

from marshmallow import Schema, SchemaOpts, fields, post_load

from marshpillow.relationship import Relationship
import inflection

# TODO: Ability to add relationships without relationship string interpretation
# TODO: Wrap model collections in a class such that __getitem__ will fullfill the relationship...
# TODO: Inherit fields and relationships from super class
# TODO: Automatically load class when relationship is fullfilled so you don't have to code in cls.load(r) in the Base class you use
class MarshpillowError(Exception):
    """ Generic MarshpillowException """

class MarshpillowInitializerError(Exception):
    """ Generic initializer exception """

# TODO: Rename "RELATIONSHIPS", "FIELDS" to something shorter
# TODO: Raw isn't loading for many fields.Nested relationships
def validate_init(fxn):
    """ Raises errors for dynamically generated __init__ definitions """

    def wrapped(*args, **kwargs):
        self = args[0]
        args = args[1:]
        fields = self.__class__.FIELDS
        field_dict = {}
        if len(args) > len(fields):
            raise TypeError("Expected {0} arguments but got {1} for {2}.__init__".format(
                    len(fields), len(args), self.__class__.__name__))
        for i, a in enumerate(args):
            f = fields[i]
            field_dict[f] = a
        for k, v in kwargs.items():
            if k in field_dict:
                raise MarshpillowInitializerError("Got multiple values for {0}".format(k))
            else:
                field_dict[k] = v
        for f in fields:
            if f not in field_dict.keys():
                raise MarshpillowInitializerError("Missing argument for {0}".format(f))
        return fxn(self, (), **field_dict)

    return wrapped

class BaseSchemaOpts(SchemaOpts):
    """ Custom Schema options """

    def __init__(self, meta):
        super().__init__(meta)
        self.allow_none = getattr(meta, 'allow_none', ())

class BaseSchema(Schema):
    OPTIONS_CLASS = BaseSchemaOpts

    def on_bind_field(self, field_name, field_obj):
        """ dynamically allow_none for fields in the "allow_none" list in SchemaMeta """
        super().on_bind_field(field_name, field_obj)
        if field_name in self.opts.allow_none:
            field_obj.allow_none = True

def add_schema(cls, *args, **kwargs):
    """ Decorator that adds a dynamically generated schema to a model """

    # add model to Base.models
    MarshpillowBase.models[cls.__name__] = cls

    # automatically generated Schema
    class ModSchema(BaseSchema):

        relationships = {}

        class Meta:
            if hasattr(cls, "ALLOW_NONE"):
                allow_none = cls.ALLOW_NONE
            additional = cls.FIELDS
            ordered = True

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._clone_fields_from_model(cls)
            self._load_one_and_many()

        def _save_relationship(self, relation):
            self.__class__.relationships[relation.name] = relation
            if relation.attr1 not in self.fields:
                self.fields[relation.attr1] = fields.Raw()

        def _attach_model_to_relation(self, relation):
            if type(relation.mod2) is str:
                relation.mod2 = cls.get_model_by_name(relation.mod2)

        def _load_one_and_many(self):
            """ create nested fields from ONE and MANY """
            """
                Types of relationships:
                    ["Address"]
                    SmartRelation(mod1, attr1, etc.)
            """
            if hasattr(cls, "RELATIONSHIPS"):
                for relation in cls.RELATIONSHIPS:
                    model = None
                    attribute_name = None
                    many = False
                    if type(relation) is str:
                        attribute_name = relation
                        model = cls.get_model_by_name(attribute_name)
                    elif issubclass(relation.__class__, Relationship):
                        self._attach_model_to_relation(relation)
                        self._save_relationship(relation)
                        if relation.many:
                            many = True
                        model = relation.mod2
                        attribute_name = relation.name
                    self.fields[inflection.underscore(attribute_name)] = fields.Nested(model.Schema, many=many)

        def _clone_fields_from_model(self, model):
            """ Clone fields defined in model to model.Schema """
            for field_name, field in model.model_fields():
                self.fields[field_name] = field

        @post_load
        def make(self, data):
            try:
                model = cls(**data)
                return model
            except MarshpillowInitializerError as e:
                pass

    # end of ModeSchema

    cls.Schema = ModSchema
    return cls

class MarshpillowBase(object):
    """ Basic model for api items """

    Schema = None
    models = {}
    unmarshall = "_unmarshall"

    @validate_init
    def __init__(self, *args, **kwargs):
        vars(self).update(kwargs)
        self.raw = None
        self.__class__.check_for_schema()
        self._unmarshall = False

    @classmethod
    def check_for_schema(cls):
        if not hasattr(cls, "Schema") or cls.Schema is None:
            raise MarshpillowError("Schema not found. @add_schema may not have been added to class definition.")

    def __getattribute__(self, name):
        schema_cls = object.__getattribute__(self, "Schema")
        unmarshal = object.__getattribute__(self, "_unmarshall")
        x = object.__getattribute__(self, name)
        if name in schema_cls.relationships:
            if unmarshal:
                r = schema_cls.relationships[name]
                if type(x) is r.mod2:
                    return x
                else:
                    new_x = self.fullfill_relationship(name)
                    if new_x is not None and new_x != []:
                        x = new_x
        return x

    def __getattr__(self, name, saveattr=False):
        schema_cls = object.__getattribute__(self, "Schema")
        if name in schema_cls.relationships:
            v = self.fullfill_relationship(name)
            if saveattr:
                setattr(self, name, v)
            return v
        v = object.__getattribute__(self, name)
        return v

    def _get_relationship(self, name):
        return self.Schema.relationships[name]

    def _has_relationship(self, name):
        schema_cls = object.__getattribute__(self, "Schema")
        return name in schema_cls.relationships

    @classmethod
    def model_fields(cls):
        members = inspect.getmembers(cls, lambda a: not (inspect.isroutine(a)))
        return [m for m in members if issubclass(m[1].__class__, fields.Field)]

    # TODO: Propogate attributes
    def fullfill_relationship(self, relationship_name):
        """
        Fullfills a relationship using "using", "ref", "fxn."

        Sample
            Promise("sample_type", <SampleType>, "sample_type_id", "find")
        """
        relationship = self._get_relationship(relationship_name)
        self._lock_unmarshalling()
        x = relationship.fullfill(self)
        self._unlock_unmarshalling()
        return x

    # def _parse_model_from_name(self, name):
    #     model_name = inflection.camelize(name)
    #     model = Base.models[model_name]
    #     model_id = getattr(self, name+"_id")
    #     m = model.find(model_id)
    #     setattr(self, name, m)
    #     return m

    @classmethod
    def to_model(cls, result, additional_opts=None):
        opts = {}
        opts.update(cls.get_schema_opts())
        if additional_opts is not None:
            opts.update(additional_opts)
        schema = cls.Schema(**opts)
        model, errors = schema.load(result)
        return model

    @classmethod
    def get_model_by_name(cls, name):
        """ Converts a snake_case model name to the model object """
        model_name = inflection.camelize(name)  # class name of the model to use
        model = cls.models[model_name]
        return model

    @classmethod
    def get_schema_opts(cls):
        if hasattr(cls, "SCHEMA_OPTS"):
            return cls.SCHEMA_OPTS
        else:
            return {}

    @classmethod
    def json_to_model(cls, data):
        m = cls.to_model(data)
        m.raw = data
        cls._unlock_unmarshalling(m)
        return m

    @classmethod
    def json_to_models(cls, data):
        m = cls.to_model(data, {"many": True})
        for r, model in zip(data, m):
            model.raw = r
            cls._unlock_unmarshalling(model)
        return m

    def _lock_unmarshalling(self):
        object.__setattr__(self, MarshpillowBase.unmarshall, False)

    def _unlock_unmarshalling(self):
        object.__setattr__(self, MarshpillowBase.unmarshall, True)

    # def fullfill(self, name, relationship):
    #     if relationship.reference is None:
    #         # get from _id
    #         x = name+"_id"
    #         if hasattr(self, x):
    #             model_id = getattr(self, x)

    @classmethod
    def find(cls, id):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("find", cls.__name__))

    @classmethod
    def where(cls, *args, **kwargs):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("where", cls.__name__))

    @classmethod
    def all(cls):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("all", cls.__name__))

    @classmethod
    def find_by_name(cls, name):
        raise NotImplementedError("method \"{0}\" is not yet implemented for {1}.".format("find_by_name", cls.__name__))

    def _propogate_attributes(self):
        """ Propogates attributes forward for vanilla Marshmallow objects and fullfilled relationship """

    # TODO: forward propogate properties if there is a relationship...from
    @classmethod
    def load(cls, data):
        """ Special load that will unmarshall dict objects or a list of dict objects """
        cls.check_for_schema()
        if type(data) is list:
            return cls.json_to_models(data)
        elif type(data) is dict:
            return cls.json_to_model(data)
        else:
            raise MarshpillowError("Data not recognized. Supply a dict or list: \"{0}\"".format(data))

    # TODO: Force unmarshalling of all or some of the relationships...
    def force(self):
        raise NotImplementedError("Force is not yet implemented")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os

from modularodm import fields
from boto.exception import BotoServerError

from framework.auth.core import Auth

from website.addons.base import exceptions
from website.addons.base import AddonUserSettingsBase, AddonNodeSettingsBase, GuidFile
from website.addons.s3.utils import get_bucket_drop_down, remove_osf_user, build_urls

class S3GuidFile(GuidFile):

    path = fields.StringField(index=True)

    @property
    def file_url(self):
        if self.path is None:
            raise ValueError('Path field must be defined.')
        return os.path.join('s3', self.path)

class AddonS3UserSettings(AddonUserSettingsBase):

    s3_osf_user = fields.StringField()
    access_key = fields.StringField()
    secret_key = fields.StringField()

    def to_json(self, user):
        rv = super(AddonS3UserSettings, self).to_json(user)
        rv['has_auth'] = self.has_auth
        return rv

    @property
    def has_auth(self):
        return bool(self.access_key and self.secret_key)

    def remove_iam_user(self):
        """Remove IAM user from Amazon.

        :return: True if successful, False if failed with expected error;
            else uncaught exception is raised

        """
        try:
            remove_osf_user(self)
            return True
        except BotoServerError as error:
            if error.code in ['InvalidClientTokenId', 'ValidationError', 'AccessDenied']:
                return False
            raise

    def revoke_auth(self, save=False):
        for node_settings in self.addons3nodesettings__authorized:
            node_settings.deauthorize(save=True)
        rv = self.remove_iam_user() if self.has_auth else True
        self.s3_osf_user, self.access_key, self.secret_key = None, None, None

        if save:
            self.save()
        return rv

    def delete(self, save=True):
        self.revoke_auth(save=False)
        super(AddonS3UserSettings, self).delete(save=save)

class AddonS3NodeSettings(AddonNodeSettingsBase):

    registration_data = fields.DictionaryField()
    bucket = fields.StringField()
    user_settings = fields.ForeignField(
        'addons3usersettings', backref='authorized'
    )

    @property
    def display_name(self):
        return u'{0}: {1}'.format(self.config.full_name, self.bucket)

    def authorize(self, user_settings, save=False):
        self.user_settings = user_settings
        self.owner.add_log(
            action='s3_node_authorized',
            params={
                'project': self.owner.parent_id,
                'node': self.owner._id,
            },
            auth=Auth(user_settings.owner),
        )
        if save:
            self.save()

    def deauthorize(self, auth=None, log=True, save=False):
        self.registration_data = {}
        self.bucket = None
        self.user_settings = None

        if log:
            self.owner.add_log(
                action='s3_node_deauthorized',
                params={
                    'project': self.owner.parent_id,
                    'node': self.owner._id,
                },
                auth=auth,
            )
        if save:
            self.save()

    def delete(self, save=True):
        self.deauthorize(log=False, save=False)
        super(AddonS3NodeSettings, self).delete(save=save)

    def serialize_waterbutler_credentials(self):
        if not self.has_auth:
            raise exceptions.AddonError('Cannot serialize credentials for S3 addon')
        return {
            'access_key': self.user_settings.access_key,
            'secret_key': self.user_settings.secret_key,
        }

    def serialize_waterbutler_settings(self):
        if not self.bucket:
            raise exceptions.AddonError('Cannot serialize settings for S3 addon')
        return {'bucket': self.bucket}

    def create_waterbutler_log(self, auth, action, metadata):
        self.owner.add_log(
            's3_{0}'.format(action),
            auth=auth,
            params={
                'project': self.owner.parent_id,
                'node': self.owner._id,
                'path': metadata['path'],
                'bucket': self.bucket,
                'urls': build_urls(self.owner, metadata['path']),
            },
        )

    def get_waterbutler_render_url(self, path, **kwargs):
        return self.owner.web_url_for('s3_view', path=path)

    def to_json(self, user):
        rv = super(AddonS3NodeSettings, self).to_json(user)

        user_settings = user.get_addon('s3')

        rv.update({
            'bucket': self.bucket or '',
            'has_bucket': self.bucket is not None,
            'user_is_owner': (
                self.user_settings and self.user_settings.owner == user
            ),
            'user_has_auth': bool(user_settings) and user_settings.has_auth,
            'node_has_auth': self.has_auth,
            'owner': None,
            'bucket_list': None,
            'is_registration': self.owner.is_registration,
        })

        if self.has_auth:
            rv['owner'] = self.user_settings.owner.fullname
            rv['owner_url'] = self.user_settings.owner.url
            rv['bucket_list'] = get_bucket_drop_down(self.user_settings)
            rv['node_has_auth'] = True

        return rv

    @property
    def is_registration(self):
        return True if self.registration_data else False

    @property
    def has_auth(self):
        return self.user_settings and self.user_settings.has_auth
        #TODO Update callbacks

    def before_register(self, node, user):
        """

        :param Node node:
        :param User user:
        :return str: Alert message

        """
        category = node.project_or_component
        if self.user_settings and self.user_settings.has_auth:
            return (
                u'The contents of S3 add-ons cannot be registered at this time; '
                u'the S3 bucket linked to this {category} will not be included '
                u'as part of this registration.'
            ).format(**locals())

    def after_fork(self, node, fork, user, save=True):
        """

        :param Node node: Original node
        :param Node fork: Forked node
        :param User user: User creating fork
        :param bool save: Save settings after callback
        :return tuple: Tuple of cloned settings and alert message

        """
        clone, _ = super(AddonS3NodeSettings, self).after_fork(
            node, fork, user, save=False
        )

        # Copy authentication if authenticated by forking user
        if self.user_settings and self.user_settings.owner == user:
            clone.user_settings = self.user_settings
            clone.bucket = self.bucket
            message = (
                'Amazon Simple Storage authorization copied to forked {cat}.'
            ).format(
                cat=fork.project_or_component,
            )
        else:
            message = (
                'Amazon Simple Storage authorization not copied to forked {cat}. You may '
                'authorize this fork on the Settings '
                'page.'
            ).format(
                cat=fork.project_or_component,
                url=fork.url + 'settings/'
            )

        if save:
            clone.save()

        return clone, message

    def before_fork(self, node, user):
        """

        :param Node node:
        :param User user:
        :return str: Alert message

        """

        if self.user_settings and self.user_settings.owner == user:
            return (
                'Because you have authenticated the S3 add-on for this '
                '{cat}, forking it will also transfer your authorization to '
                'the forked {cat}.'
            ).format(
                cat=node.project_or_component,
            )
        return (
            'Because this S3 add-on has been authenticated by a different '
            'user, forking it will not transfer authentication to the forked '
            '{cat}.'
        ).format(
            cat=node.project_or_component,
        )

    def before_remove_contributor(self, node, removed):
        """

        :param Node node:
        :param User removed:
        :return str: Alert message

        """
        if self.user_settings and self.user_settings.owner == removed:
            return (
                'The Amazon Simple Storage add-on for this {category} is authenticated '
                'by {user}. Removing this user will also remove access '
                'to {bucket} unless another contributor re-authenticates.'
            ).format(
                category=node.project_or_component,
                user=removed.fullname,
                bucket=self.bucket
            )

    def after_remove_contributor(self, node, removed):
        """

        :param Node node:
        :param User removed:
        :return str: Alert message

        """
        if self.user_settings and self.user_settings.owner == removed:
            self.user_settings = None
            self.bucket = None
            self.save()

            return (
                'Because the Amazon Simple Storage add-on for this project was authenticated '
                'by {user}, authentication information has been deleted. You '
                'can re-authenticate on the '
                'Settings page.'.format(
                    user=removed.fullname,
                    url=node.url,
                )
            )

    def after_delete(self, node, user):
        self.deauthorize(Auth(user=user), log=True, save=True)
<commit_msg>Update S3 file guid class<commit_after># -*- coding: utf-8 -*-

from modularodm import fields, Q
from modularodm.exceptions import ModularOdmException

from boto.exception import BotoServerError

from framework.auth.core import Auth

from website.addons.base import exceptions
from website.addons.base import AddonUserSettingsBase, AddonNodeSettingsBase, GuidFile
from website.addons.s3.utils import get_bucket_drop_down, remove_osf_user, build_urls

class S3GuidFile(GuidFile):

    path = fields.StringField(index=True)

    @property
    def provider(self):
        return 's3'

class AddonS3UserSettings(AddonUserSettingsBase):

    s3_osf_user = fields.StringField()
    access_key = fields.StringField()
    secret_key = fields.StringField()

    def to_json(self, user):
        rv = super(AddonS3UserSettings, self).to_json(user)
        rv['has_auth'] = self.has_auth
        return rv

    @property
    def has_auth(self):
        return bool(self.access_key and self.secret_key)

    def remove_iam_user(self):
        """Remove IAM user from Amazon.

        :return: True if successful, False if failed with expected error;
            else uncaught exception is raised

        """
        try:
            remove_osf_user(self)
            return True
        except BotoServerError as error:
            if error.code in ['InvalidClientTokenId', 'ValidationError', 'AccessDenied']:
                return False
            raise

    def revoke_auth(self, save=False):
        for node_settings in self.addons3nodesettings__authorized:
            node_settings.deauthorize(save=True)
        rv = self.remove_iam_user() if self.has_auth else True
        self.s3_osf_user, self.access_key, self.secret_key = None, None, None

        if save:
            self.save()
        return rv

    def delete(self, save=True):
        self.revoke_auth(save=False)
        super(AddonS3UserSettings, self).delete(save=save)

class AddonS3NodeSettings(AddonNodeSettingsBase):

    registration_data = fields.DictionaryField()
    bucket = fields.StringField()
    user_settings = fields.ForeignField(
        'addons3usersettings', backref='authorized'
    )

    def find_or_create_file_guid(self, path):
        try:
            return S3GuidFile.find_one(
                Q('path', 'eq', path) &
                Q('node', 'eq', self.owner)
            ), False
        except ModularOdmException:
            pass

        # Create new
        new = S3GuidFile(node=self.owner, path=path)
        new.save()
        return new, True

    @property
    def display_name(self):
        return u'{0}: {1}'.format(self.config.full_name, self.bucket)

    def authorize(self, user_settings, save=False):
        self.user_settings = user_settings
        self.owner.add_log(
            action='s3_node_authorized',
            params={
                'project': self.owner.parent_id,
                'node': self.owner._id,
            },
            auth=Auth(user_settings.owner),
        )
        if save:
            self.save()

    def deauthorize(self, auth=None, log=True, save=False):
        self.registration_data = {}
        self.bucket = None
        self.user_settings = None

        if log:
            self.owner.add_log(
                action='s3_node_deauthorized',
                params={
                    'project': self.owner.parent_id,
                    'node': self.owner._id,
                },
                auth=auth,
            )
        if save:
            self.save()

    def delete(self, save=True):
        self.deauthorize(log=False, save=False)
        super(AddonS3NodeSettings, self).delete(save=save)

    def serialize_waterbutler_credentials(self):
        if not self.has_auth:
            raise exceptions.AddonError('Cannot serialize credentials for S3 addon')
        return {
            'access_key': self.user_settings.access_key,
            'secret_key': self.user_settings.secret_key,
        }

    def serialize_waterbutler_settings(self):
        if not self.bucket:
            raise exceptions.AddonError('Cannot serialize settings for S3 addon')
        return {'bucket': self.bucket}

    def create_waterbutler_log(self, auth, action, metadata):
        self.owner.add_log(
            's3_{0}'.format(action),
            auth=auth,
            params={
                'project': self.owner.parent_id,
                'node': self.owner._id,
                'path': metadata['path'],
                'bucket': self.bucket,
                'urls': build_urls(self.owner, metadata['path']),
            },
        )

    def get_waterbutler_render_url(self, path, **kwargs):
        return self.owner.web_url_for('s3_view', path=path)

    def to_json(self, user):
        rv = super(AddonS3NodeSettings, self).to_json(user)

        user_settings = user.get_addon('s3')

        rv.update({
            'bucket': self.bucket or '',
            'has_bucket': self.bucket is not None,
            'user_is_owner': (
                self.user_settings and self.user_settings.owner == user
            ),
            'user_has_auth': bool(user_settings) and user_settings.has_auth,
            'node_has_auth': self.has_auth,
            'owner': None,
            'bucket_list': None,
            'is_registration': self.owner.is_registration,
        })

        if self.has_auth:
            rv['owner'] = self.user_settings.owner.fullname
            rv['owner_url'] = self.user_settings.owner.url
            rv['bucket_list'] = get_bucket_drop_down(self.user_settings)
            rv['node_has_auth'] = True

        return rv

    @property
    def is_registration(self):
        return True if self.registration_data else False

    @property
    def has_auth(self):
        return self.user_settings and self.user_settings.has_auth
        #TODO Update callbacks

    def before_register(self, node, user):
        """

        :param Node node:
        :param User user:
        :return str: Alert message

        """
        category = node.project_or_component
        if self.user_settings and self.user_settings.has_auth:
            return (
                u'The contents of S3 add-ons cannot be registered at this time; '
                u'the S3 bucket linked to this {category} will not be included '
                u'as part of this registration.'
            ).format(**locals())

    def after_fork(self, node, fork, user, save=True):
        """

        :param Node node: Original node
        :param Node fork: Forked node
        :param User user: User creating fork
        :param bool save: Save settings after callback
        :return tuple: Tuple of cloned settings and alert message

        """
        clone, _ = super(AddonS3NodeSettings, self).after_fork(
            node, fork, user, save=False
        )

        # Copy authentication if authenticated by forking user
        if self.user_settings and self.user_settings.owner == user:
            clone.user_settings = self.user_settings
            clone.bucket = self.bucket
            message = (
                'Amazon Simple Storage authorization copied to forked {cat}.'
            ).format(
                cat=fork.project_or_component,
            )
        else:
            message = (
                'Amazon Simple Storage authorization not copied to forked {cat}. You may '
                'authorize this fork on the Settings '
                'page.'
            ).format(
                cat=fork.project_or_component,
                url=fork.url + 'settings/'
            )

        if save:
            clone.save()

        return clone, message

    def before_fork(self, node, user):
        """

        :param Node node:
        :param User user:
        :return str: Alert message

        """

        if self.user_settings and self.user_settings.owner == user:
            return (
                'Because you have authenticated the S3 add-on for this '
                '{cat}, forking it will also transfer your authorization to '
                'the forked {cat}.'
            ).format(
                cat=node.project_or_component,
            )
        return (
            'Because this S3 add-on has been authenticated by a different '
            'user, forking it will not transfer authentication to the forked '
            '{cat}.'
        ).format(
            cat=node.project_or_component,
        )

    def before_remove_contributor(self, node, removed):
        """

        :param Node node:
        :param User removed:
        :return str: Alert message

        """
        if self.user_settings and self.user_settings.owner == removed:
            return (
                'The Amazon Simple Storage add-on for this {category} is authenticated '
                'by {user}. Removing this user will also remove access '
                'to {bucket} unless another contributor re-authenticates.'
            ).format(
                category=node.project_or_component,
                user=removed.fullname,
                bucket=self.bucket
            )

    def after_remove_contributor(self, node, removed):
        """

        :param Node node:
        :param User removed:
        :return str: Alert message

        """
        if self.user_settings and self.user_settings.owner == removed:
            self.user_settings = None
            self.bucket = None
            self.save()

            return (
                'Because the Amazon Simple Storage add-on for this project was authenticated '
                'by {user}, authentication information has been deleted. You '
                'can re-authenticate on the '
                'Settings page.'.format(
                    user=removed.fullname,
                    url=node.url,
                )
            )

    def after_delete(self, node, user):
        self.deauthorize(Auth(user=user), log=True, save=True)
<|endoftext|>


<commit_before>from snovault import (
    upgrade_step,
)
from encoded.schema_formats import is_uuid
from encoded.loadxl import read_single_sheet
from pkg_resources import resource_filename

def generate_content_map_for_page_1_2_upgrader():
    return {
        insert_item['name'] : insert_item.get('content', [])
        for insert_item in read_single_sheet(resource_filename('encoded', 'tests/data/master-inserts/'), 'page')
    }

@upgrade_step('page', '1', '2')
def page_1_2(value, system):
    '''Primarily, we delete old properties and add new property "content" from a map generated from page inserts.'''

    content_map = generate_content_map_for_page_1_2_upgrader()

    value['content'] = content_map.get(value['name'])

    if value['content'] is None:
        print('Page ' + value['name'] + ' is not in inserts or could not get "content" field from it. This is fine if intentional.')
        #raise Exception('Page ' + value['name'] + ' is not in inserts. Could not get "content" field.') # We should have an empty list, at very least (@see def generate_content_map_for_page_1_2_upgrader)

    if value.get('directory') is not None:
        del value['directory']

    if value.get('sections') is not None:
        del value['sections']

    if value.get('notification') is not None:
        del value['notification']

    if value.get('description') == '':
        del value['description']

@upgrade_step('page', '2', '3')
def page_2_3(value, system):
    '''Rename some pages, delete old unused properties'''

    # Moving Around Pages

    if value['name'] == 'help':
        value["title"] = "Help Section"
        value["description"] = "Documentation for using the 4DN Data Portal"
        value["children"] = [
            "f0f0f0f0-0000-0000-0000-aaaaaa000000",
            "f0f0f0f0-0000-0000-0000-aaaaaa000001",
            "f0f0f0f0-0000-0000-0000-aaaaaa000002",
            "f0f0f0f0-0000-0000-0000-aaaaaa000003",
            "f0f0f0f0-0000-0000-0000-aaaaaa000004",
            "f0f0f0f0-0000-0000-0000-aaaaaa000005"
        ]
        if value.get('table-of-contents') is not None:
            del value['table-of-contents']
        value['content'] = []

    if value['name'] == 'help/account-creation':
        value['name'] = "help/user-guide/account-creation"
        value['table-of-contents'] = {
            "enabled" : True
        }

    if value['name'] == "help/getting-started":
        value['name'] = "help/user-guide/getting-started"

    if value['name'] == "help/biosample":
        value['name'] = "help/user-guide/biosample-metadata"

    if value['name'] == "help/rest-api":
        value['name'] = "help/user-guide/rest-api"

    if value['name'] == "help/web-submission":
        value['name'] = "help/submitter-guide/web-submission"

    if value['name'] == "help/spreadsheet":
        value['name'] = "help/submitter-guide/spreadsheet"

    if value['name'] == "help/data-processing-pipelines":
        value['content'] = []
        value['redirect'] = { "target" : "/help/analysis/cwl-docker", "enabled" : True, "code" : 307 }
        if value.get('title') is not None:
            del value['title']

    # Remove old fields

    if value.get('table-of-contents') is not None:
        toc = value['table-of-contents']
        if toc.get('skip-depth') is not None:
            del toc['skip-depth']
        if toc.get('include-top-link') is not None:
            del toc['include-top-link']
        value['table-of-contents'] = toc

<commit_msg>Upgrader Edit<commit_after>from snovault import (
    upgrade_step,
)
from encoded.schema_formats import is_uuid
from encoded.loadxl import read_single_sheet
from pkg_resources import resource_filename

def generate_content_map_for_page_1_2_upgrader():
    return {
        insert_item['name'] : insert_item.get('content', [])
        for insert_item in read_single_sheet(resource_filename('encoded', 'tests/data/master-inserts/'), 'page')
    }

@upgrade_step('page', '1', '2')
def page_1_2(value, system):
    '''Primarily, we delete old properties and add new property "content" from a map generated from page inserts.'''

    if int(value.get('schema_version', '1')) >= 2:
        return

    content_map = generate_content_map_for_page_1_2_upgrader()

    value['content'] = content_map.get(value['name'])

    if value['content'] is None:
        print('Page ' + value['name'] + ' is not in inserts or could not get "content" field from it. This is fine if intentional.')
        #raise Exception('Page ' + value['name'] + ' is not in inserts. Could not get "content" field.') # We should have an empty list, at very least (@see def generate_content_map_for_page_1_2_upgrader)

    if value.get('directory') is not None:
        del value['directory']

    if value.get('sections') is not None:
        del value['sections']

    if value.get('notification') is not None:
        del value['notification']

    if value.get('description') == '':
        del value['description']

@upgrade_step('page', '2', '3')
def page_2_3(value, system):
    '''Rename some pages, delete old unused properties'''

    if int(value.get('schema_version', '1')) >= 3:
        return

    # Moving Around Pages

    if value['name'] == 'help':
        value["title"] = "Help Section"
        value["description"] = "Documentation for using the 4DN Data Portal"
        value["children"] = [
            "f0f0f0f0-0000-0000-0000-aaaaaa000000",
            "f0f0f0f0-0000-0000-0000-aaaaaa000001",
            "f0f0f0f0-0000-0000-0000-aaaaaa000002",
            "f0f0f0f0-0000-0000-0000-aaaaaa000003",
            "f0f0f0f0-0000-0000-0000-aaaaaa000004",
            "f0f0f0f0-0000-0000-0000-aaaaaa000005"
        ]
        if value.get('table-of-contents') is not None:
            del value['table-of-contents']
        value['content'] = []

    if value['name'] == 'help/account-creation':
        value['name'] = "help/user-guide/account-creation"
        value['table-of-contents'] = {
            "enabled" : True
        }

    if value['name'] == "help/getting-started":
        value['name'] = "help/user-guide/getting-started"

    if value['name'] == "help/biosample":
        value['name'] = "help/user-guide/biosample-metadata"

    if value['name'] == "help/rest-api":
        value['name'] = "help/user-guide/rest-api"

    if value['name'] == "help/web-submission":
        value['name'] = "help/submitter-guide/web-submission"

    if value['name'] == "help/spreadsheet":
        value['name'] = "help/submitter-guide/spreadsheet"

    if value['name'] == "help/data-processing-pipelines":
        value['content'] = []
        value['redirect'] = { "target" : "/help/analysis/cwl-docker", "enabled" : True, "code" : 307 }
        if value.get('title') is not None:
            del value['title']

    # Remove old fields

    if value.get('table-of-contents') is not None:
        toc = value['table-of-contents']
        if toc.get('skip-depth') is not None:
            del toc['skip-depth']
        if toc.get('include-top-link') is not None:
            del toc['include-top-link']
        value['table-of-contents'] = toc

<|endoftext|>