<commit_before>from __future__ import absolute_import, division, unicode_literals
from six import text_type

import re

from . import _base
from html5lib import ihatexml
from html5lib import constants
from html5lib.constants import namespaces
from html5lib.utils import moduleFactoryFactory

tag_regexp = re.compile("{([^}]*)}(.*)")

def getETreeBuilder(ElementTreeImplementation, fullTree=False):
    ElementTree = ElementTreeImplementation
    class Element(_base.Node):
        def __init__(self, name, namespace=None):
            self._name = name
            self._namespace = namespace
            self._element = ElementTree.Element(self._getETreeTag(name,
                                                                  namespace))
            if namespace is None:
                self.nameTuple = namespaces["html"], self._name
            else:
                self.nameTuple = self._namespace, self._name
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getETreeTag(self, name, namespace):
            if namespace is None:
                etree_tag = name
            else:
                etree_tag = "{%s}%s"%(namespace, name)
            return etree_tag

        def _setName(self, name):
            self._name = name
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getName(self):
            return self._name

        name = property(_getName, _setName)

        def _setNamespace(self, namespace):
            self._namespace = namespace
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getNamespace(self):
            return self._namespace

        namespace = property(_getNamespace, _setNamespace)

        def _getAttributes(self):
            return self._element.attrib

        def _setAttributes(self, attributes):
            #Delete existing attributes first
            #XXX - there may be a better way to do this...
            for key in list(self._element.attrib.keys()):
                del self._element.attrib[key]
            for key, value in attributes.items():
                if isinstance(key, tuple):
                    name = "{%s}%s"%(key[2], key[1])
                else:
                    name = key
                self._element.set(name, value)

        attributes = property(_getAttributes, _setAttributes)

        def _getChildNodes(self):
            return self._childNodes
        def _setChildNodes(self, value):
            del self._element[:]
            self._childNodes = []
            for element in value:
                self.insertChild(element)

        childNodes = property(_getChildNodes, _setChildNodes)

        def hasContent(self):
            """Return true if the node has children or text"""
            return bool(self._element.text or len(self._element))

        def appendChild(self, node):
            self._childNodes.append(node)
            self._element.append(node._element)
            node.parent = self

        def insertBefore(self, node, refNode):
            index = list(self._element).index(refNode._element)
            self._element.insert(index, node._element)
            node.parent = self

        def removeChild(self, node):
            self._element.remove(node._element)
            node.parent=None

        def insertText(self, data, insertBefore=None):
            if not(len(self._element)):
                if not self._element.text:
                    self._element.text = ""
                self._element.text += data
            elif insertBefore is None:
                #Insert the text as the tail of the last child element
                if not self._element[-1].tail:
                    self._element[-1].tail = ""
                self._element[-1].tail += data
            else:
                #Insert the text before the specified node
                children = list(self._element)
                index = children.index(insertBefore._element)
                if index > 0:
                    if not self._element[index-1].tail:
                        self._element[index-1].tail = ""
                    self._element[index-1].tail += data
                else:
                    if not self._element.text:
                        self._element.text = ""
                    self._element.text += data

        def cloneNode(self):
            element = type(self)(self.name, self.namespace)
            for name, value in self.attributes.items():
                element.attributes[name] = value
            return element

        def reparentChildren(self, newParent):
            if newParent.childNodes:
                newParent.childNodes[-1]._element.tail += self._element.text
            else:
                if not newParent._element.text:
                    newParent._element.text = ""
                if self._element.text is not None:
                    newParent._element.text += self._element.text
            self._element.text = ""
            _base.Node.reparentChildren(self, newParent)

    class Comment(Element):
        def __init__(self, data):
            #Use the superclass constructor to set all properties on the
            #wrapper element
            self._element = ElementTree.Comment(data)
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getData(self):
            return self._element.text

        def _setData(self, value):
            self._element.text = value

        data = property(_getData, _setData)

    class DocumentType(Element):
        def __init__(self, name, publicId, systemId):
            Element.__init__(self, "<!DOCTYPE>")
            self._element.text = name
            self.publicId = publicId
            self.systemId = systemId

        def _getPublicId(self):
            return self._element.get("publicId", "")

        def _setPublicId(self, value):
            if value is not None:
                self._element.set("publicId", value)

        publicId = property(_getPublicId, _setPublicId)

        def _getSystemId(self):
            return self._element.get("systemId", "")

        def _setSystemId(self, value):
            if value is not None:
                self._element.set("systemId", value)

        systemId = property(_getSystemId, _setSystemId)

    class Document(Element):
        def __init__(self):
            Element.__init__(self, "DOCUMENT_ROOT")

    class DocumentFragment(Element):
        def __init__(self):
            Element.__init__(self, "DOCUMENT_FRAGMENT")

    def testSerializer(element):
        rv = []
        finalText = None
        def serializeElement(element, indent=0):
            if not(hasattr(element, "tag")):
                element = element.getroot()
            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "DOCUMENT_ROOT":
                rv.append("#document")
                if element.text:
                    rv.append("|%s\"%s\""%(' '*(indent+2), element.text))
                if element.tail:
                    finalText = element.tail
            elif element.tag == ElementTree.Comment:
                rv.append("|%s<!-- %s -->"%(' '*indent, element.text))
            else:
                assert isinstance(element.tag, text_type), "Expected unicode, got %s, %s"%(type(element.tag), element.tag)
                nsmatch = tag_regexp.match(element.tag)

                if nsmatch is None:
                    name = element.tag
                else:
                    ns, name = nsmatch.groups()
                    prefix = constants.prefixes[ns]
                    name = "%s %s"%(prefix, name)
                rv.append("|%s<%s>"%(' '*indent, name))

                if hasattr(element, "attrib"):
                    attributes = []
                    for name, value in element.attrib.items():
                        nsmatch = tag_regexp.match(name)
                        if nsmatch is not None:
                            ns, name = nsmatch.groups()
                            prefix = constants.prefixes[ns]
                            attr_string = "%s %s"%(prefix, name)
                        else:
                            attr_string = name
                        attributes.append((attr_string, value))

                    for name, value in sorted(attributes):
                        rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
                if element.text:
                    rv.append("|%s\"%s\"" %(' '*(indent+2), element.text))
            indent += 2
            for child in element:
                serializeElement(child, indent)
            if element.tail:
                rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail))
        serializeElement(element, 0)

        if finalText is not None:
            rv.append("|%s\"%s\""%(' '*2, finalText))

        return "\n".join(rv)

    def tostring(element):
        """Serialize an element and its child nodes to a string"""
        rv = []
        finalText = None
        filter = ihatexml.InfosetFilter()
        def serializeElement(element):
            if type(element) == type(ElementTree.ElementTree):
                element = element.getroot()

            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "DOCUMENT_ROOT":
                if element.text:
                    rv.append(element.text)
                if element.tail:
                    finalText = element.tail

                for child in element:
                    serializeElement(child)

            elif type(element.tag) == type(ElementTree.Comment):
                rv.append("<!--%s-->"%(element.text,))
            else:
                #This is assumed to be an ordinary element
                if not element.attrib:
                    rv.append("<%s>"%(filter.fromXmlName(element.tag),))
                else:
                    attr = " ".join(["%s=\"%s\""%(
                                filter.fromXmlName(name), value)
                                     for name, value in element.attrib.items()])
                    rv.append("<%s %s>"%(element.tag, attr))
                if element.text:
                    rv.append(element.text)

                for child in element:
                    serializeElement(child)

                rv.append("</%s>"%(element.tag,))

            if element.tail:
                rv.append(element.tail)

        serializeElement(element)

        if finalText is not None:
            rv.append("%s\""%(' '*2, finalText))

        return "".join(rv)

    class TreeBuilder(_base.TreeBuilder):
        documentClass = Document
        doctypeClass = DocumentType
        elementClass = Element
        commentClass = Comment
        fragmentClass = DocumentFragment

        def testSerializer(self, element):
            return testSerializer(element)

        def getDocument(self):
            if fullTree:
                return self.document._element
            else:
                if self.defaultNamespace is not None:
                    return self.document._element.find(
                        "{%s}html"%self.defaultNamespace)
                else:
                    return self.document._element.find("html")

        def getFragment(self):
            return _base.TreeBuilder.getFragment(self)._element

    return locals()

getETreeModule = moduleFactoryFactory(getETreeBuilder)
<commit_msg>Don't rely on the ElementTree Comment factory being the tag attribute on Comments.<commit_after>from __future__ import absolute_import, division, unicode_literals
from six import text_type

import re

from . import _base
from html5lib import ihatexml
from html5lib import constants
from html5lib.constants import namespaces
from html5lib.utils import moduleFactoryFactory

tag_regexp = re.compile("{([^}]*)}(.*)")

def getETreeBuilder(ElementTreeImplementation, fullTree=False):
    ElementTree = ElementTreeImplementation
    ElementTreeCommentType = ElementTree.Comment("asd").tag
    class Element(_base.Node):
        def __init__(self, name, namespace=None):
            self._name = name
            self._namespace = namespace
            self._element = ElementTree.Element(self._getETreeTag(name,
                                                                  namespace))
            if namespace is None:
                self.nameTuple = namespaces["html"], self._name
            else:
                self.nameTuple = self._namespace, self._name
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getETreeTag(self, name, namespace):
            if namespace is None:
                etree_tag = name
            else:
                etree_tag = "{%s}%s"%(namespace, name)
            return etree_tag

        def _setName(self, name):
            self._name = name
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getName(self):
            return self._name

        name = property(_getName, _setName)

        def _setNamespace(self, namespace):
            self._namespace = namespace
            self._element.tag = self._getETreeTag(self._name, self._namespace)

        def _getNamespace(self):
            return self._namespace

        namespace = property(_getNamespace, _setNamespace)

        def _getAttributes(self):
            return self._element.attrib

        def _setAttributes(self, attributes):
            #Delete existing attributes first
            #XXX - there may be a better way to do this...
            for key in list(self._element.attrib.keys()):
                del self._element.attrib[key]
            for key, value in attributes.items():
                if isinstance(key, tuple):
                    name = "{%s}%s"%(key[2], key[1])
                else:
                    name = key
                self._element.set(name, value)

        attributes = property(_getAttributes, _setAttributes)

        def _getChildNodes(self):
            return self._childNodes
        def _setChildNodes(self, value):
            del self._element[:]
            self._childNodes = []
            for element in value:
                self.insertChild(element)

        childNodes = property(_getChildNodes, _setChildNodes)

        def hasContent(self):
            """Return true if the node has children or text"""
            return bool(self._element.text or len(self._element))

        def appendChild(self, node):
            self._childNodes.append(node)
            self._element.append(node._element)
            node.parent = self

        def insertBefore(self, node, refNode):
            index = list(self._element).index(refNode._element)
            self._element.insert(index, node._element)
            node.parent = self

        def removeChild(self, node):
            self._element.remove(node._element)
            node.parent=None

        def insertText(self, data, insertBefore=None):
            if not(len(self._element)):
                if not self._element.text:
                    self._element.text = ""
                self._element.text += data
            elif insertBefore is None:
                #Insert the text as the tail of the last child element
                if not self._element[-1].tail:
                    self._element[-1].tail = ""
                self._element[-1].tail += data
            else:
                #Insert the text before the specified node
                children = list(self._element)
                index = children.index(insertBefore._element)
                if index > 0:
                    if not self._element[index-1].tail:
                        self._element[index-1].tail = ""
                    self._element[index-1].tail += data
                else:
                    if not self._element.text:
                        self._element.text = ""
                    self._element.text += data

        def cloneNode(self):
            element = type(self)(self.name, self.namespace)
            for name, value in self.attributes.items():
                element.attributes[name] = value
            return element

        def reparentChildren(self, newParent):
            if newParent.childNodes:
                newParent.childNodes[-1]._element.tail += self._element.text
            else:
                if not newParent._element.text:
                    newParent._element.text = ""
                if self._element.text is not None:
                    newParent._element.text += self._element.text
            self._element.text = ""
            _base.Node.reparentChildren(self, newParent)

    class Comment(Element):
        def __init__(self, data):
            #Use the superclass constructor to set all properties on the
            #wrapper element
            self._element = ElementTree.Comment(data)
            self.parent = None
            self._childNodes = []
            self._flags = []

        def _getData(self):
            return self._element.text

        def _setData(self, value):
            self._element.text = value

        data = property(_getData, _setData)

    class DocumentType(Element):
        def __init__(self, name, publicId, systemId):
            Element.__init__(self, "<!DOCTYPE>")
            self._element.text = name
            self.publicId = publicId
            self.systemId = systemId

        def _getPublicId(self):
            return self._element.get("publicId", "")

        def _setPublicId(self, value):
            if value is not None:
                self._element.set("publicId", value)

        publicId = property(_getPublicId, _setPublicId)

        def _getSystemId(self):
            return self._element.get("systemId", "")

        def _setSystemId(self, value):
            if value is not None:
                self._element.set("systemId", value)

        systemId = property(_getSystemId, _setSystemId)

    class Document(Element):
        def __init__(self):
            Element.__init__(self, "DOCUMENT_ROOT")

    class DocumentFragment(Element):
        def __init__(self):
            Element.__init__(self, "DOCUMENT_FRAGMENT")

    def testSerializer(element):
        rv = []
        finalText = None
        def serializeElement(element, indent=0):
            if not(hasattr(element, "tag")):
                element = element.getroot()
            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "DOCUMENT_ROOT":
                rv.append("#document")
                if element.text:
                    rv.append("|%s\"%s\""%(' '*(indent+2), element.text))
                if element.tail:
                    finalText = element.tail
            elif element.tag == ElementTreeCommentType:
                rv.append("|%s<!-- %s -->"%(' '*indent, element.text))
            else:
                assert isinstance(element.tag, text_type), "Expected unicode, got %s, %s"%(type(element.tag), element.tag)
                nsmatch = tag_regexp.match(element.tag)

                if nsmatch is None:
                    name = element.tag
                else:
                    ns, name = nsmatch.groups()
                    prefix = constants.prefixes[ns]
                    name = "%s %s"%(prefix, name)
                rv.append("|%s<%s>"%(' '*indent, name))

                if hasattr(element, "attrib"):
                    attributes = []
                    for name, value in element.attrib.items():
                        nsmatch = tag_regexp.match(name)
                        if nsmatch is not None:
                            ns, name = nsmatch.groups()
                            prefix = constants.prefixes[ns]
                            attr_string = "%s %s"%(prefix, name)
                        else:
                            attr_string = name
                        attributes.append((attr_string, value))

                    for name, value in sorted(attributes):
                        rv.append('|%s%s="%s"' % (' '*(indent+2), name, value))
                if element.text:
                    rv.append("|%s\"%s\"" %(' '*(indent+2), element.text))
            indent += 2
            for child in element:
                serializeElement(child, indent)
            if element.tail:
                rv.append("|%s\"%s\"" %(' '*(indent-2), element.tail))
        serializeElement(element, 0)

        if finalText is not None:
            rv.append("|%s\"%s\""%(' '*2, finalText))

        return "\n".join(rv)

    def tostring(element):
        """Serialize an element and its child nodes to a string"""
        rv = []
        finalText = None
        filter = ihatexml.InfosetFilter()
        def serializeElement(element):
            if type(element) == type(ElementTree.ElementTree):
                element = element.getroot()

            if element.tag == "<!DOCTYPE>":
                if element.get("publicId") or element.get("systemId"):
                    publicId = element.get("publicId") or ""
                    systemId = element.get("systemId") or ""
                    rv.append( """<!DOCTYPE %s PUBLIC "%s" "%s">"""%(
                            element.text, publicId, systemId))
                else:
                    rv.append("<!DOCTYPE %s>"%(element.text,))
            elif element.tag == "DOCUMENT_ROOT":
                if element.text:
                    rv.append(element.text)
                if element.tail:
                    finalText = element.tail

                for child in element:
                    serializeElement(child)

            elif element.tag == ElementTreeCommentType:
                rv.append("<!--%s-->"%(element.text,))
            else:
                #This is assumed to be an ordinary element
                if not element.attrib:
                    rv.append("<%s>"%(filter.fromXmlName(element.tag),))
                else:
                    attr = " ".join(["%s=\"%s\""%(
                                filter.fromXmlName(name), value)
                                     for name, value in element.attrib.items()])
                    rv.append("<%s %s>"%(element.tag, attr))
                if element.text:
                    rv.append(element.text)

                for child in element:
                    serializeElement(child)

                rv.append("</%s>"%(element.tag,))

            if element.tail:
                rv.append(element.tail)

        serializeElement(element)

        if finalText is not None:
            rv.append("%s\""%(' '*2, finalText))

        return "".join(rv)

    class TreeBuilder(_base.TreeBuilder):
        documentClass = Document
        doctypeClass = DocumentType
        elementClass = Element
        commentClass = Comment
        fragmentClass = DocumentFragment

        def testSerializer(self, element):
            return testSerializer(element)

        def getDocument(self):
            if fullTree:
                return self.document._element
            else:
                if self.defaultNamespace is not None:
                    return self.document._element.find(
                        "{%s}html"%self.defaultNamespace)
                else:
                    return self.document._element.find("html")

        def getFragment(self):
            return _base.TreeBuilder.getFragment(self)._element

    return locals()

getETreeModule = moduleFactoryFactory(getETreeBuilder)
<|endoftext|>


<commit_before>#  Copyright 2011 Digg, Inc.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import collections
import errno
import logging
import os
import os.path
import shutil
import Queue

import digg.dev.hackbuilder.common
import digg.dev.hackbuilder.errors
import digg.dev.hackbuilder.plugins
from digg.dev.hackbuilder.plugin_utils import BinaryBuilder
from digg.dev.hackbuilder.plugin_utils import PackageBuilder

class BuildFileReader(object):
    """A build file reader for reading the HACK_BUILD files.

    Attributes:
        cached_build_file_targets: This is a cache to prevent build files from
            having to be processed more than once. It's keys are a repository
            paths, and it's values are the discovered build file targets for
            that path.
    """
    def __init__(self, normalizer):
        self.normalizer = normalizer
        self.cached_build_file_targets = dict()

    def get_build_file_targets_for_repo_path(self, build_file_dirname):
        if build_file_dirname in self.cached_build_file_targets:
            return self.cached_build_file_targets[build_file_dirname]

        build_file_filename = os.path.join(self.normalizer.repo_root_path,
                build_file_dirname[1:], 'HACK_BUILD')
        build_file_locals = (
                digg.dev.hackbuilder.plugins.get_all_build_file_rules(
                    build_file_dirname, self.normalizer))
        logging.info('loading build file at: %s', build_file_filename)
        execfile(build_file_filename, {}, build_file_locals)

        build_file_targets = self._get_all_targets_from_global_queue()

        self.cached_build_file_targets[build_file_dirname] = build_file_targets
        return build_file_targets

    def _get_all_targets_from_global_queue(self):
        """Fetch all the build file targets from the global variable queue.

        These build file targets are store in a global queue after each build
        file is processed.

        Returns: A set of all the build file targets that were on the queue.
        """
        global_build_file_targets_queue = (
                digg.dev.hackbuilder.plugins.build_file_targets)
        build_file_targets = set()
        while True:
            try:
                item = global_build_file_targets_queue.get_nowait()
            except Queue.Empty:
                break
            build_file_targets.add(item)
            global_build_file_targets_queue.task_done()

        return build_file_targets

class BuildFileTargetFinder(object):
    def __init__(self, normalizer, build_file_reader):
        self.normalizer = normalizer
        self.build_file_reader = build_file_reader
        self.target_ids = set()
        self.build_file_path_seen = set()

    def seed_from_cli_build_target_ids(self, arg_targets):
        self.seed_target_ids = self._get_normalized_target_ids_from_cli_args(
                arg_targets)

    def get_target_trees(self, target_ids_to_discover=None):
        if target_ids_to_discover is None:
            target_ids_to_discover = self.seed_target_ids.copy()

        dep_trees = {}
        for target_id in target_ids_to_discover:
            build_target = self.get_build_target(target_id)
            dep_targets = self.get_target_trees(
                    target_ids_to_discover=build_target.dep_ids)
            dep_trees[build_target] = dep_targets
        return dep_trees

    def get_build_target(self, target_id):
        get_build_file_targets_for_repo_path = (
                self.build_file_reader.get_build_file_targets_for_repo_path)
        build_file_path = target_id.path

        possible_build_file_targets = set()
        possible_build_file_targets.update(
                get_build_file_targets_for_repo_path(
                    build_file_path))

        build_file_targets_to_discover = set()
        for build_file_target in possible_build_file_targets:
            if build_file_target.target_id == target_id:
                return build_file_target

        raise digg.dev.hackbuilder.errors.Error(
                'No build target found for target id(%s)'
                % target_id)

    def _get_normalized_target_ids_from_cli_args(self, arg_targets):
        normalized_targets = set()
        for target in arg_targets:
            target_id = digg.dev.hackbuilder.target.TargetID.from_string(
                    target)
            normalized_targets.add(
                    self.normalizer.normalize_target_id(target_id))
        return normalized_targets

class Build(object):
    def __init__(self, build_target_trees, normalizer,
            source_path=digg.dev.hackbuilder.common.DEFAULT_SOURCE_DIR,
            build_path=digg.dev.hackbuilder.common.DEFAULT_BUILD_DIR,
            package_path=digg.dev.hackbuilder.common.DEFAULT_PACKAGE_DIR):
        self.build_target_trees = build_target_trees
        self.normalizer = normalizer

        self.source_path = source_path
        self.build_path = build_path
        self.package_path = package_path

        self.builders = {}
        if build_target_trees is not None:
            target_dep_sequences = self._get_target_dep_sequences()
            for target_deps in target_dep_sequences.itervalues():
                for target in [i[0] for i in target_deps]:
                    if target.target_id not in self.builders:
                        builder = target.builder_class(target)
                        self.builders[target.target_id] = builder

    def build(self):
        logging.info('Starting build.')
        self.create_dirs()
        self.do_create_source()
        self.do_create_build_environment()
        self.do_build_binary()
        self.do_build_package()
        logging.info('Finishing build.')

    def create_dirs(self):
        logging.info('Creating infrastructure directories')

        logging.info('Creating source directory: %s', self.source_path)
        self._mkdir_in_repo_dir(self.source_path)

        logging.info('Creating build directory: %s', self.build_path)
        self._mkdir_in_repo_dir(self.build_path)

        logging.info('Creating package directory: %s', self.package_path)
        self._mkdir_in_repo_dir(self.package_path)

    def _mkdir_in_repo_dir(self, path):
        full_path = os.path.join(self.normalizer.repo_root_path, path)
        logging.debug('Creating absolute directory: %s', full_path)
        try:
            os.mkdir(full_path)
        except OSError, e:
            if e.errno != errno.EEXIST:
                raise

    def remove_directories(self):
        logging.info('Removing infrastructure directories')

        logging.info('Removing source hierarchy: %s', self.source_path)
        self._rmtree_in_repo_dir(self.source_path)

        logging.info('Removing build hierarchy: %s', self.build_path)
        self._rmtree_in_repo_dir(self.build_path)

        logging.info('Removing package hierarchy: %s', self.package_path)
        self._rmtree_in_repo_dir(self.package_path)

    def _rmtree_in_repo_dir(self, path):
        full_path = os.path.join(self.normalizer.repo_root_path, path)
        logging.debug('Removing absolute directory: %s', full_path)
        try:
            shutil.rmtree(full_path)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise

    def do_create_source(self):
        logging.info('Entering create source tree state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, BinaryBuilder):
                        builder.do_pre_create_source_tree_work(
                                self.builders)
                    self.builders[target].do_create_source_tree_work()
                    targets_handled.add(target)

        logging.info('Exiting create source tree state')

    def do_create_build_environment(self):
        logging.info('Entering create build environment state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    builder.do_create_build_environment_work()
                    targets_handled.add(target)

        logging.info('Exiting create build environment state')

    def do_build_binary(self):
        logging.info('Entering build binary state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, BinaryBuilder):
                        builder.do_pre_build_binary_library_install(
                                self.builders)
                    builder.do_build_binary_work()
                    targets_handled.add(target)

        logging.info('Exiting build binary state')

    def do_build_package(self):
        logging.info('Entering build package state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, PackageBuilder):
                        builder.do_pre_build_package_binary_install(
                                self.builders)
                    builder.do_build_package_work()
                    targets_handled.add(target)

        logging.info('Exiting build package state')

    def _get_target_dep_sequences(self):
        target_deques = {}
        for root_node in self.build_target_trees.iteritems():
            target_deques[root_node[0].target_id] = collections.deque()
            working_deque = collections.deque()
            working_deque.append(root_node)
            while len(working_deque) > 0:
                current_node = working_deque.pop()
                target_deques[root_node[0].target_id].appendleft(
                        current_node)
                for dep in current_node[1].iteritems():
                    working_deque.append(dep)

        return target_deques
<commit_msg>Remove unused class attribute.<commit_after>#  Copyright 2011 Digg, Inc.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import collections
import errno
import logging
import os
import os.path
import shutil
import Queue

import digg.dev.hackbuilder.common
import digg.dev.hackbuilder.errors
import digg.dev.hackbuilder.plugins
from digg.dev.hackbuilder.plugin_utils import BinaryBuilder
from digg.dev.hackbuilder.plugin_utils import PackageBuilder

class BuildFileReader(object):
    """A build file reader for reading the HACK_BUILD files.

    Attributes:
        cached_build_file_targets: This is a cache to prevent build files from
            having to be processed more than once. It's keys are a repository
            paths, and it's values are the discovered build file targets for
            that path.
    """
    def __init__(self, normalizer):
        self.normalizer = normalizer
        self.cached_build_file_targets = dict()

    def get_build_file_targets_for_repo_path(self, build_file_dirname):
        if build_file_dirname in self.cached_build_file_targets:
            return self.cached_build_file_targets[build_file_dirname]

        build_file_filename = os.path.join(self.normalizer.repo_root_path,
                build_file_dirname[1:], 'HACK_BUILD')
        build_file_locals = (
                digg.dev.hackbuilder.plugins.get_all_build_file_rules(
                    build_file_dirname, self.normalizer))
        logging.info('loading build file at: %s', build_file_filename)
        execfile(build_file_filename, {}, build_file_locals)

        build_file_targets = self._get_all_targets_from_global_queue()

        self.cached_build_file_targets[build_file_dirname] = build_file_targets
        return build_file_targets

    def _get_all_targets_from_global_queue(self):
        """Fetch all the build file targets from the global variable queue.

        These build file targets are store in a global queue after each build
        file is processed.

        Returns: A set of all the build file targets that were on the queue.
        """
        global_build_file_targets_queue = (
                digg.dev.hackbuilder.plugins.build_file_targets)
        build_file_targets = set()
        while True:
            try:
                item = global_build_file_targets_queue.get_nowait()
            except Queue.Empty:
                break
            build_file_targets.add(item)
            global_build_file_targets_queue.task_done()

        return build_file_targets

class BuildFileTargetFinder(object):
    def __init__(self, normalizer, build_file_reader):
        self.normalizer = normalizer
        self.build_file_reader = build_file_reader
        self.target_ids = set()

    def seed_from_cli_build_target_ids(self, arg_targets):
        self.seed_target_ids = self._get_normalized_target_ids_from_cli_args(
                arg_targets)

    def get_target_trees(self, target_ids_to_discover=None):
        if target_ids_to_discover is None:
            target_ids_to_discover = self.seed_target_ids.copy()

        dep_trees = {}
        for target_id in target_ids_to_discover:
            build_target = self.get_build_target(target_id)
            dep_targets = self.get_target_trees(
                    target_ids_to_discover=build_target.dep_ids)
            dep_trees[build_target] = dep_targets
        return dep_trees

    def get_build_target(self, target_id):
        get_build_file_targets_for_repo_path = (
                self.build_file_reader.get_build_file_targets_for_repo_path)
        build_file_path = target_id.path

        possible_build_file_targets = set()
        possible_build_file_targets.update(
                get_build_file_targets_for_repo_path(
                    build_file_path))

        build_file_targets_to_discover = set()
        for build_file_target in possible_build_file_targets:
            if build_file_target.target_id == target_id:
                return build_file_target

        raise digg.dev.hackbuilder.errors.Error(
                'No build target found for target id(%s)'
                % target_id)

    def _get_normalized_target_ids_from_cli_args(self, arg_targets):
        normalized_targets = set()
        for target in arg_targets:
            target_id = digg.dev.hackbuilder.target.TargetID.from_string(
                    target)
            normalized_targets.add(
                    self.normalizer.normalize_target_id(target_id))
        return normalized_targets

class Build(object):
    def __init__(self, build_target_trees, normalizer,
            source_path=digg.dev.hackbuilder.common.DEFAULT_SOURCE_DIR,
            build_path=digg.dev.hackbuilder.common.DEFAULT_BUILD_DIR,
            package_path=digg.dev.hackbuilder.common.DEFAULT_PACKAGE_DIR):
        self.build_target_trees = build_target_trees
        self.normalizer = normalizer

        self.source_path = source_path
        self.build_path = build_path
        self.package_path = package_path

        self.builders = {}
        if build_target_trees is not None:
            target_dep_sequences = self._get_target_dep_sequences()
            for target_deps in target_dep_sequences.itervalues():
                for target in [i[0] for i in target_deps]:
                    if target.target_id not in self.builders:
                        builder = target.builder_class(target)
                        self.builders[target.target_id] = builder

    def build(self):
        logging.info('Starting build.')
        self.create_dirs()
        self.do_create_source()
        self.do_create_build_environment()
        self.do_build_binary()
        self.do_build_package()
        logging.info('Finishing build.')

    def create_dirs(self):
        logging.info('Creating infrastructure directories')

        logging.info('Creating source directory: %s', self.source_path)
        self._mkdir_in_repo_dir(self.source_path)

        logging.info('Creating build directory: %s', self.build_path)
        self._mkdir_in_repo_dir(self.build_path)

        logging.info('Creating package directory: %s', self.package_path)
        self._mkdir_in_repo_dir(self.package_path)

    def _mkdir_in_repo_dir(self, path):
        full_path = os.path.join(self.normalizer.repo_root_path, path)
        logging.debug('Creating absolute directory: %s', full_path)
        try:
            os.mkdir(full_path)
        except OSError, e:
            if e.errno != errno.EEXIST:
                raise

    def remove_directories(self):
        logging.info('Removing infrastructure directories')

        logging.info('Removing source hierarchy: %s', self.source_path)
        self._rmtree_in_repo_dir(self.source_path)

        logging.info('Removing build hierarchy: %s', self.build_path)
        self._rmtree_in_repo_dir(self.build_path)

        logging.info('Removing package hierarchy: %s', self.package_path)
        self._rmtree_in_repo_dir(self.package_path)

    def _rmtree_in_repo_dir(self, path):
        full_path = os.path.join(self.normalizer.repo_root_path, path)
        logging.debug('Removing absolute directory: %s', full_path)
        try:
            shutil.rmtree(full_path)
        except OSError, e:
            if e.errno != errno.ENOENT:
                raise

    def do_create_source(self):
        logging.info('Entering create source tree state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, BinaryBuilder):
                        builder.do_pre_create_source_tree_work(
                                self.builders)
                    self.builders[target].do_create_source_tree_work()
                    targets_handled.add(target)

        logging.info('Exiting create source tree state')

    def do_create_build_environment(self):
        logging.info('Entering create build environment state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    builder.do_create_build_environment_work()
                    targets_handled.add(target)

        logging.info('Exiting create build environment state')

    def do_build_binary(self):
        logging.info('Entering build binary state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, BinaryBuilder):
                        builder.do_pre_build_binary_library_install(
                                self.builders)
                    builder.do_build_binary_work()
                    targets_handled.add(target)

        logging.info('Exiting build binary state')

    def do_build_package(self):
        logging.info('Entering build package state')

        target_sequences = self._get_target_dep_sequences()

        targets_handled = set()
        #while True:
        #    # build targets from target_deques
        #    break
        for targets in target_sequences.itervalues():
            for target in [i[0].target_id for i in targets]:
                if target not in targets_handled:
                    builder = self.builders[target]
                    if isinstance(builder, PackageBuilder):
                        builder.do_pre_build_package_binary_install(
                                self.builders)
                    builder.do_build_package_work()
                    targets_handled.add(target)

        logging.info('Exiting build package state')

    def _get_target_dep_sequences(self):
        target_deques = {}
        for root_node in self.build_target_trees.iteritems():
            target_deques[root_node[0].target_id] = collections.deque()
            working_deque = collections.deque()
            working_deque.append(root_node)
            while len(working_deque) > 0:
                current_node = working_deque.pop()
                target_deques[root_node[0].target_id].appendleft(
                        current_node)
                for dep in current_node[1].iteritems():
                    working_deque.append(dep)

        return target_deques
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict

import numpy as np

import sklearn.decomposition

from scipy.stats import multivariate_normal

from cgpm.cgpm import CGpm
from cgpm.utils import general as gu

class FactorAnalysis(CGpm):
    """Factor analysis model with continuous latent variables z in a low
    dimensional space. The generative model for a vector x is

    z ~ Normal(0, I)    where z \in R^L.
    e ~ Normal(0, Psi)  where Psi = diag(v_1,...,v_D)
    x = W.z + mux + e    where W \in R^(DxL) and mux \in R^D, learning by EM.

    From standard results (Murphy Section 12.1)

        z ~ Normal(0, I)                Prior.

        x|z ~ Normal(W.z + mux, Psi)     Likelihood.

        x ~ Normal(mux, W.W'+Psi)        Marginal.

        z|x ~ Normal(m, S)              Posterior.
            S = inv(I + W'.inv(Psi).W)      (covariance)
            m = S(W'.inv(Psi).(x-mux))       (mean)

    The full joint distribution over [z,x] is then

    The mean of [z,x] is [0, mux]
    The covariance of [z,x] is (in block form)

        I           W'
      (LxL)       (LxD)

        W      W.W' + Psi
      (DxL)       (DxD)

    where the covariance W' is computed directly
    cov(z,x)    = cov(z, W.z + mux + e)
                = cov(z, W.z) + cov(z, mux) + cov(z, e)
                = cov(z, W.z)
                = cov(z,z).W'
                = W'

    Exercise: Confirm that expression for posterior z|x is consistent with
    conditioning directly on the joint [z,x] using Schur complement
    (Hint: see test suite).

    The latent variables are exposed as output variables, but may not be
    incorporated.
    """

    def __init__(self, outputs, inputs, params=None, distargs=None, rng=None):
        # Default parameter settings.
        if params is None:
            params = {}
        if distargs is None:
            distargs = {}
        # Entropy.
        if rng is None:
            rng = gu.gen_rng(1)
        # No inputs.
        if inputs:
            raise ValueError('FactorAnalysis rejects inputs: %s.' % inputs)
        # Correct outputs.
        if len(outputs) < 2:
            raise ValueError('FactorAnalysis needs >= 2 outputs: %s.' % outputs)
        if len(set(outputs)) != len(outputs):
            raise ValueError('Duplicate outputs: %s.' % outputs)
        # Find low dimensional space.
        L = distargs.get('L', None)
        if L is None:
            raise ValueError('Specify latent dimension L: %s.' % distargs)
        if L == 0:
            raise ValueError('Latent dimension at least 1: %s.' % distargs)
        # Observable and latent variable indexes.
        D = len(outputs[:-L])
        if D < L:
            raise ValueError(
                'Latent dimension exceeds observed dimension: (%s,%s)'
                % (outputs[:-L], outputs[-L:]))
        # Build the object.
        self.rng = rng
        # Dimensions.
        self.L = L
        self.D = D
        # Varible indexes.
        self.outputs = outputs
        self.latents = outputs[-self.L:]
        # Dataset.
        self.data = OrderedDict()
        self.N = 0
        # Parameters of Factor Analysis.
        self.mux = params.get('mux', np.zeros(D))
        self.Psi = params.get('Psi', np.eye(D))
        self.W = params.get('W', np.zeros((D,L)))
        # Parameters of joint distribution [x,z].
        self.mu, self.cov = self.joint_parameters()
        # Internal factor analysis model.
        self.fa = None

    def incorporate(self, rowid, query, evidence=None):
        # No duplicate observation.
        if rowid in self.data:
            raise ValueError('Already observed: %d.' % rowid)
        # No inputs.
        if evidence:
            raise ValueError('No evidence allowed: %s.' % evidence)
        if not query:
            raise ValueError('No query specified: %s.' % query)
        # No unknown variables.
        if any(q not in self.outputs for q in query):
            raise ValueError('Unknown variables: (%s,%s).'
                % (query, self.outputs))
        # No incorporation of latent variables.
        if any(q in self.latents for q in query):
            raise ValueError('Cannot incorporate latent vars: (%s,%s,%s).'
                % (query, self.outputs, self.latents))
        # Reindex the query variables.
        query_r = {self.outputs.index(q): query[q] for q in query}
        # Incorporate observed variables.
        x = [query_r.get(i, np.nan) for i in xrange(self.D)]
        # Update dataset and counts.
        self.data[rowid] = x
        self.N += 1

    def unincorporate(self, rowid):
        try:
            del self.data[rowid]
        except KeyError:
            raise ValueError('No such observation: %d.' % rowid)
        self.N -= 1

    def logpdf(self, rowid, query, evidence=None):
        # XXX Deal with observed rowids.
        if evidence is None:
            evidence = {}
        if any(q not in self.outputs for q in query):
            raise ValueError('Unknown variables: (%s,%s).'
                % (query, self.outputs))
        if any(q in evidence for q in query):
            raise ValueError('Duplicate variable: (%s,%s).' % (query, evidence))
        # Reindex variables.
        query_r = self.reindex(query.keys())
        evidence_r = self.reindex(evidence)
        # Retrieve conditional distribution.
        muG, covG = FactorAnalysis.mvn_condition(
            self.mu, self.cov, query_r.keys(), evidence_r)
        # Compute log density.
        return multivariate_normal.logpdf(query_r.values(), mean=muG, cov=covG)

    def simulate(self, rowid, query, evidence=None, N=None):
        # XXX Deal with observed rowids.
        if evidence is None:
            evidence = {}
        if any(q in evidence for q in query):
            raise ValueError('Duplicate variable: (%s,%s).' % (query, evidence))
        # Reindex variables.
        query_r = self.reindex(query)
        evidence_r = self.reindex(evidence)
        # Retrieve conditional distribution.
        muG, covG = FactorAnalysis.mvn_condition(
            self.mu, self.cov, query_r, evidence_r)
        # Generate samples.
        sample = multivariate_normal.rvs(
            mean=muG, cov=covG, size=N, random_state=self.rng)
        def get_sample(s):
            return dict(zip(query, s))
        return get_sample(sample) if N is None else map(get_sample, sample)

    def logpdf_score(self):
        def compute_logpdf(x):
            assert len(x) == self.D
            query = {i:v for i,v in enumerate(x) if not np.isnan(v)}
            return self.logpdf(-1, query, evidence=None)
        return sum(compute_logpdf(x) for x in self.data)

    def transition(self, N=None):
        X = np.asarray(self.data.values())
        # Only run inference on observations without missing entries.
        self.fa = sklearn.decomposition.FactorAnalysis(n_components=self.L)
        self.fa.fit(X[~np.any(np.isnan(X), axis=1)])
        assert self.L, self.D == self.fa.components_.shape
        # Update parameters of Factor Analysis.
        self.Psi = np.diag(self.fa.noise_variance_)
        self.mux = self.fa.mean_
        self.W = np.transpose(self.fa.components_)
        self.mu, self.cov = self.joint_parameters()

    # --------------------------------------------------------------------------
    # Internal.

    def get_params(self):
        return {
            'mu': self.mu,
            'Psi': self.Psi,
            'W': self.W
        }

    def get_distargs(self):
        return {
            'L': self.L
        }

    @staticmethod
    def name():
        return 'low_dimensional_mvn'

    @staticmethod
    def is_continuous():
        return True

    @staticmethod
    def is_conditional():
        return False

    @staticmethod
    def is_numeric():
        return True

    # --------------------------------------------------------------------------
    # Helper.

    def reindex(self, query):
        # Reindex an output variable to its index in self.mu
        # self.mu has as the first L items the last L items of self.outputs
        # and as the remaining D items the first D items of self.outputs.
        # The following diagram is useful:
        # self.outputs:  12 14 -7 5 |  11 4  3
        #                <---D=4--->|<--L=3-->
        # raw indices:   0  1  2  3 |  4  5  6
        # reindexed:     3  4  5  6 |  0  1  2
        assert isinstance(query, list)
        def convert(q):
            i = self.outputs.index(q)
            return i - self.D if q in self.latents else i + self.L
        return [convert(q) for q in query]

    def joint_parameters(self):
        mean = np.concatenate((np.zeros(self.L), self.mux))
        cov = np.row_stack((
            np.column_stack((np.eye(self.L), self.W.T)),
            np.column_stack((self.W, np.dot(self.W, self.W.T) + self.Psi))
        ))
        return mean, cov

    @staticmethod
    def mvn_marginalize(mu, cov, query, evidence):
        Q, E = query, evidence
        # Retrieve means.
        muQ = mu[Q]
        muE = mu[E]
        # Retrieve covariances.
        covQ = cov[Q][:,Q]
        covE = cov[E][:,E]
        covJ = cov[Q][:,E]
        covQE = np.row_stack((
            np.column_stack((covQ, covJ)),
            np.column_stack((covJ.T, covE))
        ))
        assert np.allclose(covQE, covQE.T)
        return muQ, muE, covQ, covE, covJ

    @staticmethod
    def mvn_condition(mu, cov, query, evidence):
        assert isinstance(query, list)
        assert isinstance(evidence, dict)
        assert len(mu) == cov.shape[0] == cov.shape[1]
        assert len(query) + len(evidence) <= len(mu)
        # Extract indexes and values from evidence.
        Ei, Ev = evidence.keys(), evidence.values()
        muQ, muE, covQ, covE, covJ = \
            FactorAnalysis.mvn_marginalize(mu, cov, query, Ei)
        # Invoke Fact 4 from, where G means given.
        # http://web4.cs.ucl.ac.uk/staff/C.Bracegirdle/bayesTheoremForGaussians.pdf
        P = np.dot(covJ, np.linalg.inv(covE))
        muG = muQ + np.dot(P, Ev - muE)
        covG = covQ - np.dot(P, covJ.T)
        return muG, covG

    # --------------------------------------------------------------------------
    # Serialization.

    def to_metadata(self):
        metadata = dict()
        metadata['outputs'] = self.outputs
        metadata['inputs'] = self.inputs
        metadata['N'] = self.N
        metadata['data'] = {self.data}
        metadata['distargs'] = self.get_distargs()
        metadata['params'] = self.get_params()
        metadata['factory'] = ('cgpm.pca.factor', 'FactorAnalysis')
        return metadata

    @classmethod
    def from_metadata(cls, metadata, rng=None):
        if rng is None:
            rng = gu.gen_rng(0)
        ldmvn = cls(
            outputs=metadata['outputs'],
            inputs=metadata['inputs'],
            distargs=metadata['distargs'],
            params=metadata['params'],
            rng=rng)
        ldmvn.data = metadata['data']
        ldmvn.N = metadata['N']
        return ldmvn
<commit_msg>Let reindex operate on dicts.<commit_after># -*- coding: utf-8 -*-

# Copyright (c) 2015-2016 MIT Probabilistic Computing Project

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict

import numpy as np

import sklearn.decomposition

from scipy.stats import multivariate_normal

from cgpm.cgpm import CGpm
from cgpm.utils import general as gu

class FactorAnalysis(CGpm):
    """Factor analysis model with continuous latent variables z in a low
    dimensional space. The generative model for a vector x is

    z ~ Normal(0, I)    where z \in R^L.
    e ~ Normal(0, Psi)  where Psi = diag(v_1,...,v_D)
    x = W.z + mux + e    where W \in R^(DxL) and mux \in R^D, learning by EM.

    From standard results (Murphy Section 12.1)

        z ~ Normal(0, I)                Prior.

        x|z ~ Normal(W.z + mux, Psi)     Likelihood.

        x ~ Normal(mux, W.W'+Psi)        Marginal.

        z|x ~ Normal(m, S)              Posterior.
            S = inv(I + W'.inv(Psi).W)      (covariance)
            m = S(W'.inv(Psi).(x-mux))       (mean)

    The full joint distribution over [z,x] is then

    The mean of [z,x] is [0, mux]
    The covariance of [z,x] is (in block form)

        I           W'
      (LxL)       (LxD)

        W      W.W' + Psi
      (DxL)       (DxD)

    where the covariance W' is computed directly
    cov(z,x)    = cov(z, W.z + mux + e)
                = cov(z, W.z) + cov(z, mux) + cov(z, e)
                = cov(z, W.z)
                = cov(z,z).W'
                = W'

    Exercise: Confirm that expression for posterior z|x is consistent with
    conditioning directly on the joint [z,x] using Schur complement
    (Hint: see test suite).

    The latent variables are exposed as output variables, but may not be
    incorporated.
    """

    def __init__(self, outputs, inputs, params=None, distargs=None, rng=None):
        # Default parameter settings.
        if params is None:
            params = {}
        if distargs is None:
            distargs = {}
        # Entropy.
        if rng is None:
            rng = gu.gen_rng(1)
        # No inputs.
        if inputs:
            raise ValueError('FactorAnalysis rejects inputs: %s.' % inputs)
        # Correct outputs.
        if len(outputs) < 2:
            raise ValueError('FactorAnalysis needs >= 2 outputs: %s.' % outputs)
        if len(set(outputs)) != len(outputs):
            raise ValueError('Duplicate outputs: %s.' % outputs)
        # Find low dimensional space.
        L = distargs.get('L', None)
        if L is None:
            raise ValueError('Specify latent dimension L: %s.' % distargs)
        if L == 0:
            raise ValueError('Latent dimension at least 1: %s.' % distargs)
        # Observable and latent variable indexes.
        D = len(outputs[:-L])
        if D < L:
            raise ValueError(
                'Latent dimension exceeds observed dimension: (%s,%s)'
                % (outputs[:-L], outputs[-L:]))
        # Build the object.
        self.rng = rng
        # Dimensions.
        self.L = L
        self.D = D
        # Varible indexes.
        self.outputs = outputs
        self.latents = outputs[-self.L:]
        # Dataset.
        self.data = OrderedDict()
        self.N = 0
        # Parameters of Factor Analysis.
        self.mux = params.get('mux', np.zeros(D))
        self.Psi = params.get('Psi', np.eye(D))
        self.W = params.get('W', np.zeros((D,L)))
        # Parameters of joint distribution [x,z].
        self.mu, self.cov = self.joint_parameters()
        # Internal factor analysis model.
        self.fa = None

    def incorporate(self, rowid, query, evidence=None):
        # No duplicate observation.
        if rowid in self.data:
            raise ValueError('Already observed: %d.' % rowid)
        # No inputs.
        if evidence:
            raise ValueError('No evidence allowed: %s.' % evidence)
        if not query:
            raise ValueError('No query specified: %s.' % query)
        # No unknown variables.
        if any(q not in self.outputs for q in query):
            raise ValueError('Unknown variables: (%s,%s).'
                % (query, self.outputs))
        # No incorporation of latent variables.
        if any(q in self.latents for q in query):
            raise ValueError('Cannot incorporate latent vars: (%s,%s,%s).'
                % (query, self.outputs, self.latents))
        # Reindex the query variables.
        query_r = {self.outputs.index(q): query[q] for q in query}
        # Incorporate observed variables.
        x = [query_r.get(i, np.nan) for i in xrange(self.D)]
        # Update dataset and counts.
        self.data[rowid] = x
        self.N += 1

    def unincorporate(self, rowid):
        try:
            del self.data[rowid]
        except KeyError:
            raise ValueError('No such observation: %d.' % rowid)
        self.N -= 1

    def logpdf(self, rowid, query, evidence=None):
        # XXX Deal with observed rowids.
        if evidence is None:
            evidence = {}
        if any(q not in self.outputs for q in query):
            raise ValueError('Unknown variables: (%s,%s).'
                % (query, self.outputs))
        if any(q in evidence for q in query):
            raise ValueError('Duplicate variable: (%s,%s).' % (query, evidence))
        # Reindex variables.
        query_r = self.reindex(query.keys())
        evidence_r = self.reindex(evidence)
        # Retrieve conditional distribution.
        muG, covG = FactorAnalysis.mvn_condition(
            self.mu, self.cov, query_r.keys(), evidence_r)
        # Compute log density.
        return multivariate_normal.logpdf(query_r.values(), mean=muG, cov=covG)

    def simulate(self, rowid, query, evidence=None, N=None):
        # XXX Deal with observed rowids.
        if evidence is None:
            evidence = {}
        if any(q in evidence for q in query):
            raise ValueError('Duplicate variable: (%s,%s).' % (query, evidence))
        # Reindex variables.
        query_r = self.reindex(query)
        evidence_r = self.reindex(evidence)
        # Retrieve conditional distribution.
        muG, covG = FactorAnalysis.mvn_condition(
            self.mu, self.cov, query_r, evidence_r)
        # Generate samples.
        sample = multivariate_normal.rvs(
            mean=muG, cov=covG, size=N, random_state=self.rng)
        def get_sample(s):
            return dict(zip(query, s))
        return get_sample(sample) if N is None else map(get_sample, sample)

    def logpdf_score(self):
        def compute_logpdf(x):
            assert len(x) == self.D
            query = {i:v for i,v in enumerate(x) if not np.isnan(v)}
            return self.logpdf(-1, query, evidence=None)
        return sum(compute_logpdf(x) for x in self.data)

    def transition(self, N=None):
        X = np.asarray(self.data.values())
        # Only run inference on observations without missing entries.
        self.fa = sklearn.decomposition.FactorAnalysis(n_components=self.L)
        self.fa.fit(X[~np.any(np.isnan(X), axis=1)])
        assert self.L, self.D == self.fa.components_.shape
        # Update parameters of Factor Analysis.
        self.Psi = np.diag(self.fa.noise_variance_)
        self.mux = self.fa.mean_
        self.W = np.transpose(self.fa.components_)
        self.mu, self.cov = self.joint_parameters()

    # --------------------------------------------------------------------------
    # Internal.

    def get_params(self):
        return {
            'mu': self.mu,
            'Psi': self.Psi,
            'W': self.W
        }

    def get_distargs(self):
        return {
            'L': self.L
        }

    @staticmethod
    def name():
        return 'low_dimensional_mvn'

    @staticmethod
    def is_continuous():
        return True

    @staticmethod
    def is_conditional():
        return False

    @staticmethod
    def is_numeric():
        return True

    # --------------------------------------------------------------------------
    # Helper.

    def reindex(self, query):
        # Reindex an output variable to its index in self.mu
        # self.mu has as the first L items the last L items of self.outputs
        # and as the remaining D items the first D items of self.outputs.
        # The following diagram is useful:
        # self.outputs:  12 14 -7 5 |  11 4  3
        #                <---D=4--->|<--L=3-->
        # raw indices:   0  1  2  3 |  4  5  6
        # reindexed:     3  4  5  6 |  0  1  2
        assert isinstance(query, (list, dict))
        def convert(q):
            i = self.outputs.index(q)
            return i - self.D if q in self.latents else i + self.L
        indexes = [convert(q) for q in query]
        if isinstance(query, list):
            return indexes
        else:
            return dict(zip(indexes, query.values()))

    def joint_parameters(self):
        mean = np.concatenate((np.zeros(self.L), self.mux))
        cov = np.row_stack((
            np.column_stack((np.eye(self.L), self.W.T)),
            np.column_stack((self.W, np.dot(self.W, self.W.T) + self.Psi))
        ))
        return mean, cov

    @staticmethod
    def mvn_marginalize(mu, cov, query, evidence):
        Q, E = query, evidence
        # Retrieve means.
        muQ = mu[Q]
        muE = mu[E]
        # Retrieve covariances.
        covQ = cov[Q][:,Q]
        covE = cov[E][:,E]
        covJ = cov[Q][:,E]
        covQE = np.row_stack((
            np.column_stack((covQ, covJ)),
            np.column_stack((covJ.T, covE))
        ))
        assert np.allclose(covQE, covQE.T)
        return muQ, muE, covQ, covE, covJ

    @staticmethod
    def mvn_condition(mu, cov, query, evidence):
        assert isinstance(query, list)
        assert isinstance(evidence, dict)
        assert len(mu) == cov.shape[0] == cov.shape[1]
        assert len(query) + len(evidence) <= len(mu)
        # Extract indexes and values from evidence.
        Ei, Ev = evidence.keys(), evidence.values()
        muQ, muE, covQ, covE, covJ = \
            FactorAnalysis.mvn_marginalize(mu, cov, query, Ei)
        # Invoke Fact 4 from, where G means given.
        # http://web4.cs.ucl.ac.uk/staff/C.Bracegirdle/bayesTheoremForGaussians.pdf
        P = np.dot(covJ, np.linalg.inv(covE))
        muG = muQ + np.dot(P, Ev - muE)
        covG = covQ - np.dot(P, covJ.T)
        return muG, covG

    # --------------------------------------------------------------------------
    # Serialization.

    def to_metadata(self):
        metadata = dict()
        metadata['outputs'] = self.outputs
        metadata['inputs'] = self.inputs
        metadata['N'] = self.N
        metadata['data'] = {self.data}
        metadata['distargs'] = self.get_distargs()
        metadata['params'] = self.get_params()
        metadata['factory'] = ('cgpm.pca.factor', 'FactorAnalysis')
        return metadata

    @classmethod
    def from_metadata(cls, metadata, rng=None):
        if rng is None:
            rng = gu.gen_rng(0)
        ldmvn = cls(
            outputs=metadata['outputs'],
            inputs=metadata['inputs'],
            distargs=metadata['distargs'],
            params=metadata['params'],
            rng=rng)
        ldmvn.data = metadata['data']
        ldmvn.N = metadata['N']
        return ldmvn
<|endoftext|>


<commit_before>#!/usr/bin/python
import sys
import argparse
import subprocess
import shutil
import os.path
import platform
from subprocess import check_call
from subprocess import call

# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("-b", "--build", action='store_true')
parser.add_argument("-p", "--publish", action='store_true')
parser.add_argument("-c", "--clean", action='store_true')
parser.add_argument("-e", "--eclipse", action='store_true')
parser.add_argument("-t", "--test", action='store_true')
parser.add_argument("-a", "--all", action='store_true')
parser.add_argument("-v", "--version", default="SNAPSHOT", action='store_true')
args = parser.parse_args()

# Some helper variables
os.environ['MGEN_BUILD_VERSION'] = args.version
compiler = "mgen-compiler/target/mgen-compiler-assembly-" + args.version + ".jar"
compile_1 = "java -jar ../" + compiler + " "
javagenpath = "../mgen-javagenerator/target"
cppgenpath = "../mgen-cppgenerator/target"
jsgenpath = "../mgen-javascriptgenerator/target"
pluginPaths = ' plugin_paths="' + javagenpath + "," + cppgenpath + "," + jsgenpath + '"'

# Some helper fcns
def compile3(workingDir, project, outPath):
	check_call(compile_1 + project + pluginPaths + ' output_path="' + outPath + '"', cwd=workingDir, shell=True)

def compile(workingDir, project):
	compile3(workingDir, project, ".")

def fastbuild_step1():
	check_call(('sbt '
				'"project mgen_api" publish-local '
				'"project mgen_idlparser" publish-local '
				'"project mgen_jsonschemaparser" publish-local '
				'"project mgen_protobufparser" publish-local '
				'"project mgen_xmlschemaparser" publish-local '
				'"project mgen_idlgenerator" publish-local '
				'"project mgen_javalib" publish-local '
				'"project mgen_compiler" assembly publish-local '
				'"project mgen_javagenerator" publish-local '
				'"project mgen_cppgenerator" publish-local '
				'"project mgen_javascriptgenerator" publish-local '), shell=True)

def fastbuild_generate_code():
	check_call(compile_1 + 'model/project.xml plugin_paths="../mgen-javagenerator/target"', cwd="mgen-visualdesigner", shell=True)

def fastbuild_step2():
	check_call(('sbt "project mgen_visualdesigner" assembly publish-local '), shell=True)

def tests_generate_code(): # Ideally here we'd just generate once, not nLangs times.
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile3("mgen-integrationtests", 'models/depend/project.xml', "generated/depends")
	compile3("mgen-integrationtests", 'models/write/project.xml', "generated/write")
	compile3("mgen-integrationtests", 'models/read/project.xml', "generated/read")

def mkFolder(path):
	if not os.path.exists(path):
		os.makedirs(path)

def mkTestData(path, cfg, projname):
	if platform.system()=="Windows":
		check_call("msbuild " + projname + ".sln /p:Configuration=" + cfg, cwd=path, shell=True)
		check_call(cfg + "\\" + projname + ".exe", cwd=path, shell=True)
	else:
		check_call("make", cwd=path, shell=True)
		check_call("./" + projname, cwd=path, shell=True)

def tests_integration_generate_data():
	cfg = "RelwithDebInfo"
	dGenFolder = "mgen-integrationtests/generated/depends/data_generator"
	wGenFolder = "mgen-integrationtests/generated/write/data_generator"
	rGenFolder = "mgen-integrationtests/generated/read/data_generator"
	mkFolder(dGenFolder)
	mkFolder(wGenFolder)
	mkFolder(rGenFolder)
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/depends', cwd=dGenFolder, shell=True)
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/write', cwd=wGenFolder, shell=True)
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/read', cwd=rGenFolder, shell=True)
	mkTestData(dGenFolder, cfg, "generate_depends_testdata")
	mkTestData(wGenFolder, cfg, "generate_write_testdata")
	mkTestData(rGenFolder, cfg, "generate_read_testdata")

# clean
if args.clean or args.all:
	check_call("sbt clean", shell=True)
	#TODO: Remove all src_generated folders for tests and visualdesigner
	#TODO: Remove cpp test applications build folders

# build
if args.build or args.all or len(sys.argv) == 1:
	fastbuild_step1()
	fastbuild_generate_code()
	fastbuild_step2()

# test
if args.test or args.all:
	tests_generate_code()
	tests_integration_generate_data()
	#tests_integration_run()
	#tests_normal_run()
	#run integrationtests
	#run normal tests
	print("mgen-python-test: Not yet implemented!")

# eclipse
if args.eclipse or args.all:
	check_call('sbt eclipse', shell=True)

# publish
if args.publish or args.all:
	print("mgen-python-publish: Not yet implemented!")
<commit_msg>more python work...nuff for now<commit_after>#!/usr/bin/python
import sys
import argparse
import subprocess
import shutil
import os.path
import platform
import distutils.dir_util
from subprocess import check_call
from subprocess import call

# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("-b", "--build", action='store_true')
parser.add_argument("-p", "--publish", action='store_true')
parser.add_argument("-c", "--clean", action='store_true')
parser.add_argument("-e", "--eclipse", action='store_true')
parser.add_argument("-t", "--test", action='store_true')
parser.add_argument("-a", "--all", action='store_true')
parser.add_argument("-v", "--version", default="SNAPSHOT", action='store_true')
args = parser.parse_args()

# Some helper variables
os.environ['MGEN_BUILD_VERSION'] = args.version
compiler = "mgen-compiler/target/mgen-compiler-assembly-" + args.version + ".jar"
compile_1 = "java -jar ../" + compiler + " "
javagenpath = "../mgen-javagenerator/target"
cppgenpath = "../mgen-cppgenerator/target"
jsgenpath = "../mgen-javascriptgenerator/target"
pluginPaths = ' plugin_paths="' + javagenpath + "," + cppgenpath + "," + jsgenpath + '"'

# Some helper fcns
def compile3(workingDir, project, outPath):
	check_call(compile_1 + project + pluginPaths + ' output_path="' + outPath + '"', cwd=workingDir, shell=True)

def compile(workingDir, project):
	compile3(workingDir, project, ".")

def fastbuild_step1():
	check_call(('sbt '
				'"project mgen_api" publish-local '
				'"project mgen_idlparser" publish-local '
				'"project mgen_jsonschemaparser" publish-local '
				'"project mgen_protobufparser" publish-local '
				'"project mgen_xmlschemaparser" publish-local '
				'"project mgen_idlgenerator" publish-local '
				'"project mgen_javalib" publish-local '
				'"project mgen_compiler" assembly publish-local '
				'"project mgen_javagenerator" publish-local '
				'"project mgen_cppgenerator" publish-local '
				'"project mgen_javascriptgenerator" publish-local '), shell=True)

def fastbuild_generate_code():
	check_call(compile_1 + 'model/project.xml plugin_paths="../mgen-javagenerator/target"', cwd="mgen-visualdesigner", shell=True)

def fastbuild_step2():
	check_call(('sbt "project mgen_visualdesigner" assembly publish-local '), shell=True)

def tests_generate_code(): # Ideally here we'd just generate once, not nLangs times.
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-javalib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-cpplib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/transient_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/defaultvalues_testmodel/project.xml')
	compile("mgen-javascriptlib", '../mgen-compiler/src/test/resources/defaultvaluesreq_testmodel/project.xml')
	compile3("mgen-integrationtests", 'models/depend/project.xml', "generated/depends")
	compile3("mgen-integrationtests", 'models/write/project.xml', "generated/write")
	compile3("mgen-integrationtests", 'models/read/project.xml', "generated/read")

def mkFolder(path):
	if not os.path.exists(path):
		os.makedirs(path)

def rmFolder(path):
	if os.path.exists(path):
		shutil.rmtree(path)

def copyTree(src, dst):
	distutils.dir_util.copy_tree(src, dst)

def mkTestData(path, cfg, projname):
	if platform.system()=="Windows":
		check_call("msbuild " + projname + ".sln /p:Configuration=" + cfg, cwd=path, shell=True)
		check_call(cfg + "\\" + projname + ".exe", cwd=path, shell=True)
	else:
		check_call("make", cwd=path, shell=True)
		check_call("./" + projname, cwd=path, shell=True)

def tests_integration_cpp():
	cfg = "RelwithDebInfo"
	dGenFolder = "mgen-integrationtests/generated/depends/data_generator"
	wGenFolder = "mgen-integrationtests/generated/write/data_generator"
	rGenFolder = "mgen-integrationtests/generated/read/data_generator"
	mkFolder(dGenFolder)
	mkFolder(wGenFolder)
	mkFolder(rGenFolder)
	mkFolder("mgen-integrationtests/generated/depends/data_generated")
	mkFolder("mgen-integrationtests/generated/write/data_generated")
	mkFolder("mgen-integrationtests/generated/read/data_generated")
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/depends', cwd=dGenFolder, shell=True)
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/write', cwd=wGenFolder, shell=True)
	check_call('cmake -DCMAKE_BUILD_TYPE=' + cfg + ' ../../../build/read', cwd=rGenFolder, shell=True)
	mkTestData(dGenFolder, cfg, "generate_depends_testdata")
	mkTestData(wGenFolder, cfg, "generate_write_testdata")
	mkTestData(rGenFolder, cfg, "generate_read_testdata")

def tests_integration_java():
	dGenFolder = "mgen-integrationtests/javacheck/depends"
	wGenFolder = "mgen-integrationtests/javacheck/write"
	rGenFolder = "mgen-integrationtests/javacheck/read"
	copyTree("mgen-integrationtests/generated/depends/src_generated/java", dGenFolder + "/src_generated/test/java")
	copyTree("mgen-integrationtests/generated/write/src_generated/java", wGenFolder + "/src_generated/test/java")
	copyTree("mgen-integrationtests/generated/read/src_generated/java", rGenFolder + "/src_generated/test/java")
	check_call("sbt test", cwd=dGenFolder, shell=True)
	check_call("sbt test", cwd=wGenFolder, shell=True)
	check_call("sbt test", cwd=rGenFolder, shell=True)

def tests_integration_js():
	base = "mgen-integrationtests/javascriptcheck"
	dGenFolder = base + "/depends"
	wGenFolder = base + "/write"
	rGenFolder = base + "/read"
	copyTree("mgen-integrationtests/generated/depends/src_generated/javascript", dGenFolder + "/src_generated/test/javascript")
	copyTree("mgen-integrationtests/generated/write/src_generated/javascript", wGenFolder + "/src_generated/test/javascript")
	copyTree("mgen-integrationtests/generated/read/src_generated/javascript", rGenFolder + "/src_generated/test/javascript")
	# Can't batch, need sbt root project for that, but then we ouldnt find the resources :S
	check_call("sbt jasmine", cwd=dGenFolder, shell=True)
	check_call("sbt jasmine", cwd=wGenFolder, shell=True)
	check_call("sbt jasmine", cwd=rGenFolder, shell=True)

def tests_normal():
	# Can this be done in one batch? Doesn't seem so,
	# seems like cwd is wierd when batching test data isnt
	# copied over to the test dir
	check_call("sbt test", cwd="mgen-javalib", shell=True)
	# javascriptlib tests depend on mgen compiler....incorrectly. So they cant be run
	#check_call("sbt test", cwd="mgen-javascriptlib", shell=True)
	#  TODO: Cpp tests

# clean
if args.clean or args.all:
	check_call("sbt clean", shell=True)
	#TODO: Remove all src_generated folders for integration tests, tests and visualdesigner
	#TODO: Remove cpp test applications build folders
	#TODO: Remove generated root folder in integration tests

# build
if args.build or args.all or len(sys.argv) == 1:
	fastbuild_step1()
	fastbuild_generate_code()
	fastbuild_step2()

# test
if args.test or args.all:
	tests_generate_code()
	tests_integration_cpp()
	tests_integration_java()
	tests_integration_js()
	tests_normal()

# eclipse
if args.eclipse or args.all:
	check_call('sbt eclipse', shell=True)

# publish
if args.publish or args.all:
	print("mgen-python-publish: Not yet implemented!")
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Description:
#   Watch for file changes under ./src, and
#   trigger the configured gradle command on change.
#   Default command is 'gradle check'.
#
# Example:
#   python build.py                         # Runs 'gradle check'
#   python build.py clean compileJava test  # Runs 'gradle clean compileJava test'
#
# Dependencies:
#   watchdog
#
import sys
from os.path import dirname, realpath, join
import time
import subprocess
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileModifiedEvent

CURRENT_PATH = realpath(dirname(__file__))

class FileChangeHandler(FileSystemEventHandler):
    def __init__(self, cmd):
        self.cmd = cmd

    def on_modified(self, event):
        if (type(event) == FileModifiedEvent):
            print('%s: change in file %s detected.' % (time.asctime(), self.get_path(event)))
            subprocess.call(self.cmd)

    def get_path(self, event):
        return event.src_path.replace(CURRENT_PATH, '')

def wait_for_keyboard_interrupt(observer):
    try:
        while True:
            time.sleep(0.1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()

if __name__ == '__main__':
    cmd = ['gradle'] + sys.argv[1:] if len(sys.argv) > 1 else ['gradle', 'check']
    path = 'src'

    observer = Observer()
    observer.schedule(FileChangeHandler(cmd), path=path, recursive=True)

    print('Now watching changes in files under: \n\t%s\nThe following command will be run on any change: \n\t%s\n' %
        (join(CURRENT_PATH, path), ' '.join(cmd)))

    observer.start()
    wait_for_keyboard_interrupt(observer)
<commit_msg>Improved examples and shebang for homebrew'ed Mac.<commit_after>#!/usr/local/bin/python
#
# Description:
#   Watch for file changes under ./src, and
#   trigger the configured gradle command on change.
#   Default command is 'gradle check'.
#
# Example:
#   ./build.py                         # Runs 'gradle check'
#   ./build.py clean compileJava test  # Runs 'gradle clean compileJava test'
#
# Dependencies:
#   watchdog
#
import sys
from os.path import dirname, realpath, join
import time
import subprocess
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler, FileModifiedEvent

CURRENT_PATH = realpath(dirname(__file__))

class FileChangeHandler(FileSystemEventHandler):
    def __init__(self, cmd):
        self.cmd = cmd

    def on_modified(self, event):
        if (type(event) == FileModifiedEvent):
            print('%s: change in file %s detected.' % (time.asctime(), self.get_path(event)))
            subprocess.call(self.cmd)

    def get_path(self, event):
        return event.src_path.replace(CURRENT_PATH, '')

def wait_for_keyboard_interrupt(observer):
    try:
        while True:
            time.sleep(0.1)
    except KeyboardInterrupt:
        observer.stop()
    observer.join()

if __name__ == '__main__':
    cmd = ['gradle'] + sys.argv[1:] if len(sys.argv) > 1 else ['gradle', 'check']
    path = 'src'

    observer = Observer()
    observer.schedule(FileChangeHandler(cmd), path=path, recursive=True)

    print('Now watching changes in files under: \n\t%s\nThe following command will be run on any change: \n\t%s\n' %
        (join(CURRENT_PATH, path), ' '.join(cmd)))

    observer.start()
    wait_for_keyboard_interrupt(observer)
<|endoftext|>