<commit_before># Cura Storage Provider
#
# Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
    Support functions for partitioning.
"""

import parted
import pyanaconda.storage
import pywbem
from wrapper.common import storage

TYPE_MBR = 'MBR'
TYPE_EMBR = 'EMBR'
TYPE_GPT = 'GPT'
LABEL_MBR = 'msdos'
LABEL_GPT = 'gpt'
GPT_SIZE = 34

def getLogicalPartitionStart(partition):
    """ Return starting sector of logical partition metadata."""
    disk = partition.disk
    ext = disk.format.partedDisk.getExtendedPartition()
    if not ext:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find extended partition.')

    metadata = None
    part = ext.nextPartition()
    while part is not None:
        if (part.type & parted.PARTITION_LOGICAL
                and part.type & parted.PARTITION_METADATA):
            metadata = part
        if part.path == partition.path:
            break
        part = part.nextPartition()

    print 'partition', part, 'metadata', metadata
    if not part:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find the partition on the disk.')
    if not metadata:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find metadata for the partition.')

    return metadata.geometry.start

def deletePartition(partition):
    # Removing runs as a transaction, I get errors when using
    #action = pyanaconda.storage.deviceaction.ActionDestroyDevice(partition)
    #storage.devicetree.registerAction(action)
    #action.execute()
    #storage.devicetree._actions = []
    storage.destroyDevice(partition)
    storage.devicetree.processActions(dryRun=False)
    storage.devicetree._actions = []

def createPartition(partition):
    action = pyanaconda.storage.deviceaction.ActionCreateDevice(partition)
    storage.devicetree.registerAction(action)
    pyanaconda.storage.partitioning.doPartitioning(storage=storage)
    # patched Anaconda is needed for this to work
    partition.disk.format.resetPartedDisk()
    action.execute()
    storage.devicetree._actions = []

def createMBR(disk):
    if storage.deviceDeps(disk):
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Previous partition table must be empty.')
    fmt = pyanaconda.storage.formats.getFormat('disklabel', labelType=LABEL_MBR)
    action = pyanaconda.storage.deviceaction.ActionCreateFormat(disk, fmt)
    storage.devicetree.registerAction(action)
    action.execute()

def createGPT(disk):
    if storage.deviceDeps(disk):
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Previous partition table must be empty.')
    fmt = pyanaconda.storage.formats.getFormat('disklabel', labelType=LABEL_GPT)
    action = pyanaconda.storage.deviceaction.ActionCreateFormat(disk, fmt)
    storage.devicetree.registerAction(action)
    action.execute()
<commit_msg>providers: Fixed DiskPartitionConfigurationService freezing when deleting a partition.<commit_after># Cura Storage Provider
#
# Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
    Support functions for partitioning.
"""

import parted
import pyanaconda.storage
import pywbem
from wrapper.common import storage

TYPE_MBR = 'MBR'
TYPE_EMBR = 'EMBR'
TYPE_GPT = 'GPT'
LABEL_MBR = 'msdos'
LABEL_GPT = 'gpt'
GPT_SIZE = 34

def getLogicalPartitionStart(partition):
    """ Return starting sector of logical partition metadata."""
    disk = partition.disk
    ext = disk.format.partedDisk.getExtendedPartition()
    if not ext:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find extended partition.')

    metadata = None
    part = ext.nextPartition()
    while part is not None:
        if (part.type & parted.PARTITION_LOGICAL
                and part.type & parted.PARTITION_METADATA):
            metadata = part
        if part.path == partition.path:
            break
        part = part.nextPartition()

    print 'partition', part, 'metadata', metadata
    if not part:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find the partition on the disk.')
    if not metadata:
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Cannot find metadata for the partition.')

    return metadata.geometry.start

def deletePartition(partition):
    # I get errors when using
    #action = pyanaconda.storage.deviceaction.ActionDestroyDevice(partition)
    #storage.devicetree.registerAction(action)
    #action.execute()
    #storage.devicetree._actions = []
    #
    # Because:
    # - execute() calls devicetree._removeDevice
    #    -> it calls
    #        part.disk.format.removePartition(part.partedPartition)
    #
    # - execute calls part->destroy()
    #    -> it calls again part.disk.format.removePartition(part.partedPartition)

    # TODO: fix Anaconda to reliably remove partitions.
    # dirty hack
    storage.devicetree._removeDevice(partition)
    partition.disk.format.commit()
    partition.partedDevice.removeFromCache()

def createPartition(partition):
    action = pyanaconda.storage.deviceaction.ActionCreateDevice(partition)
    storage.devicetree.registerAction(action)
    pyanaconda.storage.partitioning.doPartitioning(storage=storage)
    # patched Anaconda is needed for this to work
    partition.disk.format.resetPartedDisk()
    action.execute()
    storage.devicetree._actions = []

def createMBR(disk):
    if storage.deviceDeps(disk):
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Previous partition table must be empty.')
    fmt = pyanaconda.storage.formats.getFormat('disklabel', labelType=LABEL_MBR)
    action = pyanaconda.storage.deviceaction.ActionCreateFormat(disk, fmt)
    storage.devicetree.registerAction(action)
    action.execute()

def createGPT(disk):
    if storage.deviceDeps(disk):
        raise pywbem.CIMError(pywbem.CIM_ERR_FAILED, 'Previous partition table must be empty.')
    fmt = pyanaconda.storage.formats.getFormat('disklabel', labelType=LABEL_GPT)
    action = pyanaconda.storage.deviceaction.ActionCreateFormat(disk, fmt)
    storage.devicetree.registerAction(action)
    action.execute()
<|endoftext|>


<commit_before>#
# Copyright (c) 2010-2011 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import logging
log = logging.getLogger(__name__)

from twisted.internet import reactor
from twisted.internet import task as ti_task
from twisted.web import error as tw_error
from xml.dom import minidom

from rmake3.lib.twisted_extras import tools
from rpath_repeater.utils.http import HTTPClientFactory
from rpath_repeater.utils.xmlutils import XML

class ReportingMixIn(object):
    """
    Assumes:
        resultsLocation
        ReportingXmlTag (class variable)
    """
    _serializer = None
    retryCount = 5
    retryInterval = 3

    def postResults(self, elt=None, method=None, location=None,
            collapsible=False, retry=True):
        if method is None:
            method = 'PUT'
        if location is None:
            location = self.getResultsUrl()
        host, port, path = self._getResultsLocation(location)
        if not path:
            return
        if elt is None:
            dom = minidom.parseString(self.job.data)
            elt = dom.firstChild
        if isinstance(elt, basestring):
            # We were given an XML string, no need to postprocess it
            data = elt
        else:
            elt = self.postprocessXmlNode(elt)
            data = self.toXml(elt)
        headers = {
            'Content-Type' : 'application/xml; charset="utf-8"',
            'Host' : host, }
        self.postprocessHeaders(elt, headers)
        # Serialize posts to make sure they arrive in chronological order
        if self._serializer is None:
            self._serializer = tools.Serializer()
        connArgs = (host, port)
        factArgs = dict(url=path, method=method, postdata=data,
                headers=headers)
        retries = self.retryCount if retry else 0
        return self._serializer.call(self._doPost, collapsible=collapsible,
                args=(connArgs, factArgs, retries))

    def _doPost(self, connArgs, factArgs, retries=0):
        fact = HTTPClientFactory(**factArgs)
        @fact.deferred.addCallback
        def processResult(result):
            log.debug("Received result for %s: %s", host, result)
            return result
        @fact.deferred.addErrback
        def processError(error):
            if (retries and error.check(tw_error.Error)
                    and error.value.status == '401'):
                log.debug("Got authorization error posting status update, "
                        "trying again")
                return ti_task.deferLater(reactor, self.retryInterval,
                        self._doPost, connArgs, factArgs, retries - 1)
            log.error("Error posting status update for job %s of type %s: %s",
                    self.job.job_uuid, self.job.job_type,
                    error.getErrorMessage())
        host, port = connArgs
        reactor.connectTCP(host, port, fact)
        return fact.deferred

    def getResultsUrl(self):
        if self.resultsLocation:
            return self.resultsLocation
        return self.jobUrl

    def _getResultsLocation(self, location):
        if location is None:
            return None, None, None
        if hasattr(location, 'host'):
            return location.host, location.port, location.path
        host = location.get('host', 'localhost')
        port = location.get('port', 80)
        path = location.get('path')
        return host, port, path

    def postStatus(self):
        el = self.newJobElement()
        xml = self.toXml(el)
        return self.postResults(xml, location=self.jobUrl)

    def postFailure(self, method=None):
        el = XML.Element(self.ReportingXmlTag)
        return self.postResults(el, method=method)

    def postprocessXmlNode(self, elt):
        return elt

    def postprocessHeaders(self, elt, headers):
        pass

    def newJobElement(self):
        T = XML.Text
        status = self.job.status
        if status.failed:
            state = 'Failed'
        elif status.completed:
            state = 'Completed'
        else:
            state = 'Running'
        return XML.Element("job",
            T("job_uuid", self.job.job_uuid),
            T("job_state", state),
            T("status_code", status.code),
            T("status_text", status.text),
            T("status_detail", status.detail or ''),
            )

    def addJobResults(self, job, results):
        resultsNode = XML.Element("results", results)
        job.appendChild(resultsNode)
        return job

    @classmethod
    def toXml(cls, elt):
        return XML.toString(elt)
<commit_msg>Translate task status of 0 to 'job queued'<commit_after>#
# Copyright (c) 2010-2011 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import logging
log = logging.getLogger(__name__)

from twisted.internet import reactor
from twisted.internet import task as ti_task
from twisted.web import error as tw_error
from xml.dom import minidom

from rmake3.lib.twisted_extras import tools
from rpath_repeater.utils.http import HTTPClientFactory
from rpath_repeater.utils.xmlutils import XML

class ReportingMixIn(object):
    """
    Assumes:
        resultsLocation
        ReportingXmlTag (class variable)
    """
    _serializer = None
    retryCount = 5
    retryInterval = 3

    def postResults(self, elt=None, method=None, location=None,
            collapsible=False, retry=True):
        if method is None:
            method = 'PUT'
        if location is None:
            location = self.getResultsUrl()
        host, port, path = self._getResultsLocation(location)
        if not path:
            return
        if elt is None:
            dom = minidom.parseString(self.job.data)
            elt = dom.firstChild
        if isinstance(elt, basestring):
            # We were given an XML string, no need to postprocess it
            data = elt
        else:
            elt = self.postprocessXmlNode(elt)
            data = self.toXml(elt)
        headers = {
            'Content-Type' : 'application/xml; charset="utf-8"',
            'Host' : host, }
        self.postprocessHeaders(elt, headers)
        # Serialize posts to make sure they arrive in chronological order
        if self._serializer is None:
            self._serializer = tools.Serializer()
        connArgs = (host, port)
        factArgs = dict(url=path, method=method, postdata=data,
                headers=headers)
        retries = self.retryCount if retry else 0
        return self._serializer.call(self._doPost, collapsible=collapsible,
                args=(connArgs, factArgs, retries))

    def _doPost(self, connArgs, factArgs, retries=0):
        fact = HTTPClientFactory(**factArgs)
        @fact.deferred.addCallback
        def processResult(result):
            log.debug("Received result for %s: %s", host, result)
            return result
        @fact.deferred.addErrback
        def processError(error):
            if (retries and error.check(tw_error.Error)
                    and error.value.status == '401'):
                log.debug("Got authorization error posting status update, "
                        "trying again")
                return ti_task.deferLater(reactor, self.retryInterval,
                        self._doPost, connArgs, factArgs, retries - 1)
            log.error("Error posting status update for job %s of type %s: %s",
                    self.job.job_uuid, self.job.job_type,
                    error.getErrorMessage())
        host, port = connArgs
        reactor.connectTCP(host, port, fact)
        return fact.deferred

    def getResultsUrl(self):
        if self.resultsLocation:
            return self.resultsLocation
        return self.jobUrl

    def _getResultsLocation(self, location):
        if location is None:
            return None, None, None
        if hasattr(location, 'host'):
            return location.host, location.port, location.path
        host = location.get('host', 'localhost')
        port = location.get('port', 80)
        path = location.get('path')
        return host, port, path

    def postStatus(self):
        el = self.newJobElement()
        xml = self.toXml(el)
        return self.postResults(xml, location=self.jobUrl)

    def postFailure(self, method=None):
        el = XML.Element(self.ReportingXmlTag)
        return self.postResults(el, method=method)

    def postprocessXmlNode(self, elt):
        return elt

    def postprocessHeaders(self, elt, headers):
        pass

    def newJobElement(self):
        T = XML.Text
        status = self.job.status
        if status.failed:
            state = 'Failed'
        elif status.completed:
            state = 'Completed'
        else:
            state = 'Running'
        if status.code == 0:
            code, text = 100, 'Job queued'
        else:
            code, text = status.code, status.text
        return XML.Element("job",
            T("job_uuid", self.job.job_uuid),
            T("job_state", state),
            T("status_code", code),
            T("status_text", text),
            T("status_detail", status.detail or ''),
            )

    def addJobResults(self, job, results):
        resultsNode = XML.Element("results", results)
        job.appendChild(resultsNode)
        return job

    @classmethod
    def toXml(cls, elt):
        return XML.toString(elt)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import shutil
import subprocess
import sys

from setuptools import setup, find_packages, Distribution
import setuptools.command.build_ext as _build_ext

# Ideally, we could include these files by putting them in a
# MANIFEST.in or using the package_data argument to setup, but the
# MANIFEST.in gets applied at the very beginning when setup.py runs
# before these files have been created, so we have to move the files
# manually.

# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.

ray_files = [
    "ray/core/src/ray/thirdparty/redis/src/redis-server",
    "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
    "ray/core/src/plasma/plasma_store_server",
    "ray/core/src/ray/raylet/liblocal_scheduler_library_python.so",
    "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
    "ray/WebUI.ipynb"
]

# These are the directories where automatically generated Python flatbuffer
# bindings are created.
generated_python_directories = [
    "ray/core/generated", "ray/core/generated/ray",
    "ray/core/generated/ray/protocol"
]

optional_ray_files = []

ray_ui_files = [
    "ray/core/src/catapult_files/index.html",
    "ray/core/src/catapult_files/trace_viewer_full.html"
]

ray_autoscaler_files = [
    "ray/autoscaler/aws/example-full.yaml",
    "ray/autoscaler/gcp/example-full.yaml",
    "ray/autoscaler/local/example-full.yaml",
]

if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
    ray_files += [
        "ray/core/src/credis/build/src/libmember.so",
        "ray/core/src/credis/build/src/libmaster.so",
        "ray/core/src/credis/redis/src/redis-server"
    ]

# The UI files are mandatory if the INCLUDE_UI environment variable equals 1.
# Otherwise, they are optional.
if "INCLUDE_UI" in os.environ and os.environ["INCLUDE_UI"] == "1":
    ray_files += ray_ui_files
else:
    optional_ray_files += ray_ui_files

optional_ray_files += ray_autoscaler_files

extras = {"rllib": ["pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"]}

class build_ext(_build_ext.build_ext):
    def run(self):
        # Note: We are passing in sys.executable so that we use the same
        # version of Python to build pyarrow inside the build.sh script. Note
        # that certain flags will not be passed along such as --user or sudo.
        # TODO(rkn): Fix this.
        subprocess.check_call(["../build.sh", "-p", sys.executable])

        # We also need to install pyarrow along with Ray, so make sure that the
        # relevant non-Python pyarrow files get copied.
        pyarrow_files = []
        for (root, dirs, filenames) in os.walk("./ray/pyarrow_files/pyarrow"):
            for name in filenames:
                pyarrow_files.append(os.path.join(root, name))

        files_to_include = ray_files + pyarrow_files

        # Copy over the autogenerated flatbuffer Python bindings.
        for directory in generated_python_directories:
            for filename in os.listdir(directory):
                if filename[-3:] == ".py":
                    files_to_include.append(os.path.join(directory, filename))

        for filename in files_to_include:
            self.move_file(filename)

        # Try to copy over the optional files.
        for filename in optional_ray_files:
            try:
                self.move_file(filename)
            except Exception:
                print("Failed to copy optional file {}. This is ok."
                      .format(filename))

    def move_file(self, filename):
        # TODO(rkn): This feels very brittle. It may not handle all cases. See
        # https://github.com/apache/arrow/blob/master/python/setup.py for an
        # example.
        source = filename
        destination = os.path.join(self.build_lib, filename)
        # Create the target directory if it doesn't already exist.
        parent_directory = os.path.dirname(destination)
        if not os.path.exists(parent_directory):
            os.makedirs(parent_directory)
        print("Copying {} to {}.".format(source, destination))
        shutil.copy(source, destination)

class BinaryDistribution(Distribution):
    def has_ext_modules(self):
        return True

def find_version(*filepath):
    # Extract version information from filepath
    here = os.path.abspath(os.path.dirname(__file__))
    with open(os.path.join(here, *filepath)) as fp:
        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                                  fp.read(), re.M)
        if version_match:
            return version_match.group(1)
        raise RuntimeError("Unable to find version string.")

setup(
    name="ray",
    version=find_version("ray", "__init__.py"),
    description=("A system for parallel and distributed Python that unifies "
                 "the ML ecosystem."),
    long_description=open("../README.rst").read(),
    url="https://github.com/ray-project/ray",
    keywords=("ray distributed parallel machine-learning "
              "reinforcement-learning deep-learning python"),
    packages=find_packages(),
    cmdclass={"build_ext": build_ext},
    # The BinaryDistribution argument triggers build_ext.
    distclass=BinaryDistribution,
    install_requires=[
        "numpy",
        "funcsigs",
        "click",
        "colorama",
        "pytest",
        "pyyaml",
        "redis",
        "faulthandler;python_version<'3'",
        "setproctitle",
        # The six module is required by pyarrow.
        "six >= 1.0.0",
        "flatbuffers"
    ],
    setup_requires=["cython >= 0.27, < 0.28"],
    extras_require=extras,
    entry_points={
        "console_scripts": [
            "ray=ray.scripts.scripts:main",
            "rllib=ray.rllib.scripts:cli [rllib]"
        ]
    },
    include_package_data=True,
    zip_safe=False,
    license="Apache 2.0")
<commit_msg>Update redis version in setup.py (#3333)<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import shutil
import subprocess
import sys

from setuptools import setup, find_packages, Distribution
import setuptools.command.build_ext as _build_ext

# Ideally, we could include these files by putting them in a
# MANIFEST.in or using the package_data argument to setup, but the
# MANIFEST.in gets applied at the very beginning when setup.py runs
# before these files have been created, so we have to move the files
# manually.

# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.

ray_files = [
    "ray/core/src/ray/thirdparty/redis/src/redis-server",
    "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
    "ray/core/src/plasma/plasma_store_server",
    "ray/core/src/ray/raylet/liblocal_scheduler_library_python.so",
    "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
    "ray/WebUI.ipynb"
]

# These are the directories where automatically generated Python flatbuffer
# bindings are created.
generated_python_directories = [
    "ray/core/generated", "ray/core/generated/ray",
    "ray/core/generated/ray/protocol"
]

optional_ray_files = []

ray_ui_files = [
    "ray/core/src/catapult_files/index.html",
    "ray/core/src/catapult_files/trace_viewer_full.html"
]

ray_autoscaler_files = [
    "ray/autoscaler/aws/example-full.yaml",
    "ray/autoscaler/gcp/example-full.yaml",
    "ray/autoscaler/local/example-full.yaml",
]

if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
    ray_files += [
        "ray/core/src/credis/build/src/libmember.so",
        "ray/core/src/credis/build/src/libmaster.so",
        "ray/core/src/credis/redis/src/redis-server"
    ]

# The UI files are mandatory if the INCLUDE_UI environment variable equals 1.
# Otherwise, they are optional.
if "INCLUDE_UI" in os.environ and os.environ["INCLUDE_UI"] == "1":
    ray_files += ray_ui_files
else:
    optional_ray_files += ray_ui_files

optional_ray_files += ray_autoscaler_files

extras = {"rllib": ["pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"]}

class build_ext(_build_ext.build_ext):
    def run(self):
        # Note: We are passing in sys.executable so that we use the same
        # version of Python to build pyarrow inside the build.sh script. Note
        # that certain flags will not be passed along such as --user or sudo.
        # TODO(rkn): Fix this.
        subprocess.check_call(["../build.sh", "-p", sys.executable])

        # We also need to install pyarrow along with Ray, so make sure that the
        # relevant non-Python pyarrow files get copied.
        pyarrow_files = []
        for (root, dirs, filenames) in os.walk("./ray/pyarrow_files/pyarrow"):
            for name in filenames:
                pyarrow_files.append(os.path.join(root, name))

        files_to_include = ray_files + pyarrow_files

        # Copy over the autogenerated flatbuffer Python bindings.
        for directory in generated_python_directories:
            for filename in os.listdir(directory):
                if filename[-3:] == ".py":
                    files_to_include.append(os.path.join(directory, filename))

        for filename in files_to_include:
            self.move_file(filename)

        # Try to copy over the optional files.
        for filename in optional_ray_files:
            try:
                self.move_file(filename)
            except Exception:
                print("Failed to copy optional file {}. This is ok."
                      .format(filename))

    def move_file(self, filename):
        # TODO(rkn): This feels very brittle. It may not handle all cases. See
        # https://github.com/apache/arrow/blob/master/python/setup.py for an
        # example.
        source = filename
        destination = os.path.join(self.build_lib, filename)
        # Create the target directory if it doesn't already exist.
        parent_directory = os.path.dirname(destination)
        if not os.path.exists(parent_directory):
            os.makedirs(parent_directory)
        print("Copying {} to {}.".format(source, destination))
        shutil.copy(source, destination)

class BinaryDistribution(Distribution):
    def has_ext_modules(self):
        return True

def find_version(*filepath):
    # Extract version information from filepath
    here = os.path.abspath(os.path.dirname(__file__))
    with open(os.path.join(here, *filepath)) as fp:
        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                                  fp.read(), re.M)
        if version_match:
            return version_match.group(1)
        raise RuntimeError("Unable to find version string.")

setup(
    name="ray",
    version=find_version("ray", "__init__.py"),
    description=("A system for parallel and distributed Python that unifies "
                 "the ML ecosystem."),
    long_description=open("../README.rst").read(),
    url="https://github.com/ray-project/ray",
    keywords=("ray distributed parallel machine-learning "
              "reinforcement-learning deep-learning python"),
    packages=find_packages(),
    cmdclass={"build_ext": build_ext},
    # The BinaryDistribution argument triggers build_ext.
    distclass=BinaryDistribution,
    install_requires=[
        "numpy",
        "funcsigs",
        "click",
        "colorama",
        "pytest",
        "pyyaml",
        "redis~=2.10.6",
        "faulthandler;python_version<'3'",
        "setproctitle",
        # The six module is required by pyarrow.
        "six >= 1.0.0",
        "flatbuffers"
    ],
    setup_requires=["cython >= 0.27, < 0.28"],
    extras_require=extras,
    entry_points={
        "console_scripts": [
            "ray=ray.scripts.scripts:main",
            "rllib=ray.rllib.scripts:cli [rllib]"
        ]
    },
    include_package_data=True,
    zip_safe=False,
    license="Apache 2.0")
<|endoftext|>


<commit_before>from itertools import chain
import os
import re
import shutil
import subprocess
import sys

from setuptools import setup, find_packages, Distribution
import setuptools.command.build_ext as _build_ext

# Ideally, we could include these files by putting them in a
# MANIFEST.in or using the package_data argument to setup, but the
# MANIFEST.in gets applied at the very beginning when setup.py runs
# before these files have been created, so we have to move the files
# manually.

# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
ray_files = [
    "ray/core/src/ray/thirdparty/redis/src/redis-server",
    "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
    "ray/core/src/plasma/plasma_store_server",
    "ray/_raylet.so",
    "ray/core/src/ray/raylet/raylet_monitor",
    "ray/core/src/ray/gcs/gcs_server",
    "ray/core/src/ray/raylet/raylet",
    "ray/dashboard/dashboard.py",
    "ray/streaming/_streaming.so",
]

build_java = os.getenv("RAY_INSTALL_JAVA") == "1"
if build_java:
    ray_files.append("ray/jars/ray_dist.jar")

# These are the directories where automatically generated Python protobuf
# bindings are created.
generated_python_directories = [
    "ray/core/generated",
    "ray/streaming/generated",
]

optional_ray_files = []

ray_autoscaler_files = [
    "ray/autoscaler/aws/example-full.yaml",
    "ray/autoscaler/azure/example-full.yaml",
    "ray/autoscaler/gcp/example-full.yaml",
    "ray/autoscaler/local/example-full.yaml",
    "ray/autoscaler/kubernetes/example-full.yaml",
    "ray/autoscaler/kubernetes/kubectl-rsync.sh",
    "ray/autoscaler/ray-schema.json"
]

ray_project_files = [
    "ray/projects/schema.json", "ray/projects/templates/cluster_template.yaml",
    "ray/projects/templates/project_template.yaml",
    "ray/projects/templates/requirements.txt"
]

ray_dashboard_files = [
    os.path.join(dirpath, filename)
    for dirpath, dirnames, filenames in os.walk("ray/dashboard/client/build")
    for filename in filenames
]

optional_ray_files += ray_autoscaler_files
optional_ray_files += ray_project_files
optional_ray_files += ray_dashboard_files

if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
    ray_files += [
        "ray/core/src/credis/build/src/libmember.so",
        "ray/core/src/credis/build/src/libmaster.so",
        "ray/core/src/credis/redis/src/redis-server"
    ]

extras = {
    "debug": [],
    "dashboard": [],
    "serve": ["uvicorn", "pygments", "werkzeug", "flask", "pandas", "blist"],
    "tune": ["tabulate", "tensorboardX"]
}

extras["rllib"] = extras["tune"] + [
    "atari_py",
    "dm_tree",
    "gym[atari]",
    "lz4",
    "opencv-python-headless",
    "pyyaml",
    "scipy",
]

extras["streaming"] = ["msgpack >= 0.6.2"]

extras["all"] = list(set(chain.from_iterable(extras.values())))

class build_ext(_build_ext.build_ext):
    def run(self):
        # Note: We are passing in sys.executable so that we use the same
        # version of Python to build packages inside the build.sh script. Note
        # that certain flags will not be passed along such as --user or sudo.
        # TODO(rkn): Fix this.
        command = ["../build.sh", "-p", sys.executable]
        if build_java:
            # Also build binaries for Java if the above env variable exists.
            command += ["-l", "python,java"]
        subprocess.check_call(command)

        # We also need to install pickle5 along with Ray, so make sure that the
        # relevant non-Python pickle5 files get copied.
        pickle5_files = self.walk_directory("./ray/pickle5_files/pickle5")

        thirdparty_files = self.walk_directory("./ray/thirdparty_files")

        files_to_include = ray_files + pickle5_files + thirdparty_files

        # Copy over the autogenerated protobuf Python bindings.
        for directory in generated_python_directories:
            for filename in os.listdir(directory):
                if filename[-3:] == ".py":
                    files_to_include.append(os.path.join(directory, filename))

        for filename in files_to_include:
            self.move_file(filename)

        # Try to copy over the optional files.
        for filename in optional_ray_files:
            try:
                self.move_file(filename)
            except Exception:
                print("Failed to copy optional file {}. This is ok."
                      .format(filename))

    def walk_directory(self, directory):
        file_list = []
        for (root, dirs, filenames) in os.walk(directory):
            for name in filenames:
                file_list.append(os.path.join(root, name))
        return file_list

    def move_file(self, filename):
        # TODO(rkn): This feels very brittle. It may not handle all cases. See
        # https://github.com/apache/arrow/blob/master/python/setup.py for an
        # example.
        source = filename
        destination = os.path.join(self.build_lib, filename)
        # Create the target directory if it doesn't already exist.
        parent_directory = os.path.dirname(destination)
        if not os.path.exists(parent_directory):
            os.makedirs(parent_directory)
        if not os.path.exists(destination):
            print("Copying {} to {}.".format(source, destination))
            shutil.copy(source, destination, follow_symlinks=True)

class BinaryDistribution(Distribution):
    def has_ext_modules(self):
        return True

def find_version(*filepath):
    # Extract version information from filepath
    here = os.path.abspath(os.path.dirname(__file__))
    with open(os.path.join(here, *filepath)) as fp:
        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                                  fp.read(), re.M)
        if version_match:
            return version_match.group(1)
        raise RuntimeError("Unable to find version string.")

requires = [
    "numpy >= 1.16",
    "filelock",
    "jsonschema",
    "funcsigs",
    "click",
    "colorama",
    "packaging",
    "pytest",
    "pyyaml",
    "jsonschema",
    "redis>=3.3.2",
    # NOTE: Don't upgrade the version of six! Doing so causes installation
    # problems. See https://github.com/ray-project/ray/issues/4169.
    "six >= 1.0.0",
    "faulthandler;python_version<'3.3'",
    "protobuf >= 3.8.0",
    "cloudpickle",
    "py-spy >= 0.2.0",
    "aiohttp",
    "google",
    "grpcio"
]

setup(
    name="ray",
    version=find_version("ray", "__init__.py"),
    author="Ray Team",
    author_email="ray-dev@googlegroups.com",
    description=("A system for parallel and distributed Python that unifies "
                 "the ML ecosystem."),
    long_description=open("../README.rst").read(),
    url="https://github.com/ray-project/ray",
    keywords=("ray distributed parallel machine-learning "
              "reinforcement-learning deep-learning python"),
    packages=find_packages(),
    cmdclass={"build_ext": build_ext},
    # The BinaryDistribution argument triggers build_ext.
    distclass=BinaryDistribution,
    install_requires=requires,
    setup_requires=["cython >= 0.29"],
    extras_require=extras,
    entry_points={
        "console_scripts": [
            "ray=ray.scripts.scripts:main",
            "rllib=ray.rllib.scripts:cli [rllib]", "tune=ray.tune.scripts:cli"
        ]
    },
    include_package_data=True,
    zip_safe=False,
    license="Apache 2.0")
<commit_msg>Remove duplicate jsonschema from setup.py (#7665)<commit_after>from itertools import chain
import os
import re
import shutil
import subprocess
import sys

from setuptools import setup, find_packages, Distribution
import setuptools.command.build_ext as _build_ext

# Ideally, we could include these files by putting them in a
# MANIFEST.in or using the package_data argument to setup, but the
# MANIFEST.in gets applied at the very beginning when setup.py runs
# before these files have been created, so we have to move the files
# manually.

# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
ray_files = [
    "ray/core/src/ray/thirdparty/redis/src/redis-server",
    "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
    "ray/core/src/plasma/plasma_store_server",
    "ray/_raylet.so",
    "ray/core/src/ray/raylet/raylet_monitor",
    "ray/core/src/ray/gcs/gcs_server",
    "ray/core/src/ray/raylet/raylet",
    "ray/dashboard/dashboard.py",
    "ray/streaming/_streaming.so",
]

build_java = os.getenv("RAY_INSTALL_JAVA") == "1"
if build_java:
    ray_files.append("ray/jars/ray_dist.jar")

# These are the directories where automatically generated Python protobuf
# bindings are created.
generated_python_directories = [
    "ray/core/generated",
    "ray/streaming/generated",
]

optional_ray_files = []

ray_autoscaler_files = [
    "ray/autoscaler/aws/example-full.yaml",
    "ray/autoscaler/azure/example-full.yaml",
    "ray/autoscaler/gcp/example-full.yaml",
    "ray/autoscaler/local/example-full.yaml",
    "ray/autoscaler/kubernetes/example-full.yaml",
    "ray/autoscaler/kubernetes/kubectl-rsync.sh",
    "ray/autoscaler/ray-schema.json"
]

ray_project_files = [
    "ray/projects/schema.json", "ray/projects/templates/cluster_template.yaml",
    "ray/projects/templates/project_template.yaml",
    "ray/projects/templates/requirements.txt"
]

ray_dashboard_files = [
    os.path.join(dirpath, filename)
    for dirpath, dirnames, filenames in os.walk("ray/dashboard/client/build")
    for filename in filenames
]

optional_ray_files += ray_autoscaler_files
optional_ray_files += ray_project_files
optional_ray_files += ray_dashboard_files

if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
    ray_files += [
        "ray/core/src/credis/build/src/libmember.so",
        "ray/core/src/credis/build/src/libmaster.so",
        "ray/core/src/credis/redis/src/redis-server"
    ]

extras = {
    "debug": [],
    "dashboard": [],
    "serve": ["uvicorn", "pygments", "werkzeug", "flask", "pandas", "blist"],
    "tune": ["tabulate", "tensorboardX"]
}

extras["rllib"] = extras["tune"] + [
    "atari_py",
    "dm_tree",
    "gym[atari]",
    "lz4",
    "opencv-python-headless",
    "pyyaml",
    "scipy",
]

extras["streaming"] = ["msgpack >= 0.6.2"]

extras["all"] = list(set(chain.from_iterable(extras.values())))

class build_ext(_build_ext.build_ext):
    def run(self):
        # Note: We are passing in sys.executable so that we use the same
        # version of Python to build packages inside the build.sh script. Note
        # that certain flags will not be passed along such as --user or sudo.
        # TODO(rkn): Fix this.
        command = ["../build.sh", "-p", sys.executable]
        if build_java:
            # Also build binaries for Java if the above env variable exists.
            command += ["-l", "python,java"]
        subprocess.check_call(command)

        # We also need to install pickle5 along with Ray, so make sure that the
        # relevant non-Python pickle5 files get copied.
        pickle5_files = self.walk_directory("./ray/pickle5_files/pickle5")

        thirdparty_files = self.walk_directory("./ray/thirdparty_files")

        files_to_include = ray_files + pickle5_files + thirdparty_files

        # Copy over the autogenerated protobuf Python bindings.
        for directory in generated_python_directories:
            for filename in os.listdir(directory):
                if filename[-3:] == ".py":
                    files_to_include.append(os.path.join(directory, filename))

        for filename in files_to_include:
            self.move_file(filename)

        # Try to copy over the optional files.
        for filename in optional_ray_files:
            try:
                self.move_file(filename)
            except Exception:
                print("Failed to copy optional file {}. This is ok."
                      .format(filename))

    def walk_directory(self, directory):
        file_list = []
        for (root, dirs, filenames) in os.walk(directory):
            for name in filenames:
                file_list.append(os.path.join(root, name))
        return file_list

    def move_file(self, filename):
        # TODO(rkn): This feels very brittle. It may not handle all cases. See
        # https://github.com/apache/arrow/blob/master/python/setup.py for an
        # example.
        source = filename
        destination = os.path.join(self.build_lib, filename)
        # Create the target directory if it doesn't already exist.
        parent_directory = os.path.dirname(destination)
        if not os.path.exists(parent_directory):
            os.makedirs(parent_directory)
        if not os.path.exists(destination):
            print("Copying {} to {}.".format(source, destination))
            shutil.copy(source, destination, follow_symlinks=True)

class BinaryDistribution(Distribution):
    def has_ext_modules(self):
        return True

def find_version(*filepath):
    # Extract version information from filepath
    here = os.path.abspath(os.path.dirname(__file__))
    with open(os.path.join(here, *filepath)) as fp:
        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                                  fp.read(), re.M)
        if version_match:
            return version_match.group(1)
        raise RuntimeError("Unable to find version string.")

requires = [
    "numpy >= 1.16",
    "filelock",
    "jsonschema",
    "funcsigs",
    "click",
    "colorama",
    "packaging",
    "pytest",
    "pyyaml",
    "redis>=3.3.2",
    # NOTE: Don't upgrade the version of six! Doing so causes installation
    # problems. See https://github.com/ray-project/ray/issues/4169.
    "six >= 1.0.0",
    "faulthandler;python_version<'3.3'",
    "protobuf >= 3.8.0",
    "cloudpickle",
    "py-spy >= 0.2.0",
    "aiohttp",
    "google",
    "grpcio"
]

setup(
    name="ray",
    version=find_version("ray", "__init__.py"),
    author="Ray Team",
    author_email="ray-dev@googlegroups.com",
    description=("A system for parallel and distributed Python that unifies "
                 "the ML ecosystem."),
    long_description=open("../README.rst").read(),
    url="https://github.com/ray-project/ray",
    keywords=("ray distributed parallel machine-learning "
              "reinforcement-learning deep-learning python"),
    packages=find_packages(),
    cmdclass={"build_ext": build_ext},
    # The BinaryDistribution argument triggers build_ext.
    distclass=BinaryDistribution,
    install_requires=requires,
    setup_requires=["cython >= 0.29"],
    extras_require=extras,
    entry_points={
        "console_scripts": [
            "ray=ray.scripts.scripts:main",
            "rllib=ray.rllib.scripts:cli [rllib]", "tune=ray.tune.scripts:cli"
        ]
    },
    include_package_data=True,
    zip_safe=False,
    license="Apache 2.0")
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Created on Thu 6 March 2014

Contains testing routines for `SolarCoreModel.py`.

@author Kristoffer Braekken
"""
import SolarCoreModel

from numpy import log10

def opacity_test(tol=1.e-10):
    """
    Function for testing that the opacity is fetched correctly.
    """
    # Test values
    T = 10**(5.) # Feth 5.00 row
    rho = 1.e-6 # Fetch -5.0 column

    ans = log10(SolarCoreModel.kappa(T, rho))
    if abs(ans - (-0.068)) < tol:
        print 'Sucess.'
    else:
        print 'Fail.\n10**kappa =', ans, 'and not -0.068.'

if __name__ == '__main__':
    opacity_test()
<commit_msg>TODO: Fix test to take care of units.<commit_after>#!/usr/bin/env python
"""
Created on Thu 6 March 2014

Contains testing routines for `SolarCoreModel.py`.

@author Kristoffer Braekken
"""
import SolarCoreModel

from numpy import log10

def opacity_test(tol=1.e-10):
    """
    Function for testing that the opacity is fetched correctly.
    """
    # Test values
    T = 10**(5.) # Feth 5.00 row
    rho = 1.e-6 # Fetch -5.0 column
    rho /= 1.e3; rho *= 1./1e6 # Convert to SI units [kg m^-3]

    ans = log10(SolarCoreModel.kappa(T, rho))
    if abs(ans - (-0.068)) < tol:
        print 'Sucess.'
    else:
        print 'Fail.\n10**kappa =', ans, 'and not -0.068.'

if __name__ == '__main__':
    opacity_test()
<|endoftext|>


<commit_before>import os
import PidData.make_xml_configs as pidc
pjoin = os.path.join

# all of them should define this
t_config_file = "{state}_{plc}.{ext}"
t_data_file = "{plc}.{ext}"
# all of them should define this
t_product_file = "Fit_{state}_{plc}.{ext}"

def write_conf(  output_path, config_path ="./" ) :
	template = """
<?xml version="1.0" encoding="UTF-8"?>
<config>

	<jobType>SimultaneousTPid</jobType>

	<SimultaneousPid>
		<Logger color="true" logLevel="info" globalLogLevel="info"/>
		<Reporter> <output width="1600" height="1200" /> </Reporter>

		<input> <data url="{data_file}"/> </input>
		<output path="{output_path}"> <data>{product_file}</data> </output>

		<FitRange>
			<ptBins min="10" max="100" />
			<centralityBins>0, 1, 2, 3, 4, 5, 6</centralityBins>
			<charges>1, -1</charges>
		</FitRange>

		<Timing zdOnly="0.2" useZdEnhanced="0.2" useZbEnhanced="0.2" nSigZdEnhanced="3.0" nSigZbEnhanced="6.0" />

		<ParameterFixing>

			<deltaMu zb="2.0" zd="2.0" />

			<zb>
				<Pi min="0.6" max="1.0"/>
				<K min="0.6" max="1.0" />

			</zb>
			<zd>
				<Pi min="0.6" max="1.0"/>
				<K min="0.6" max="1.0" />

			</zd>

		</ParameterFixing>

		<Bichsel>

dedxBichsel.root

			<method>0</method>
		</Bichsel>

		<ZRecentering>
			<centerSpecies>{plc}</centerSpecies>
			<sigma tof="0.012" dedx="0.07"/>
			<method>nonlinear</method>
		</ZRecentering>

		<DataPurity nSigmaPi="3.0" nSigmaK="3.0" nSigmaE="3.0" nSigmaAboveP="3.0" />

		<Include url="../common/fitSchema_eff_1.xml" />

		<histograms>
			<Histo name="yield" title="yield" xBins="binning.pt" />
		</histograms>

	</SimultaneousPid>

	<Include url="../common/binning.xml" />

</config>

	"""

	states 		= ( "Corr", "PostCorr" )
	plcs 		= ( "Pi", "K", "P" )
	for state in states :
		for plc in plcs :
			data_file = pjoin( output_path, pidc.t_product_file.format( state=state, plc=plc, ext="root" ) )

			with open( pjoin( config_path, t_config_file.format( state=state, plc=plc, ext="xml" ) ), 'w' ) as f :
				f.write( template.format( plc=plc, data_file=data_file, output_path=output_path, product_file=t_product_file.format( state=state, plc=plc, ext="root" ) ) )

def write( output_path, config_path ="./" ) :

	if not os.path.exists(config_path):
		os.makedirs(config_path)

	write_conf( output_path, config_path )

# if __name__ == "__main__":
# 	parser = argparse.ArgumentParser( description="Creates the configs for TpcEff Tasks" );
# 	parser.add_argument( "data_path", help="Path to folder containg embedding data in rcpPicoDst format. Files should be named {plc}_{charge_str}_{mc,rc}.root" )
# 	parser.add_argument( "output_path", help="Path to folder to output products" )

# 	args = parser.parse_args()

# 	write_histo_conf( args.data_path, args.output_path )
# 	# fitter reads data from and produeces to the output path
# 	write_fit_config( args.output_path, args.output_path )<commit_msg>updated Fitter template<commit_after>import os
import PidData.make_xml_configs as pidc
pjoin = os.path.join

# all of them should define this
t_config_file = "{state}_{plc}.{ext}"
t_data_file = "{plc}.{ext}"
# all of them should define this
t_product_file = "Fit_{state}_{plc}.{ext}"

def write_conf(  output_path, config_path ="./" ) :
	template = """
<?xml version="1.0" encoding="UTF-8"?>
<config>

	<jobType>SimultaneousTPid</jobType>

	<SimultaneousPid>
		<Logger color="true" logLevel="info" globalLogLevel="info"/>
		<Reporter> <output width="1600" height="1200" /> </Reporter>

		<input> <data url="{data_file}"/> </input>
		<output path="{output_path}"> <data>{product_file}</data> </output>

		<FitRange>
			<ptBins min="10" max="100" />
			<centralityBins>0, 1, 2, 3, 4, 5, 6</centralityBins>
			<charges>1, -1</charges>
		</FitRange>

		<Timing zdOnly="0.2" useZdEnhanced="0.2" useZbEnhanced="0.2" nSigZdEnhanced="3.0" nSigZbEnhanced="6.0" />

		<ParameterFixing>

			<deltaMu zb="0.75" zd="0.75" />

			<zb>
				<Pi min="0.6" max="1.0"/>
				<K min="0.6" max="1.0" />

			</zb>
			<zd>
				<Pi min="0.6" max="1.0"/>
				<K min="0.6" max="1.0" />

			</zd>

		</ParameterFixing>

		<Bichsel>

dedxBichsel.root

			<method>0</method>
		</Bichsel>

		<ZRecentering>
			<centerSpecies>{plc}</centerSpecies>
			<sigma tof="0.012" dedx="0.07"/>
			<method>nonlinear</method>
		</ZRecentering>

		<DataPurity nSigmaPi="3.0" nSigmaK="3.0" nSigmaE="3.0" nSigmaAboveP="3.0" />

		<Include url="../common/fitSchema_eff_1.xml" />

		<histograms>
			<Histo name="yield" title="yield" xBins="binning.pt" />
		</histograms>

	</SimultaneousPid>

	<Include url="../common/binning.xml" />

</config>

	"""

	states 		= ( "Corr", "PostCorr" )
	plcs 		= ( "Pi", "K", "P" )
	for state in states :
		for plc in plcs :
			data_file = pjoin( output_path, pidc.t_product_file.format( state=state, plc=plc, ext="root" ) )

			with open( pjoin( config_path, t_config_file.format( state=state, plc=plc, ext="xml" ) ), 'w' ) as f :
				f.write( template.format( plc=plc, data_file=data_file, output_path=output_path, product_file=t_product_file.format( state=state, plc=plc, ext="root" ) ) )

def write( output_path, config_path ="./" ) :

	if not os.path.exists(config_path):
		os.makedirs(config_path)

	write_conf( output_path, config_path )

# if __name__ == "__main__":
# 	parser = argparse.ArgumentParser( description="Creates the configs for TpcEff Tasks" );
# 	parser.add_argument( "data_path", help="Path to folder containg embedding data in rcpPicoDst format. Files should be named {plc}_{charge_str}_{mc,rc}.root" )
# 	parser.add_argument( "output_path", help="Path to folder to output products" )

# 	args = parser.parse_args()

# 	write_histo_conf( args.data_path, args.output_path )
# 	# fitter reads data from and produeces to the output path
# 	write_fit_config( args.output_path, args.output_path )<|endoftext|>


<commit_before>#!/usr/bin/env python
import tables
import Queue
import multiprocessing
from operator import itemgetter
import time
import itertools

class DataProcessor(multiprocessing.Process):
    def __init__(self, sim_file_path, work_queue, result_queue):

        # job management stuff
        self.work_queue = work_queue
        self.result_queue = result_queue
        self.kill_received = False

        self.sim_file_path = sim_file_path

        super(DataProcessor, self).__init__()

    def run(self):
        self.sim_file = tables.open_file(self.sim_file_path, 'r')

        sweep_table = self.sim_file.get_node('/event_sweep_table')
        event_table = self.sim_file.get_node('/event_table')
        block_info_table = self.sim_file.get_node('/block_info_table')
        pt1getter = itemgetter(3,4,5)
        pt4getter = itemgetter(18,19,20)
        while not self.kill_received:

            # get a task
            try:
                event_range = self.work_queue.get_nowait()
            except Queue.Empty:
                break

            #sections = self.vc_sys.geometry.sectionsSortedByID(section_list)

            print 'processing events {} - {}'.format(*event_range)
            results = {}
            for evnum in range(*event_range):
                areas = {}
                total_slip = 0.0
                slip_records = 0
                surface_rupture_length = 0.0
                #print evnum, event_table[evnum]
                for sweep in sweep_table[event_table[evnum]['start_sweep_rec']:event_table[evnum]['end_sweep_rec']]:
                    eleid = sweep['block_id']
                    #trace_sum = block_info_table[eleid]['m_trace_flag_pt1'] + block_info_table[eleid]['m_trace_flag_pt2'] + block_info_table[eleid]['m_trace_flag_pt3'] + block_info_table[eleid]['m_trace_flag_pt4']

                    if block_info_table[eleid]['m_trace_flag_pt1'] > 0:
                        #pt1 = (block_info_table[eleid]['m_x_pt1'], block_info_table[eleid]['m_y_pt1'], block_info_table[eleid]['m_z_pt1'])
                        #pt4 = (block_info_table[eleid]['m_x_pt4'], block_info_table[eleid]['m_y_pt4'], block_info_table[eleid]['m_z_pt4'])
                        surface_rupture_length += (sum((x-y)**2.0 for x, y in itertools.izip(pt1getter(block_info_table[eleid]),pt4getter(block_info_table[eleid]))))**0.5

                    areas[eleid] = sweep['area']
                    total_slip += sweep['slip']
                    slip_records += 1
                #print areas.keys()
                results[evnum] = {'average_slip':total_slip/float(slip_records), 'area':sum(areas.values()), 'surface_rupture_length':surface_rupture_length, 'involved_elements':areas.keys()}

                #event_elements[i] = set(events.get_event_elements(i))

            # the actual processing
            #layered_sections = {}
            #print '    layering %i sections '%(len(sections))
            #for n, s in enumerate(sections):
            #    sid = s.sid
                #print sid
            #    curr_section = []
            #    for eid in s.selement_ids:
            #        if self.vc_sys.geometry.elements[eid].onTrace():
                        #print s.elementsAlongDip(eid)
            #            curr_section.append( s.elementsAlongDip(eid) )

            #    curr_section_np = np.array(curr_section)

            #    layered_sections[s.sid] = curr_section_np.T

            # store the result
            self.result_queue.put(results)

        self.sim_file.close()

class VCSimData(object):
    def __init__(self, file_path=None):
        self.file = None
        self.file_path = file_path
        self.do_event_area = False
        self.do_event_average_slip = False
        if self.file_path is not None:
            self.open_file(self.file_path)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        if self.file is not None:
            self.file.close()

    def open_file(self, file_path):
        if self.file_path is None:
            self.file_path = file_path
        self.file = tables.open_file(self.file_path)

        if 'event_area' not in self.file.root.event_table.colnames:
            self.do_event_area = True

        if 'event_average_slip' not in self.file.root.event_table.colnames:
            self.do_event_average_slip = True

        if self.do_event_area or self.do_event_average_slip:
            self.calculate_additional_data()

    def calculate_additional_data(self):
    # get the new data
        print 'getting new data'
        start_time = time.time()
        num_processes = multiprocessing.cpu_count()
        #num_processes = 1
        total_events = self.file.root.event_table.nrows
        #close the current file
        self.file.close()

        #break the work up
        seg = int(round(float(total_events)/float(num_processes)))
        work_queue = multiprocessing.Queue()
        for i in range(num_processes):
            if i == num_processes - 1:
                end_index = total_events
            else:
                end_index = seg*int(i + 1)
            work_queue.put((int(i) * seg , end_index))

        # create a queue to pass to workers to store the results
        result_queue = multiprocessing.Queue()

        # spawn workers
        for i in range(num_processes):
            worker = DataProcessor(self.file_path, work_queue, result_queue)
            worker.start()

        # collect the results off the queue
        results = {}
        for i in range(num_processes):
            #current_results = result_queue.get()
            #for k in current_results.keys():
            #    print k, current_results[k]
            results = dict(results, **result_queue.get())
            #results.append(result_queue.get())
        #print len(results)
        #data_process_results_sorted = []
        #for k in sorted(results.keys()):
        #    print k, results[k]
        data_process_results_sorted = [results[key] for key in sorted(results.keys())]

        print 'Done! {} seconds'.format(time.time() - start_time)
        #print data_process_results_sorted
        '''
    #create the new table
        self.file = tables.open_file(self.file_path, 'a')
        table = self.file.root.event_table

        # Get a description of table in dictionary format
        descr = table.description._v_colObjects
        descr2 = descr.copy()

        # Add a column to description
        if self.do_event_area:
            descr2['event_area'] = tables.Float64Col(dflt=0.0)

        if self.do_event_average_slip:
            descr2['event_average_slip'] = tables.Float64Col(dflt=0.0)

        # Create a new table with the new description
        table2 = self.file.create_table('/', 'tmp', descr2, 'Event Table')

        # Copy the user attributes
        table.attrs._f_copy(table2)

        # Fill the rows of new table with default values
        for i in xrange(table.nrows):
            table2.row.append()

        # Flush the rows to disk
        table2.flush()

        # Copy the columns of source table to destination
        for col in descr:
            getattr(table2.cols, col)[:] = getattr(table.cols, col)[:]

        # Fill the new column
        if self.do_event_area:
            table2.cols.event_area[:] = [ x[3] for x in data_process_results_sorted ]

        if self.do_event_average_slip:
            table2.cols.event_average_slip[:] = [ x[1]/float(x[2]) for x in data_process_results_sorted ]

        # Remove the original table
        table.remove()

        # Move table2 to table
        table2.move('/','event_table')

        # Print the new table
        #print "Contents of the table with column added:", self.file.root.event_table[:]

        # Finally, close the file
        self.file.close()

    #open the file with the new table
        self.file = tables.open_file(self.file_path)
        '''
        #self.file.root.event_table.col('event_num')
'''
# All access to the file goes through a single instance of this class.
# It contains several queues that are used to communicate with other
# processes.
# The read_queue is used for requests to read data from the HDF5 file.
# A list of result_queues is used to send data back to client processes.
# The write_queue is used for requests to modify the HDF5 file.
# One end of a pipe (shutdown) is used to signal the process to terminate.
class FileAccess(multiprocessing.Process):

    def __init__(self, h5_path, node_name, read_queue, result_queues, write_queue, shutdown):
        self.h5_path = h5_path
        self.node_name = node_name
        self.read_queue = read_queue
        self.result_queues = result_queues
        self.write_queue = write_queue
        self.shutdown = shutdown
        self.block_period = .01
        super(FileAccess, self).__init__()

    def run(self):
        self.h5_file = tables.open_file(self.h5_path, 'r+')
        self.table = self.h5_file.get_node('/{}'.format(self.node_name))
        another_loop = True
        while another_loop:

            # Check if the process has received the shutdown signal.
            if self.shutdown.poll():
                another_loop = False

            # Check for any data requests in the read_queue.
            try:
                row_num, proc_num = self.read_queue.get(True, self.block_period)
                # look up the appropriate result_queue for this data processor
                # instance
                result_queue = self.result_queues[proc_num]
                print 'processor {0} reading from row {1}'.format(proc_num, row_num)
                result_queue.put(self.read_data(row_num))
                another_loop = True
            except Queue.Empty:
                pass

            # Check for any write requests in the write_queue.
            try:
                row_num, data = self.write_queue.get(True, self.block_period)
                print 'writing row', row_num
                self.write_data(row_num, data)
                another_loop = True
            except Queue.Empty:
                pass

        # close the HDF5 file before shutting down
        self.h5_file.close()

    def read_data(self, row_num):
        return self.table[row_num, :]

    def write_data(self, row_num, data):
        self.table[row_num, :] = data

# This class represents a process that does work by reading and writing to the
# HDF5 file.  It does this by sending requests to the FileAccess class instance
# through its read and write queues.  The data results are sent back through
# the result_queue.
# Its actions are logged to a text file.
class DataProcessor(multiprocessing.Process):

    def __init__(self, read_queue, result_queue, write_queue, proc_num):
        self.read_queue = read_queue
        self.result_queue = result_queue
        self.write_queue = write_queue
        self.proc_num = proc_num
        #self.output_file = output_file
        super(DataProcessor, self).__init__()

    def run(self):
        #self.output_file = open(self.output_file, 'w')
        # read a random row from the file
        row_num = random.randint(0, self.array_size - 1)
        self.read_queue.put((row_num, self.proc_num))
        self.output_file.write(str(row_num) + '\n')
        self.output_file.write(str(self.result_queue.get()) + '\n')

        # modify a random row to equal 11 * (self.proc_num + 1)
        #row_num = random.randint(0, self.array_size - 1)
        #new_data = (numpy.zeros((1, self.array_size), 'i8') +
        #            11 * (self.proc_num + 1))
        #self.write_queue.put((row_num, new_data))

        # pause, then read the modified row
        #time.sleep(0.015)
        #self.read_queue.put((row_num, self.proc_num))
        #self.output_file.write(str(row_num) + '\n')
        #self.output_file.write(str(self.result_queue.get()) + '\n')
        #self.output_file.close()
'''
<commit_msg>more caching work<commit_after>#!/usr/bin/env python
import tables
import Queue
import multiprocessing
from operator import itemgetter
import time
import itertools

class DataProcessor(multiprocessing.Process):
    def __init__(self, sim_file_path, work_queue, result_queue):

        # job management stuff
        self.work_queue = work_queue
        self.result_queue = result_queue
        self.kill_received = False

        self.sim_file_path = sim_file_path

        super(DataProcessor, self).__init__()

    def run(self):
        self.sim_file = tables.open_file(self.sim_file_path, 'r')

        sweep_table = self.sim_file.get_node('/event_sweep_table')
        event_table = self.sim_file.get_node('/event_table')
        block_info_table = self.sim_file.get_node('/block_info_table')
        pt1getter = itemgetter(3,4,5)
        pt4getter = itemgetter(18,19,20)
        while not self.kill_received:

            # get a task
            try:
                event_range = self.work_queue.get_nowait()
            except Queue.Empty:
                break

            #sections = self.vc_sys.geometry.sectionsSortedByID(section_list)

            print 'processing events {} - {}'.format(*event_range)
            results = {}
            for evnum in range(*event_range):
                areas = {}
                total_slip = 0.0
                slip_records = 0
                surface_rupture_length = 0.0
                #print evnum, event_table[evnum]
                for sweep in sweep_table[event_table[evnum]['start_sweep_rec']:event_table[evnum]['end_sweep_rec']]:
                    eleid = sweep['block_id']
                    #trace_sum = block_info_table[eleid]['m_trace_flag_pt1'] + block_info_table[eleid]['m_trace_flag_pt2'] + block_info_table[eleid]['m_trace_flag_pt3'] + block_info_table[eleid]['m_trace_flag_pt4']

                    if block_info_table[eleid]['m_trace_flag_pt1'] > 0:
                        #pt1 = (block_info_table[eleid]['m_x_pt1'], block_info_table[eleid]['m_y_pt1'], block_info_table[eleid]['m_z_pt1'])
                        #pt4 = (block_info_table[eleid]['m_x_pt4'], block_info_table[eleid]['m_y_pt4'], block_info_table[eleid]['m_z_pt4'])
                        surface_rupture_length += (sum((x-y)**2.0 for x, y in itertools.izip(pt1getter(block_info_table[eleid]),pt4getter(block_info_table[eleid]))))**0.5

                    areas[eleid] = sweep['area']
                    total_slip += sweep['slip']
                    slip_records += 1
                #print areas.keys()
                results[evnum] = {'average_slip':total_slip/float(slip_records), 'area':sum(areas.values()), 'surface_rupture_length':surface_rupture_length, 'involved_elements':areas.keys()}

                #event_elements[i] = set(events.get_event_elements(i))

            # the actual processing
            #layered_sections = {}
            #print '    layering %i sections '%(len(sections))
            #for n, s in enumerate(sections):
            #    sid = s.sid
                #print sid
            #    curr_section = []
            #    for eid in s.selement_ids:
            #        if self.vc_sys.geometry.elements[eid].onTrace():
                        #print s.elementsAlongDip(eid)
            #            curr_section.append( s.elementsAlongDip(eid) )

            #    curr_section_np = np.array(curr_section)

            #    layered_sections[s.sid] = curr_section_np.T

            # store the result
            self.result_queue.put(results)

        self.sim_file.close()

class VCSimData(object):
    def __init__(self, file_path=None):
        self.file = None
        self.file_path = file_path

        self.do_event_area = False
        self.do_event_average_slip = False
        self.do_event_surface_rupture_length = False
        self.do_event_involved_elements = False

        if self.file_path is not None:
            self.open_file(self.file_path)

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        if self.file is not None:
            self.file.close()

    def open_file(self, file_path):
        if self.file_path is None:
            self.file_path = file_path
        self.file = tables.open_file(self.file_path)

        if 'event_area' not in self.file.root.event_table.colnames:
            self.do_event_area = True

        if 'event_average_slip' not in self.file.root.event_table.colnames:
            self.do_event_average_slip = True

        if 'event_surface_rupture_length' not in self.file.root.event_table.colnames:
            self.do_event_surface_rupture_length = True

        if 'event_involved_elements' not in self.file.root.event_table.colnames:
            self.do_event_involved_elements = True

        if self.do_event_area or self.do_event_average_slip or self.do_event_surface_rupture_length or self.do_event_involved_elements:
            self.calculate_additional_data()

    def calculate_additional_data(self):
    # get some info from the open file and then close it
        eleid_max_digits = len(str(self.file.root.block_info_table.nrows))
        total_events = self.file.root.event_table.nrows
        #close the current file
        self.file.close()

    # get the new data
        print 'getting new data'
        start_time = time.time()
        num_processes = multiprocessing.cpu_count()
        #num_processes = 1

        #break the work up
        seg = int(round(float(total_events)/float(num_processes)))
        work_queue = multiprocessing.Queue()
        for i in range(num_processes):
            if i == num_processes - 1:
                end_index = total_events
            else:
                end_index = seg*int(i + 1)
            work_queue.put((int(i) * seg , end_index))

        # create a queue to pass to workers to store the results
        result_queue = multiprocessing.Queue()

        # spawn workers
        for i in range(num_processes):
            worker = DataProcessor(self.file_path, work_queue, result_queue)
            worker.start()

        # collect the results off the queue
        results = {}
        for i in range(num_processes):
            #current_results = result_queue.get()
            #for k in current_results.keys():
            #    print k, current_results[k]
            results = dict(results, **result_queue.get())
            #results.append(result_queue.get())
        #print len(results)
        #data_process_results_sorted = []
        #for k in sorted(results.keys()):
        #    print k, results[k]
        self.max_involved_elements = []
        def find_max_involved_elements(dat, self, key):
            #print self.max_involved_elements
            if len(dat['involved_elements']) > len(self.max_involved_elements):
                print key
                self.max_involved_elements = dat['involved_elements']
            return dat
        data_process_results_sorted = [find_max_involved_elements(results[key], self, key) for key in sorted(results.keys())]

        max_str_len = len(','.join([str(x) for x in self.max_involved_elements]))

        print 'Done! {} seconds'.format(time.time() - start_time)
        #print data_process_results_sorted

    #create the new table
        self.file = tables.open_file(self.file_path, 'a')
        table = self.file.root.event_table

        # Get a description of table in dictionary format
        descr = table.description._v_colObjects
        descr2 = descr.copy()

        # Add a column to description
        if self.do_event_area:
            descr2['event_area'] = tables.Float64Col(dflt=0.0)

        if self.do_event_average_slip:
            descr2['event_average_slip'] = tables.Float64Col(dflt=0.0)

        if self.do_event_surface_rupture_length:
            descr2['event_surface_rupture_length'] = tables.Float64Col(dflt=0.0)

        if self.do_event_involved_elements:
            descr2['event_involved_elements'] = tables.StringCol(max_str_len)

        #print descr2
        # Create a new table with the new description
        table2 = self.file.create_table('/', 'tmp', descr2, 'Event Table')

        # Copy the user attributes
        table.attrs._f_copy(table2)

        # Fill the rows of new table with default values
        for i in xrange(table.nrows):
            table2.row.append()

        # Flush the rows to disk
        table2.flush()

        # Copy the columns of source table to destination
        for col in descr:
            getattr(table2.cols, col)[:] = getattr(table.cols, col)[:]

        # Fill the new column
        if self.do_event_area:
            table2.cols.event_area[:] = [ x['area'] for x in data_process_results_sorted ]

        if self.do_event_average_slip:
            table2.cols.event_average_slip[:] = [ x['average_slip'] for x in data_process_results_sorted ]

        if self.do_event_surface_rupture_length:
            table2.cols.event_surface_rupture_length[:] = [ x['surface_rupture_length'] for x in data_process_results_sorted ]

        if self.do_event_involved_elements:
            table2.cols.event_involved_elements[:] = [ ','.join([str(y) for y in x['involved_elements']]) for x in data_process_results_sorted ]

        # Remove the original table
        table.remove()

        # Move table2 to table
        table2.move('/','event_table')

        # Print the new table
        #print "Contents of the table with column added:", self.file.root.event_table[:]

        # Finally, close the file
        self.file.close()

    #open the file with the new table
        self.file = tables.open_file(self.file_path)

        #self.file.root.event_table.col('event_num')
'''
# All access to the file goes through a single instance of this class.
# It contains several queues that are used to communicate with other
# processes.
# The read_queue is used for requests to read data from the HDF5 file.
# A list of result_queues is used to send data back to client processes.
# The write_queue is used for requests to modify the HDF5 file.
# One end of a pipe (shutdown) is used to signal the process to terminate.
class FileAccess(multiprocessing.Process):

    def __init__(self, h5_path, node_name, read_queue, result_queues, write_queue, shutdown):
        self.h5_path = h5_path
        self.node_name = node_name
        self.read_queue = read_queue
        self.result_queues = result_queues
        self.write_queue = write_queue
        self.shutdown = shutdown
        self.block_period = .01
        super(FileAccess, self).__init__()

    def run(self):
        self.h5_file = tables.open_file(self.h5_path, 'r+')
        self.table = self.h5_file.get_node('/{}'.format(self.node_name))
        another_loop = True
        while another_loop:

            # Check if the process has received the shutdown signal.
            if self.shutdown.poll():
                another_loop = False

            # Check for any data requests in the read_queue.
            try:
                row_num, proc_num = self.read_queue.get(True, self.block_period)
                # look up the appropriate result_queue for this data processor
                # instance
                result_queue = self.result_queues[proc_num]
                print 'processor {0} reading from row {1}'.format(proc_num, row_num)
                result_queue.put(self.read_data(row_num))
                another_loop = True
            except Queue.Empty:
                pass

            # Check for any write requests in the write_queue.
            try:
                row_num, data = self.write_queue.get(True, self.block_period)
                print 'writing row', row_num
                self.write_data(row_num, data)
                another_loop = True
            except Queue.Empty:
                pass

        # close the HDF5 file before shutting down
        self.h5_file.close()

    def read_data(self, row_num):
        return self.table[row_num, :]

    def write_data(self, row_num, data):
        self.table[row_num, :] = data

# This class represents a process that does work by reading and writing to the
# HDF5 file.  It does this by sending requests to the FileAccess class instance
# through its read and write queues.  The data results are sent back through
# the result_queue.
# Its actions are logged to a text file.
class DataProcessor(multiprocessing.Process):

    def __init__(self, read_queue, result_queue, write_queue, proc_num):
        self.read_queue = read_queue
        self.result_queue = result_queue
        self.write_queue = write_queue
        self.proc_num = proc_num
        #self.output_file = output_file
        super(DataProcessor, self).__init__()

    def run(self):
        #self.output_file = open(self.output_file, 'w')
        # read a random row from the file
        row_num = random.randint(0, self.array_size - 1)
        self.read_queue.put((row_num, self.proc_num))
        self.output_file.write(str(row_num) + '\n')
        self.output_file.write(str(self.result_queue.get()) + '\n')

        # modify a random row to equal 11 * (self.proc_num + 1)
        #row_num = random.randint(0, self.array_size - 1)
        #new_data = (numpy.zeros((1, self.array_size), 'i8') +
        #            11 * (self.proc_num + 1))
        #self.write_queue.put((row_num, new_data))

        # pause, then read the modified row
        #time.sleep(0.015)
        #self.read_queue.put((row_num, self.proc_num))
        #self.output_file.write(str(row_num) + '\n')
        #self.output_file.write(str(self.result_queue.get()) + '\n')
        #self.output_file.close()
'''
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Roles in this namespace are meant to provide `Ruby on Rails <http://rubyonrails.org/>`_ applications utility methods for Debian distributions.
'''

from provy.core import Role
from provy.more.debian.package.aptitude import AptitudeRole
from provy.more.debian.package.gem import GemRole
from provy.more.debian.web.nginx import NginxRole

PACKAGES_TO_INSTALL = (
    'build-essential',
    'zlib1g-dev',
    'libssl-dev',
    'libpq-dev',
    'subversion',
    'libcurl4-openssl-dev',
    'libmysqlclient-dev',
    'libpcre3-dev',
    'libxslt1-dev',
    'libperl-dev',
    'libgcrypt11-dev',
    'libcrypto++-dev',
    'sqlite3',
    'libsqlite3-dev',
)

class RailsRole(Role):
    '''
    This role provides `Ruby on Rails <http://rubyonrails.org/>`_ application utilities for Debian distributions.

    Example:
    ::

        from provy.core import Role
        from provy.more.debian import RailsRole

        class MySampleRole(Role):
            def provision(self):
                with self.using(RailsRole) as role:
                    role.ensure_site_disabled('default')
                    role.create_site(site='my-site', path='/home/myuser/my-site')
                    role.ensure_site_enabled('my-site')
    '''

    def __available_site_for(self, name):
        return '/etc/nginx/sites-available/%s' % name

    def __enabled_site_for(self, name):
        return '/etc/nginx/sites-enabled/%s' % name

    def provision(self):
        '''
        Installs `Ruby on Rails <http://rubyonrails.org/>`_ dependencies.
        This method should be called upon if overriden in base classes, or Rails won't work properly in the remote server.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    self.provision_role(RailsRole) # does not need to be called if using with block.
        '''
        self.register_template_loader('provy.more.debian.web')

        self.__install_system_packages()
        self.__install_gem_packages()
        self.__install_nginx_module()
        self.__create_nginx_configurations()
        self.__create_nginx_directories()

    def __create_nginx_directories(self):
        self.ensure_dir('/etc/nginx/sites-available', sudo=True)
        self.ensure_dir('/etc/nginx/sites-enabled', sudo=True)
        self.ensure_dir('/etc/nginx/conf.d', sudo=True)

    def __create_nginx_configurations(self):
        self.update_file('rails.nginx.conf.template', '/etc/nginx/conf/nginx.conf', sudo=True)
        self.update_file('rails.nginx.init.template', '/etc/init.d/nginx', sudo=True)
        self.change_file_mode('/etc/init.d/nginx', 755)

    def __install_gem_packages(self):
        with self.using(GemRole) as role:
            role.ensure_package_installed('bundler')
            role.ensure_package_installed('passenger')

    def __install_system_packages(self):
        with self.using(AptitudeRole) as role:
            for package in PACKAGES_TO_INSTALL:
                role.ensure_package_installed(package)

    def __install_nginx_module(self):
        if not self.remote_exists_dir('/etc/nginx'):
            self.ensure_dir('/var/log/nginx', sudo=True)
            self.log("passenger-nginx not found! Installing...")
            self.execute('passenger-install-nginx-module --auto --auto-download --prefix=/etc/nginx', sudo=True, stdout=False)
            self.log("passenger-nginx installed.")

    def cleanup(self):
        '''
        Restarts nginx if any changes have been made.

        There's no need to call this method manually.
        '''
        super(RailsRole, self).cleanup()
        if 'must-restart-nginx' in self.context and self.context['must-restart-nginx']:
            self.restart()

    def ensure_site_disabled(self, site):
        '''
        Ensures that the specified site is removed from Nginx list of enabled sites.

        :param site: Name of the site to disable.
        :type site: :class:`str`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_site_disabled('default')
        '''
        with self.using(NginxRole) as nginx:
            nginx.ensure_site_disabled(site)

    def ensure_site_enabled(self, site):
        '''
        Ensures that a symlink is created for the specified site at Nginx list of enabled sites from the list of available sites.

        :param site: Name of the site to enable.
        :type site: :class:`str`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_site_enabled('my-site')
        '''
        with self.using(NginxRole) as nginx:
            nginx.ensure_site_enabled(site)

    def create_site(self, site, host, path, port=80, options={}):
        '''
        Adds a website with the specified template to Nginx list of available sites.

        .. warning::

            Do not forget to call :meth:`ensure_site_enabled` after a call to create_site, or your site won't be enabled.

        :param site: Name of the site to enable.
        :type site: :class:`str`
        :param host: Server domain that Nginx should respond by.
        :type host: :class:`str`
        :param path: Path of the rails app.
        :type path: :class:`str`
        :param port: Port that Nginx will listen in. Defaults to `80`.
        :type port: :class:`int`
        :param options: Options to pass to the Nginx template. Defaults to empty dict (`{}`).
        :type options: :class:`int`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.create_site(site='my-site', host='localhost www.mysite.com',
                                         port=8888, path='/home/myuser/my-rails-site')
        '''

        template = "rails-nginx.template"
        options['host'] = host
        options['path'] = path
        result = self.update_file(template,
                                  self.__available_site_for(site),
                                  options=options, sudo=True)

        self.execute('cd %s && bundle install --without development test --deployment' % path, user=self.context['owner'], stdout=True)
        if result:
            self.log('Installing gems with bundler')
            self.log('%s nginx site created!' % site)
            self.ensure_restart()

    def ensure_restart(self):
        '''
        Ensures that Nginx gets restarted on cleanup. There's no need to call this method as any changes to Nginx will trigger it.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_restart()
        '''
        self.context['must-restart-nginx'] = True

    def restart(self):
        '''
        Forcefully restarts nginx.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.restart()
        '''
        with self.using(NginxRole) as nginx:
            nginx.restart()
<commit_msg>Removing useless method from RailsRole<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Roles in this namespace are meant to provide `Ruby on Rails <http://rubyonrails.org/>`_ applications utility methods for Debian distributions.
'''

from provy.core import Role
from provy.more.debian.package.aptitude import AptitudeRole
from provy.more.debian.package.gem import GemRole
from provy.more.debian.web.nginx import NginxRole

PACKAGES_TO_INSTALL = (
    'build-essential',
    'zlib1g-dev',
    'libssl-dev',
    'libpq-dev',
    'subversion',
    'libcurl4-openssl-dev',
    'libmysqlclient-dev',
    'libpcre3-dev',
    'libxslt1-dev',
    'libperl-dev',
    'libgcrypt11-dev',
    'libcrypto++-dev',
    'sqlite3',
    'libsqlite3-dev',
)

class RailsRole(Role):
    '''
    This role provides `Ruby on Rails <http://rubyonrails.org/>`_ application utilities for Debian distributions.

    Example:
    ::

        from provy.core import Role
        from provy.more.debian import RailsRole

        class MySampleRole(Role):
            def provision(self):
                with self.using(RailsRole) as role:
                    role.ensure_site_disabled('default')
                    role.create_site(site='my-site', path='/home/myuser/my-site')
                    role.ensure_site_enabled('my-site')
    '''

    def __available_site_for(self, name):
        return '/etc/nginx/sites-available/%s' % name

    def provision(self):
        '''
        Installs `Ruby on Rails <http://rubyonrails.org/>`_ dependencies.
        This method should be called upon if overriden in base classes, or Rails won't work properly in the remote server.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    self.provision_role(RailsRole) # does not need to be called if using with block.
        '''
        self.register_template_loader('provy.more.debian.web')

        self.__install_system_packages()
        self.__install_gem_packages()
        self.__install_nginx_module()
        self.__create_nginx_configurations()
        self.__create_nginx_directories()

    def __create_nginx_directories(self):
        self.ensure_dir('/etc/nginx/sites-available', sudo=True)
        self.ensure_dir('/etc/nginx/sites-enabled', sudo=True)
        self.ensure_dir('/etc/nginx/conf.d', sudo=True)

    def __create_nginx_configurations(self):
        self.update_file('rails.nginx.conf.template', '/etc/nginx/conf/nginx.conf', sudo=True)
        self.update_file('rails.nginx.init.template', '/etc/init.d/nginx', sudo=True)
        self.change_file_mode('/etc/init.d/nginx', 755)

    def __install_gem_packages(self):
        with self.using(GemRole) as role:
            role.ensure_package_installed('bundler')
            role.ensure_package_installed('passenger')

    def __install_system_packages(self):
        with self.using(AptitudeRole) as role:
            for package in PACKAGES_TO_INSTALL:
                role.ensure_package_installed(package)

    def __install_nginx_module(self):
        if not self.remote_exists_dir('/etc/nginx'):
            self.ensure_dir('/var/log/nginx', sudo=True)
            self.log("passenger-nginx not found! Installing...")
            self.execute('passenger-install-nginx-module --auto --auto-download --prefix=/etc/nginx', sudo=True, stdout=False)
            self.log("passenger-nginx installed.")

    def cleanup(self):
        '''
        Restarts nginx if any changes have been made.

        There's no need to call this method manually.
        '''
        super(RailsRole, self).cleanup()
        if 'must-restart-nginx' in self.context and self.context['must-restart-nginx']:
            self.restart()

    def ensure_site_disabled(self, site):
        '''
        Ensures that the specified site is removed from Nginx list of enabled sites.

        :param site: Name of the site to disable.
        :type site: :class:`str`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_site_disabled('default')
        '''
        with self.using(NginxRole) as nginx:
            nginx.ensure_site_disabled(site)

    def ensure_site_enabled(self, site):
        '''
        Ensures that a symlink is created for the specified site at Nginx list of enabled sites from the list of available sites.

        :param site: Name of the site to enable.
        :type site: :class:`str`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_site_enabled('my-site')
        '''
        with self.using(NginxRole) as nginx:
            nginx.ensure_site_enabled(site)

    def create_site(self, site, host, path, port=80, options={}):
        '''
        Adds a website with the specified template to Nginx list of available sites.

        .. warning::

            Do not forget to call :meth:`ensure_site_enabled` after a call to create_site, or your site won't be enabled.

        :param site: Name of the site to enable.
        :type site: :class:`str`
        :param host: Server domain that Nginx should respond by.
        :type host: :class:`str`
        :param path: Path of the rails app.
        :type path: :class:`str`
        :param port: Port that Nginx will listen in. Defaults to `80`.
        :type port: :class:`int`
        :param options: Options to pass to the Nginx template. Defaults to empty dict (`{}`).
        :type options: :class:`int`

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.create_site(site='my-site', host='localhost www.mysite.com',
                                         port=8888, path='/home/myuser/my-rails-site')
        '''

        template = "rails-nginx.template"
        options['host'] = host
        options['path'] = path
        result = self.update_file(template,
                                  self.__available_site_for(site),
                                  options=options, sudo=True)

        self.execute('cd %s && bundle install --without development test --deployment' % path, user=self.context['owner'], stdout=True)
        if result:
            self.log('Installing gems with bundler')
            self.log('%s nginx site created!' % site)
            self.ensure_restart()

    def ensure_restart(self):
        '''
        Ensures that Nginx gets restarted on cleanup. There's no need to call this method as any changes to Nginx will trigger it.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.ensure_restart()
        '''
        self.context['must-restart-nginx'] = True

    def restart(self):
        '''
        Forcefully restarts nginx.

        Example:
        ::

            from provy.core import Role
            from provy.more.debian import RailsRole

            class MySampleRole(Role):
                def provision(self):
                    with self.using(RailsRole) as role:
                        role.restart()
        '''
        with self.using(NginxRole) as nginx:
            nginx.restart()
<|endoftext|>