<commit_before>from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import os

try:
    import ctypes
    from ctypes import *

    from compas.interop.cpp.xdarray import Array1D
    from compas.interop.cpp.xdarray import Array2D

except (ImportError, SystemError):
    compas.raise_if_not_ironpython()

HERE = os.path.dirname(__file__)

SO = os.path.join(HERE, '_smoothing_cpp', 'smoothing.so')
DLL = os.path.join(HERE, '_smoothing_cpp', 'smoothing.dll')

__author__     = ['Tom Van Mele <vanmelet@ethz.ch>']
__copyright__  = 'Copyright 2017, Block Research Group - ETH Zurich'
__license__    = 'MIT License'
__email__      = 'vanmelet@ethz.ch'

__all__ = [
    'smooth_centroid_cpp',
]

def smooth_centroid_cpp(vertices, adjacency, fixed, kmax=100, callback=None, callback_args=None):
    """
    """
    try:
        smoothing = ctypes.cdll.LoadLibrary(SO)
    except Exception:
        try:
            smoothing = ctypes.cdll.LoadLibrary(DLL)
        except Exception:
            print(SO)
            print(DLL)
            raise
    # --------------------------------------------------------------------------
    # callback
    # --------------------------------------------------------------------------
    if callback:
        assert callable(callback), 'The provided callback is not callable.'
    else:
        def callback(k, xyz):
            pass

    v = len(vertices)
    nbrs = [len(adjacency[i]) for i in range(v)]
    neighbours = [adjacency[i] + [0] * (10 - nbrs[i]) for i in range(v)]
    # --------------------------------------------------------------------------
    # call
    # --------------------------------------------------------------------------
    c_nbrs = Array1D(nbrs, 'int')
    c_fixed = Array1D(fixed, 'int')
    c_vertices = Array2D(vertices, 'double')
    c_neighbours = Array2D(neighbours, 'int')
    c_callback = CFUNCTYPE(None, c_int)

    def wrapper(k):
        callback(k, c_vertices.cdata)

    smoothing.smooth_centroid.argtypes = [
        c_int,
        c_nbrs.ctype,
        c_fixed.ctype,
        c_vertices.ctype,
        c_neighbours.ctype,
        c_int,
        c_callback
    ]

    smoothing.smooth_centroid(
        c_int(v),
        c_nbrs.cdata,
        c_fixed.cdata,
        c_vertices.cdata,
        c_neighbours.cdata,
        c_int(kmax),
        c_callback(wrapper)
    )

    return c_vertices.pydata

# ==============================================================================
# Main
# ==============================================================================

if __name__ == "__main__":

    import compas
    from compas.datastructures import Mesh
    from compas.plotters import MeshPlotter

    kmax = 50

    # make a mesh
    # and set the default vertex and edge attributes

    mesh = Mesh.from_obj(compas.get('faces.obj'))

    # extract numerical data from the datastructure

    vertices  = mesh.get_vertices_attributes(('x', 'y', 'z'))
    adjacency = [mesh.vertex_neighbours(key) for key in mesh.vertices()]
    fixed     = [int(mesh.vertex_degree(key) == 2) for key in mesh.vertices()]

    slider = list(mesh.vertices_where({'x': (-0.1, 0.1), 'y': (9.9, 10.1)}))[0]

    # make a plotter for (dynamic) visualization
    # and define a callback function
    # for plotting the intermediate configurations

    plotter = MeshPlotter(mesh, figsize=(10, 7))

    def callback(k, xyz):
        print(k)

        if k < kmax - 1:
            xyz[slider][0] = 0.1 * (k + 1)

        plotter.update_vertices()
        plotter.update_edges()
        plotter.update(pause=0.001)

        for key, attr in mesh.vertices(True):
            attr['x'] = xyz[key][0]
            attr['y'] = xyz[key][1]
            attr['z'] = xyz[key][2]

    # plot the lines of the original configuration of the mesh
    # as a reference

    lines = []
    for u, v in mesh.edges():
        lines.append({
            'start': mesh.vertex_coordinates(u, 'xy'),
            'end'  : mesh.vertex_coordinates(v, 'xy'),
            'color': '#cccccc',
            'width': 0.5
        })

    plotter.draw_lines(lines)

    # draw the vertices and edges in the starting configuration
    # and pause for a second before starting the dynamic visualization

    plotter.draw_vertices(facecolor={key: '#000000' for key in mesh.vertices() if mesh.vertex_degree(key) == 2})
    plotter.draw_edges()

    plotter.update(pause=0.5)

    # run the smoother

    xyz = smooth_centroid_cpp(vertices, adjacency, fixed, kmax=kmax, callback=callback)

    # keep the plot alive

    plotter.show()
<commit_msg>compas_rhino from mac import error fixed<commit_after>from __future__ import print_function
from __future__ import absolute_import
from __future__ import division

import compas
import os

try:
    import ctypes
    from ctypes import *

    from compas.interop.cpp.xdarray import Array1D
    from compas.interop.cpp.xdarray import Array2D

except (ImportError, SystemError):
    compas.raise_if_not_ironpython()

HERE = os.path.dirname(__file__)

SO = os.path.join(HERE, '_smoothing_cpp', 'smoothing.so')
DLL = os.path.join(HERE, '_smoothing_cpp', 'smoothing.dll')

__author__     = ['Tom Van Mele <vanmelet@ethz.ch>']
__copyright__  = 'Copyright 2017, Block Research Group - ETH Zurich'
__license__    = 'MIT License'
__email__      = 'vanmelet@ethz.ch'

__all__ = [
    'smooth_centroid_cpp',
]

def smooth_centroid_cpp(vertices, adjacency, fixed, kmax=100, callback=None, callback_args=None):
    """
    """
    try:
        smoothing = ctypes.cdll.LoadLibrary(SO)
    except Exception:
        try:
            smoothing = ctypes.cdll.LoadLibrary(DLL)
        except Exception:
            print(SO)
            print(DLL)
            raise
    # --------------------------------------------------------------------------
    # callback
    # --------------------------------------------------------------------------
    if callback:
        assert callable(callback), 'The provided callback is not callable.'
    else:
        def callback(k, xyz):
            pass

    v = len(vertices)
    nbrs = [len(adjacency[i]) for i in range(v)]
    neighbours = [adjacency[i] + [0] * (10 - nbrs[i]) for i in range(v)]
    # --------------------------------------------------------------------------
    # call
    # --------------------------------------------------------------------------
    c_nbrs = Array1D(nbrs, 'int')
    c_fixed = Array1D(fixed, 'int')
    c_vertices = Array2D(vertices, 'double')
    c_neighbours = Array2D(neighbours, 'int')
    c_callback = CFUNCTYPE(None, c_int)

    def wrapper(k):
        callback(k, c_vertices.cdata)

    smoothing.smooth_centroid.argtypes = [
        c_int,
        c_nbrs.ctype,
        c_fixed.ctype,
        c_vertices.ctype,
        c_neighbours.ctype,
        c_int,
        c_callback
    ]

    smoothing.smooth_centroid(
        c_int(v),
        c_nbrs.cdata,
        c_fixed.cdata,
        c_vertices.cdata,
        c_neighbours.cdata,
        c_int(kmax),
        c_callback(wrapper)
    )

    return c_vertices.pydata

# ==============================================================================
# Main
# ==============================================================================

if __name__ == "__main__":

    import compas
    from compas.datastructures import Mesh
    from compas.plotters import MeshPlotter

    kmax = 50

    # make a mesh
    # and set the default vertex and edge attributes

    mesh = Mesh.from_obj(compas.get('faces.obj'))

    # extract numerical data from the datastructure

    vertices  = mesh.get_vertices_attributes(('x', 'y', 'z'))
    adjacency = [mesh.vertex_neighbours(key) for key in mesh.vertices()]
    fixed     = [int(mesh.vertex_degree(key) == 2) for key in mesh.vertices()]

    slider = list(mesh.vertices_where({'x': (-0.1, 0.1), 'y': (9.9, 10.1)}))[0]

    # make a plotter for (dynamic) visualization
    # and define a callback function
    # for plotting the intermediate configurations

    plotter = MeshPlotter(mesh, figsize=(10, 7))

    def callback(k, xyz):
        print(k)

        if k < kmax - 1:
            xyz[slider][0] = 0.1 * (k + 1)

        plotter.update_vertices()
        plotter.update_edges()
        plotter.update(pause=0.001)

        for key, attr in mesh.vertices(True):
            attr['x'] = xyz[key][0]
            attr['y'] = xyz[key][1]
            attr['z'] = xyz[key][2]

    # plot the lines of the original configuration of the mesh
    # as a reference

    lines = []
    for u, v in mesh.edges():
        lines.append({
            'start': mesh.vertex_coordinates(u, 'xy'),
            'end'  : mesh.vertex_coordinates(v, 'xy'),
            'color': '#cccccc',
            'width': 0.5
        })

    plotter.draw_lines(lines)

    # draw the vertices and edges in the starting configuration
    # and pause for a second before starting the dynamic visualization

    plotter.draw_vertices(facecolor={key: '#000000' for key in mesh.vertices() if mesh.vertex_degree(key) == 2})
    plotter.draw_edges()

    plotter.update(pause=0.5)

    # run the smoother

    xyz = smooth_centroid_cpp(vertices, adjacency, fixed, kmax=kmax, callback=callback)

    # keep the plot alive

    plotter.show()
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Octopus(Package):
    """A real-space finite-difference (time-dependent) density-functional
    theory code."""

    homepage = "http://www.tddft.org/programs/octopus/"
    url      = "http://www.tddft.org/programs/octopus/down.php?file=5.0.1/octopus-5.0.1.tar.gz"

    version('5.0.1', '2b6392ab67b843f9d4ca7413fc07e822')

    depends_on('blas')
    depends_on('gsl')
    depends_on('lapack')
    depends_on('libxc')
    depends_on('mpi')
    depends_on('fftw+mpi')

    # optional dependencies:
    # TODO: scalapack, metis, parmetis, netcdf, etsf-io, sparskit, arpack,
    # feast, libfm, pfft, isf, pnfft

    def install(self, spec, prefix):
        lapack = spec['lapack'].lapack_libs
        blas = spec['blas'].blas_libs
        args = []
        args.extend([
            '--prefix=%s' % prefix,
            '--with-blas=%s' % blas.ld_flags,
            '--with-lapack=%s' % lapack.ld_flags,
            '--with-gsl-prefix=%s' % spec['gsl'].prefix,
            '--with-libxc-prefix=%s' % spec['libxc'].prefix,
            'CC=%s' % spec['mpi'].mpicc,
            'FC=%s' % spec['mpi'].mpifc,
            '--enable-mpi',
            '--with-fft-lib=-L%s -lfftw3' % spec['fftw'].prefix.lib
            # --with-blacs=${prefix}/lib/libscalapack.dylib
            # --with-netcdf-prefix=netcdf-fortran
            # --with-etsf-io-prefix=
            # --with-sparskit=${prefix}/lib/libskit.a
            # --with-pfft-prefix=${prefix} --with-mpifftw-prefix=${prefix}
            # --with-arpack=${prefix}/lib/libarpack.dylib
            # --with-parpack=${prefix}/lib/libparpack.dylib
            # --with-metis-prefix=${prefix} --with-parmetis-prefix=${prefix}
            # --with-berkeleygw-prefix=${prefix}
        ])

        # When preprocessor expands macros (i.e. CFLAGS) defined as quoted
        # strings the result may be > 132 chars and is terminated.
        # This will look to a compiler as an Unterminated character constant
        # and produce Line truncated errors. To vercome this, add flags to
        # let compiler know that the entire line is meaningful.
        # TODO: For the lack of better approach, assume that clang is mixed
        # with GNU fortran.
        if spec.satisfies('%clang') or spec.satisfies('%gcc'):
            args.extend([
                'FCFLAGS=-O2 -ffree-line-length-none'
            ])

        configure(*args)
        make()
        # short tests take forever...
        # make('check-short')
        make('install')
<commit_msg>Features/octopus 6.0 (#2819)<commit_after>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Octopus(Package):
    """A real-space finite-difference (time-dependent) density-functional
    theory code."""

    homepage = "http://www.tddft.org/programs/octopus/"
    base_url = "http://www.tddft.org/programs/octopus/down.php?file="

    version('6.0', '5d1168c2a8d7fd9cb9492eaebaa7182e')
    version('5.0.1', '2b6392ab67b843f9d4ca7413fc07e822')

    # Sample url is:
    # "http://www.tddft.org/programs/octopus/down.php?file=5.0.1/octopus-5.0.1.tar.gz"
    def url_for_version(self, version):
        return '{0}/{1}/octopus-{1}.tar.gz'.format(Octopus.base_url,
                                                   version.dotted)

    variant('scalapack', default=False,
            description='Compile with Scalapack')
    variant('metis', default=True,
            description='Compile with METIS')
    variant('parmetis', default=False,
            description='Compile with ParMETIS')
    variant('netcdf', default=False,
            description='Compile with Netcdf')
    variant('arpack-ng', default=False,
            description='Compile with ARPACK-ng')

    depends_on('blas')
    depends_on('gsl')
    depends_on('lapack')
    depends_on('libxc')
    depends_on('mpi')
    depends_on('fftw+mpi')
    depends_on('metis@5:', when='+metis')
    depends_on('parmetis', when='+parmetis')
    depends_on('scalapack', when='+scalapack')
    depends_on('netcdf-fortran', when='+netcdf')
    depends_on('arpack-ng', when='+arpack-ng')

    # optional dependencies:
    # TODO: parmetis, etsf-io, sparskit,
    # feast, libfm, pfft, isf, pnfft

    def install(self, spec, prefix):
        arpack = find_libraries(['libarpack'], root=spec[
                                'arpack-ng'].prefix.lib, shared=True)
        lapack = spec['lapack'].lapack_libs
        blas = spec['blas'].blas_libs
        args = []
        args.extend([
            '--prefix=%s' % prefix,
            '--with-blas=%s' % blas.ld_flags,
            '--with-lapack=%s' % lapack.ld_flags,
            '--with-gsl-prefix=%s' % spec['gsl'].prefix,
            '--with-libxc-prefix=%s' % spec['libxc'].prefix,
            'CC=%s' % spec['mpi'].mpicc,
            'FC=%s' % spec['mpi'].mpifc,
            '--enable-mpi',
            '--with-fft-lib=-L%s -lfftw3' % spec['fftw'].prefix.lib,
        ])
        if '+metis' in spec:
            args.extend([
                '--with-metis-prefix=%s' % spec['metis'].prefix,
            ])
        if '+parmetis' in spec:
            args.extend([
                '--with-parmetis-prefix=%s' % spec['parmetis'].prefix,
            ])
        if '+netcdf' in spec:
            args.extend([
                '--with-netcdf-prefix=%s' % spec['netcdf-fortran'].prefix,
                '--with-netcdf-include=%s' %
                spec['netcdf-fortran'].prefix.include,
            ])
        if '+arpack-ng' in spec:
            args.extend([
                '--with-arpack={0}'.format(arpack.joined()),
            ])
        if '+scalapack' in spec:
            args.extend([
                '--with-blacs=%s' % spec['scalapack'].scalapack_libs,
                '--with-scalapack=%s' % spec['scalapack'].scalapack_libs,
            ])
            # --with-etsf-io-prefix=
            # --with-sparskit=${prefix}/lib/libskit.a
            # --with-pfft-prefix=${prefix} --with-mpifftw-prefix=${prefix}
            # --with-parpack=${prefix}/lib/libparpack.dylib
            # --with-parmetis-prefix=${prefix}
            # --with-berkeleygw-prefix=${prefix}

        # When preprocessor expands macros (i.e. CFLAGS) defined as quoted
        # strings the result may be > 132 chars and is terminated.
        # This will look to a compiler as an Unterminated character constant
        # and produce Line truncated errors. To vercome this, add flags to
        # let compiler know that the entire line is meaningful.
        # TODO: For the lack of better approach, assume that clang is mixed
        # with GNU fortran.
        if spec.satisfies('%clang') or spec.satisfies('%gcc'):
            args.extend([
                'FCFLAGS=-O2 -ffree-line-length-none'
            ])

        configure(*args)
        make()
        # short tests take forever...
        # make('check-short')
        make('install')
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
HTTP Client
===========

The client library the manager uses to communicate with
the master server.
"""

import json
from collections import namedtuple
from functools import partial

import treq
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol, connectionDone
from twisted.python import log
from twisted.web.client import Response as _Response, GzipDecoder, ResponseDone

from pyfarm.core.enums import STRING_TYPES, NOTSET
from pyfarm.core.logger import getLogger

Request = namedtuple(
    "Request",
    ["method", "url", "headers", "data"])

logger = getLogger("agent.http")

class Response(Protocol):
    """
    This class receives the incoming response body from a request
    constructs some convenience methods and attributes around the data.

    :param deferred:
        The deferred object which contains the target callback
        and errback.

    :param response:
        The initial response object which will be passed along
        to the target deferred.

    :param request:
        Named tuple object containing the method name, url, headers, and data.
    """
    def __init__(self, deferred, response, request):
        assert isinstance(deferred, Deferred)
        assert isinstance(response, (_Response, GzipDecoder))
        assert isinstance(request, Request)

        # internal attributes
        self._done = False
        self._body = ""
        self._deferred = deferred

        # main public attributes
        self.request = request
        self.response = response

        # convenience attributes constructed
        # from the public attributes
        self.url = self.request.url
        self.code = self.response.code
        self.content_type = None

        # consume the response headers
        self.headers = {}
        for header_key, header_value in response.headers.getAllRawHeaders():
            if len(header_value) == 1:
                header_value = header_value[0]
            self.headers[header_key] = header_value

        # determine the content type
        if "Content-Type" in self.headers:
            self.content_type = self.headers["Content-Type"]

    def data(self):
        """
        Returns the data currently contained in the buffer.

        :raises RuntimeError:
            Raised if this method id called before all data
            has been received.
        """
        if not self._done:
            raise RuntimeError("Response not yet received.")
        return self._body

    def json(self, loader=json.loads):
        """
        Returns the json data from the incoming request

        :raises RuntimeError:
            Raised if this method id called before all data
            has been received.

        :raises ValueError:
            Raised if the content type for this request is not
            application/json.
        """
        if not self._done:
            raise RuntimeError("Response not yet received.")
        elif self.content_type != "application/json":
            raise ValueError("Not an application/json response.")
        else:
            return loader(self._body)

    def dataReceived(self, data):
        """
        Overrides :meth:`.Protocol.dataReceived` and appends data
        to ``_body``.
        """
        self._body += data

    def connectionLost(self, reason=connectionDone):
        """
        Overrides :meth:`.Protocol.connectionLost` and sets the ``_done``
        when complete.  When called with :class:`.ResponseDone` for ``reason``
        this method will call the callback on ``_deferred``
        """
        if reason.type is ResponseDone:
            self._done = True
            logger.debug(
                "%s %s (code: %s: body: %r)",
                self.request.method, self.request.url, self.code, self._body)
            self._deferred.callback(self)
        else:
            self._deferred.errback(reason)

def request(method, url, **kwargs):
    """
    Wrapper around :func:`treq.request` with some added arguments
    and validation.

    :param str method:
        The HTTP method to use when making the request.

    :param str url:
        The url this request will be made to.

    :type data: str, list, tuple, set, dict
    :keyword data:
        The data to send along with some types of requests
        such as ``POST`` or ``PUT``

    :keyword dict headers:
        The headers to send along with the request to
        ``url``.  Currently only single values per header
        are supported.

    :keyword function callback:
        The function to deliver an instance of :class:`Response`
        once we receive and unpack a response.

    :keyword function errback:
        The function to deliver an error message to.  By default
        this will use :func:`.log.err`.

    :keyword class response_class:
        The class to use to unpack the internal response.  This is mainly
        used by the unittests but could be used elsewhere to add some
        custom behavior to the unpack process for the incoming response.
    """
    # check assumptions for arguments
    assert method in ("HEAD", "GET", "POST", "PUT", "PATCH", "DELETE")
    assert isinstance(url, STRING_TYPES) and url

    data = kwargs.pop("data", NOTSET)
    headers = kwargs.pop("headers", {})
    callback = kwargs.pop("callback", None)
    errback = kwargs.pop("errback", log.err)
    response_class = kwargs.pop("response_class", Response)

    # check assumptions for keywords
    assert callable(callback) and callable(errback)
    assert data is NOTSET or \
           isinstance(data, tuple(list(STRING_TYPES) + [dict, list]))

    # add our default headers
    headers.setdefault("Content-Type", ["application/json"])
    headers.setdefault("User-Agent", ["PyFarm (agent) 1.0"])

    # ensure all values in the headers are lists (needed by Twisted)
    for header, value in headers.items():
        if isinstance(value, STRING_TYPES):
            headers[header] = [value]

        # for our purposes we should not expect headers with more
        # than one value for now
        elif isinstance(value, (list, tuple, set)):
            assert len(value) == 1

        else:
            raise NotImplementedError(
                "cannot handle header values with type %s" % type(value))

    def unpack_response(response):
        deferred = Deferred()
        deferred.addCallback(callback)

        # Deliver the body onto an instance of the response
        # object along with the original request.  Finally
        # the request and response via an instance of `Response`
        # to the outer scope's callback function.
        response.deliverBody(
            response_class(
                deferred, response,
                Request(method=method, url=url, data=data, headers=headers)))

        return deferred

    # prepare to send the data
    request_data = data  # so we don't modify the original data
    if request_data is not NOTSET and \
                    headers["Content-Type"] == ["application/json"]:
        request_data = json.dumps(data)

    elif data is not NOTSET:
        raise NotImplementedError(
            "Don't know how to dump data for %s" % headers["Content-Type"])

    logmsg = "Queued request `%s %s`" % (method, url)

    # prepare keyword arguments
    kwargs.update(headers=headers)
    if request_data is not NOTSET:
        kwargs.update(data=request_data)
        logmsg += " data: %r" % data

    # setup the request from treq
    logger.debug(logmsg)
    deferred = treq.request(method, url, **kwargs)
    deferred.addCallback(unpack_response)
    deferred.addErrback(errback)

    return deferred

head = partial(request, "HEAD")
get = partial(request, "GET")
post = partial(request, "POST")
put = partial(request, "PUT")
patch = partial(request, "PATCH")
delete = partial(request, "DELETE")
<commit_msg>adding method<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
HTTP Client
===========

The client library the manager uses to communicate with
the master server.
"""

import json
from collections import namedtuple
from functools import partial

import treq
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol, connectionDone
from twisted.python import log
from twisted.web.client import Response as _Response, GzipDecoder, ResponseDone

from pyfarm.core.enums import STRING_TYPES, NOTSET
from pyfarm.core.logger import getLogger

Request = namedtuple(
    "Request",
    ["method", "url", "headers", "data"])

logger = getLogger("agent.http")

class Response(Protocol):
    """
    This class receives the incoming response body from a request
    constructs some convenience methods and attributes around the data.

    :param deferred:
        The deferred object which contains the target callback
        and errback.

    :param response:
        The initial response object which will be passed along
        to the target deferred.

    :param request:
        Named tuple object containing the method name, url, headers, and data.
    """
    def __init__(self, deferred, response, request):
        assert isinstance(deferred, Deferred)
        assert isinstance(response, (_Response, GzipDecoder))
        assert isinstance(request, Request)

        # internal attributes
        self._done = False
        self._body = ""
        self._deferred = deferred

        # main public attributes
        self.request = request
        self.response = response

        # convenience attributes constructed
        # from the public attributes
        self.method = self.request.method
        self.url = self.request.url
        self.code = self.response.code
        self.content_type = None

        # consume the response headers
        self.headers = {}
        for header_key, header_value in response.headers.getAllRawHeaders():
            if len(header_value) == 1:
                header_value = header_value[0]
            self.headers[header_key] = header_value

        # determine the content type
        if "Content-Type" in self.headers:
            self.content_type = self.headers["Content-Type"]

    def data(self):
        """
        Returns the data currently contained in the buffer.

        :raises RuntimeError:
            Raised if this method id called before all data
            has been received.
        """
        if not self._done:
            raise RuntimeError("Response not yet received.")
        return self._body

    def json(self, loader=json.loads):
        """
        Returns the json data from the incoming request

        :raises RuntimeError:
            Raised if this method id called before all data
            has been received.

        :raises ValueError:
            Raised if the content type for this request is not
            application/json.
        """
        if not self._done:
            raise RuntimeError("Response not yet received.")
        elif self.content_type != "application/json":
            raise ValueError("Not an application/json response.")
        else:
            return loader(self._body)

    def dataReceived(self, data):
        """
        Overrides :meth:`.Protocol.dataReceived` and appends data
        to ``_body``.
        """
        self._body += data

    def connectionLost(self, reason=connectionDone):
        """
        Overrides :meth:`.Protocol.connectionLost` and sets the ``_done``
        when complete.  When called with :class:`.ResponseDone` for ``reason``
        this method will call the callback on ``_deferred``
        """
        if reason.type is ResponseDone:
            self._done = True
            logger.debug(
                "%s %s (code: %s: body: %r)",
                self.request.method, self.request.url, self.code, self._body)
            self._deferred.callback(self)
        else:
            self._deferred.errback(reason)

def request(method, url, **kwargs):
    """
    Wrapper around :func:`treq.request` with some added arguments
    and validation.

    :param str method:
        The HTTP method to use when making the request.

    :param str url:
        The url this request will be made to.

    :type data: str, list, tuple, set, dict
    :keyword data:
        The data to send along with some types of requests
        such as ``POST`` or ``PUT``

    :keyword dict headers:
        The headers to send along with the request to
        ``url``.  Currently only single values per header
        are supported.

    :keyword function callback:
        The function to deliver an instance of :class:`Response`
        once we receive and unpack a response.

    :keyword function errback:
        The function to deliver an error message to.  By default
        this will use :func:`.log.err`.

    :keyword class response_class:
        The class to use to unpack the internal response.  This is mainly
        used by the unittests but could be used elsewhere to add some
        custom behavior to the unpack process for the incoming response.
    """
    # check assumptions for arguments
    assert method in ("HEAD", "GET", "POST", "PUT", "PATCH", "DELETE")
    assert isinstance(url, STRING_TYPES) and url

    data = kwargs.pop("data", NOTSET)
    headers = kwargs.pop("headers", {})
    callback = kwargs.pop("callback", None)
    errback = kwargs.pop("errback", log.err)
    response_class = kwargs.pop("response_class", Response)

    # check assumptions for keywords
    assert callable(callback) and callable(errback)
    assert data is NOTSET or \
           isinstance(data, tuple(list(STRING_TYPES) + [dict, list]))

    # add our default headers
    headers.setdefault("Content-Type", ["application/json"])
    headers.setdefault("User-Agent", ["PyFarm (agent) 1.0"])

    # ensure all values in the headers are lists (needed by Twisted)
    for header, value in headers.items():
        if isinstance(value, STRING_TYPES):
            headers[header] = [value]

        # for our purposes we should not expect headers with more
        # than one value for now
        elif isinstance(value, (list, tuple, set)):
            assert len(value) == 1

        else:
            raise NotImplementedError(
                "cannot handle header values with type %s" % type(value))

    def unpack_response(response):
        deferred = Deferred()
        deferred.addCallback(callback)

        # Deliver the body onto an instance of the response
        # object along with the original request.  Finally
        # the request and response via an instance of `Response`
        # to the outer scope's callback function.
        response.deliverBody(
            response_class(
                deferred, response,
                Request(method=method, url=url, data=data, headers=headers)))

        return deferred

    # prepare to send the data
    request_data = data  # so we don't modify the original data
    if request_data is not NOTSET and \
                    headers["Content-Type"] == ["application/json"]:
        request_data = json.dumps(data)

    elif data is not NOTSET:
        raise NotImplementedError(
            "Don't know how to dump data for %s" % headers["Content-Type"])

    logmsg = "Queued request `%s %s`" % (method, url)

    # prepare keyword arguments
    kwargs.update(headers=headers)
    if request_data is not NOTSET:
        kwargs.update(data=request_data)
        logmsg += " data: %r" % data

    # setup the request from treq
    logger.debug(logmsg)
    deferred = treq.request(method, url, **kwargs)
    deferred.addCallback(unpack_response)
    deferred.addErrback(errback)

    return deferred

head = partial(request, "HEAD")
get = partial(request, "GET")
post = partial(request, "POST")
put = partial(request, "PUT")
patch = partial(request, "PATCH")
delete = partial(request, "DELETE")
<|endoftext|>


<commit_before>import numpy

def confusion_matrix(rater_a, rater_b,
		 min_rating=None, max_rating=None):
    """
    Returns the confusion matrix between rater's ratings
    """
    assert(len(rater_a)==len(rater_b))
    if min_rating is None:
        min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
        max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    num_ratings = max_rating - min_rating + 1
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a,b in zip(rater_a,rater_b):
        conf_mat[a-min_rating][b-min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    """
    Returns the counts of each type of rating that a rater made
    """
    if min_rating is None: min_rating = reduce(min, ratings)
    if max_rating is None: max_rating = reduce(max, ratings)
    num_ratings = max_rating - min_rating + 1
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
	    hist_ratings[r-min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(rater_a, rater_b,
                             min_rating = None, max_rating = None):
    """
    Calculates the quadratic weighted kappa, a measure of
    agreement between two raters
    """
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
	min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
	max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
				     min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
	for j in range(num_ratings):
	    expected_count = (hist_rater_a[i]*hist_rater_b[j]
			      / num_scored_items)
	    d = pow(i-j,2.0) / pow(num_ratings-1, 2.0)
	    numerator += d*conf_mat[i][j] / num_scored_items
	    denominator += d*expected_count / num_scored_items

    return 1.0 - numerator / denominator

def mean_quadratic_weighted_kappa(kappas, weights=None):
    """
    Calculates the mean kappa in the z-space
    """
    kappas = numpy.array(kappas, dtype=float)
    if weights is None:
        weights = numpy.ones(numpy.shape(kappas))
    else:
        weights = weights / numpy.mean(weights)

    # ensure that kappas are in the range [-.999, .999]
    kappas = numpy.array([min(x, .999) for x in kappas])
    kappas = numpy.array([max(x, -.999) for x in kappas])

    z = 0.5 * numpy.log( (1+kappas)/(1-kappas) ) * weights
    z = numpy.mean(z)
    kappa = (numpy.exp(2*z)-1) / (numpy.exp(2*z)+1)
    return kappa
<commit_msg>Updated text<commit_after>import numpy

def confusion_matrix(rater_a, rater_b,
		 min_rating=None, max_rating=None):
    """
    Returns the confusion matrix between rater's ratings
    """
    assert(len(rater_a)==len(rater_b))
    if min_rating is None:
        min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
        max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    num_ratings = max_rating - min_rating + 1
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a,b in zip(rater_a,rater_b):
        conf_mat[a-min_rating][b-min_rating] += 1
    return conf_mat

def histogram(ratings, min_rating=None, max_rating=None):
    """
    Returns the counts of each type of rating that a rater made
    """
    if min_rating is None: min_rating = reduce(min, ratings)
    if max_rating is None: max_rating = reduce(max, ratings)
    num_ratings = max_rating - min_rating + 1
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
	    hist_ratings[r-min_rating] += 1
    return hist_ratings

def quadratic_weighted_kappa(rater_a, rater_b,
                             min_rating = None, max_rating = None):
    """
    Calculates the quadratic weighted kappa
    scoreQuadraticWeightedKappa calculates the quadratic weighted kappa
    value, which is a measure of inter-rater agreement between two raters
    that provide discrete numeric ratings.  Potential values range from -1
    (representing complete disagreement) to 1 (representing complete
    agreement).  A kappa value of 0 is expected if all agreement is due to
    chance.

    scoreQuadraticWeightedKappa(rater_a, rater_b), where rater_a and rater_b
    each correspond to a list of integer ratings.  These lists must have the
    same length.

    The ratings should be integers, and it is assumed that they contain
    the complete range of possible ratings.

    score_quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating
    is the minimum possible rating, and max_rating is the maximum possible
    rating
    """
    assert(len(rater_a) == len(rater_b))
    if min_rating is None:
	min_rating = min(reduce(min, rater_a), reduce(min, rater_b))
    if max_rating is None:
	max_rating = max(reduce(max, rater_a), reduce(max, rater_b))
    conf_mat = confusion_matrix(rater_a, rater_b,
				     min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    for i in range(num_ratings):
	for j in range(num_ratings):
	    expected_count = (hist_rater_a[i]*hist_rater_b[j]
			      / num_scored_items)
	    d = pow(i-j,2.0) / pow(num_ratings-1, 2.0)
	    numerator += d*conf_mat[i][j] / num_scored_items
	    denominator += d*expected_count / num_scored_items

    return 1.0 - numerator / denominator

def mean_quadratic_weighted_kappa(kappas, weights=None):
    """
    Calculates the mean of the quadratic
    weighted kappas after applying Fisher's r-to-z transform, which is
    approximately a variance-stabilizing transformation.  This
    transformation is undefined if one of the kappas is 1.0, so all kappa
    values are capped in the range (-0.999, 0.999).  The reverse
    transformation is then applied before returning the result.

    mean_quadratic_weighted_kappa(kappas), where kappas is a vector of
    kappa values

    mean_quadratic_weighted_kappa(kappas, weights), where weights is a vector
    of weights that is the same size as kappas.  Weights are applied in the
    z-space
    """
    kappas = numpy.array(kappas, dtype=float)
    if weights is None:
        weights = numpy.ones(numpy.shape(kappas))
    else:
        weights = weights / numpy.mean(weights)

    # ensure that kappas are in the range [-.999, .999]
    kappas = numpy.array([min(x, .999) for x in kappas])
    kappas = numpy.array([max(x, -.999) for x in kappas])

    z = 0.5 * numpy.log( (1+kappas)/(1-kappas) ) * weights
    z = numpy.mean(z)
    kappa = (numpy.exp(2*z)-1) / (numpy.exp(2*z)+1)
    return kappa
<|endoftext|>


<commit_before>#!/usr/bin/python
# encoding: utf-8

# (c) 2013, Matthias Vogelgesang <matthias.vogelgesang@gmail.com>
# (c) 2014, Justin Lecher <jlec@gentoo.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: zypper_repository
author: "Matthias Vogelgesang (@matze)"
version_added: "1.4"
short_description: Add and remove Zypper repositories
description:
    - Add or remove Zypper repositories on SUSE and openSUSE
options:
    name:
        required: false
        default: none
        description:
            - A name for the repository. Not required when adding repofiles.
    repo:
        required: false
        default: none
        description:
            - URI of the repository or .repo file. Required when state=present.
    state:
        required: false
        choices: [ "absent", "present" ]
        default: "present"
        description:
            - A source string state.
    description:
        required: false
        default: none
        description:
            - A description of the repository
    disable_gpg_check:
        description:
            - Whether to disable GPG signature checking of
              all packages. Has an effect only if state is
              I(present).
            - Needs zypper version >= 1.6.2.
        required: false
        default: "no"
        choices: [ "yes", "no" ]
    autorefresh:
        description:
            - Enable autorefresh of the repository.
        required: false
        default: "yes"
        choices: [ "yes", "no" ]
        aliases: [ "refresh" ]
    priority:
        description:
            - Set priority of repository. Packages will always be installed
              from the repository with the smallest priority number.
            - Needs zypper version >= 1.12.25.
        required: false
        version_added: "2.1"
    overwrite_multiple:
        description:
            - Overwrite multiple repository entries, if repositories with both name and
              URL already exist.
        required: false
        default: "no"
        choices: [ "yes", "no" ]
        version_added: "2.1"
    auto_import_keys:
        description:
            - Automatically import the gpg signing key of the new or changed repository.
            - Has an effect only if state is I(present). Has no effect on existing (unchanged) repositories or in combination with I(absent).
            - Implies runrefresh.
        required: false
        default: "no"
        choices: ["yes", "no"]
        version_added: "2.2"
    runrefresh:
        description:
            - Refresh the package list of the given repository.
            - Can be used with repo=* to refresh all repositories.
        required: false
        default: "no"
        choices: ["yes", "no"]
        version_added: "2.2"

requirements:
    - "zypper >= 1.0  # included in openSuSE >= 11.1 or SuSE Linux Enterprise Server/Desktop >= 11.0"
    - python-xml
'''

EXAMPLES = '''
# Add NVIDIA repository for graphics drivers
- zypper_repository: name=nvidia-repo repo='ftp://download.nvidia.com/opensuse/12.2' state=present

# Remove NVIDIA repository
- zypper_repository: name=nvidia-repo repo='ftp://download.nvidia.com/opensuse/12.2' state=absent

# Add python development repository
- zypper_repository: repo=http://download.opensuse.org/repositories/devel:/languages:/python/SLE_11_SP3/devel:languages:python.repo

# Refresh all repos
- zypper_repository: repo=* runrefresh=yes

# Add a repo and add it's gpg key
- zypper_repository: repo=http://download.opensuse.org/repositories/systemsmanagement/openSUSE_Leap_42.1/ auto_import_keys=yes

# Force refresh of a repository
- zypper_repository: repo=http://my_internal_ci_repo/repo name=my_ci_repo state=present runrefresh=yes
'''

REPO_OPTS = ['alias', 'name', 'priority', 'enabled', 'autorefresh', 'gpgcheck']

from distutils.version import LooseVersion

def _get_cmd(*args):
    """Combines the non-interactive zypper command with arguments/subcommands"""
    cmd = ['/usr/bin/zypper', '--quiet', '--non-interactive']
    cmd.extend(args)

    return cmd

def _parse_repos(module):
    """parses the output of zypper --xmlout repos and return a parse repo dictionary"""
    cmd = _get_cmd('--xmlout', 'repos')

    from xml.dom.minidom import parseString as parseXML
    rc, stdout, stderr = module.run_command(cmd, check_rc=False)
    if rc == 0:
        repos = []
        dom = parseXML(stdout)
        repo_list = dom.getElementsByTagName('repo')
        for repo in repo_list:
            opts = {}
            for o in REPO_OPTS:
                opts[o] = repo.getAttribute(o)
            opts['url'] = repo.getElementsByTagName('url')[0].firstChild.data
            # A repo can be uniquely identified by an alias + url
            repos.append(opts)
        return repos
    # exit code 6 is ZYPPER_EXIT_NO_REPOS (no repositories defined)
    elif rc == 6:
        return []
    else:
        module.fail_json(msg='Failed to execute "%s"' % " ".join(cmd), rc=rc, stdout=stdout, stderr=stderr)

def _repo_changes(realrepo, repocmp):
    "Check whether the 2 given repos have different settings."
    for k in repocmp:
        if repocmp[k] and k not in realrepo:
            return True

    for k, v in realrepo.items():
        if k in repocmp and repocmp[k]:
            valold = str(repocmp[k] or "")
            valnew = v or ""
            if k == "url":
                valold, valnew = valold.rstrip("/"), valnew.rstrip("/")
            if valold != valnew:
                return True
    return False

def repo_exists(module, repodata, overwrite_multiple):
    """Check whether the repository already exists.

        returns (exists, mod, old_repos)
            exists: whether a matching (name, URL) repo exists
            mod: whether there are changes compared to the existing repo
            old_repos: list of matching repos
    """
    existing_repos = _parse_repos(module)

    # look for repos that have matching alias or url to the one searched
    repos = []
    for kw in ['alias', 'url']:
        name = repodata[kw]
        for oldr in existing_repos:
            if repodata[kw] == oldr[kw] and oldr not in repos:
                repos.append(oldr)

    if len(repos) == 0:
        # Repo does not exist yet
        return (False, False, None)
    elif len(repos) == 1:
        # Found an existing repo, look for changes
        has_changes = _repo_changes(repos[0], repodata)
        return (True, has_changes, repos)
    elif len(repos) >= 2:
        if overwrite_multiple:
            # Found two repos and want to overwrite_multiple
            return (True, True, repos)
        else:
            errmsg = 'More than one repo matched "%s": "%s".' % (name, repos)
            errmsg += ' Use overwrite_multiple to allow more than one repo to be overwritten'
            module.fail_json(msg=errmsg)

def addmodify_repo(module, repodata, old_repos, zypper_version, warnings):
    "Adds the repo, removes old repos before, that would conflict."
    repo = repodata['url']
    cmd = _get_cmd('addrepo', '--check')
    if repodata['name']:
        cmd.extend(['--name', repodata['name']])

    # priority on addrepo available since 1.12.25
    # https://github.com/openSUSE/zypper/blob/b9b3cb6db76c47dc4c47e26f6a4d2d4a0d12b06d/package/zypper.changes#L327-L336
    if repodata['priority']:
        if zypper_version >= LooseVersion('1.12.25'):
            cmd.extend(['--priority', str(repodata['priority'])])
        else:
            warnings.append("Setting priority only available for zypper >= 1.12.25. Ignoring priority argument.")

    if repodata['enabled'] == '0':
        cmd.append('--disable')

    # gpgcheck available since 1.6.2
    # https://github.com/openSUSE/zypper/blob/b9b3cb6db76c47dc4c47e26f6a4d2d4a0d12b06d/package/zypper.changes#L2446-L2449
    # the default changed in the past, so don't assume a default here and show warning for old zypper versions
    if zypper_version >= LooseVersion('1.6.2'):
        if repodata['gpgcheck'] == '1':
            cmd.append('--gpgcheck')
        else:
            cmd.append('--no-gpgcheck')
    else:
        warnings.append("Enabling/disabling gpgcheck only available for zypper >= 1.6.2. Using zypper default value.")

    if repodata['autorefresh'] == '1':
        cmd.append('--refresh')

    cmd.append(repo)

    if not repo.endswith('.repo'):
        cmd.append(repodata['alias'])

    if old_repos is not None:
        for oldrepo in old_repos:
            remove_repo(module, oldrepo['url'])

    rc, stdout, stderr = module.run_command(cmd, check_rc=False)
    return rc, stdout, stderr

def remove_repo(module, repo):
    "Removes the repo."
    cmd = _get_cmd('removerepo', repo)

    rc, stdout, stderr = module.run_command(cmd, check_rc=True)
    return rc, stdout, stderr

def get_zypper_version(module):
    rc, stdout, stderr = module.run_command(['/usr/bin/zypper', '--version'])
    if rc != 0 or not stdout.startswith('zypper '):
        return LooseVersion('1.0')
    return LooseVersion(stdout.split()[1])

def runrefreshrepo(module, auto_import_keys=False, shortname=None):
    "Forces zypper to refresh repo metadata."
    cmd = _get_cmd('refresh', '--force')
    if auto_import_keys:
        cmd.append('--gpg-auto-import-keys')
    if shortname is not None:
        cmd.extend(['-r', shortname])

    rc, stdout, stderr = module.run_command(cmd, check_rc=True)
    return rc, stdout, stderr

def main():
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(required=False),
            repo=dict(required=False),
            state=dict(choices=['present', 'absent'], default='present'),
            runrefresh=dict(required=False, default='no', type='bool'),
            description=dict(required=False),
            disable_gpg_check = dict(required=False, default=False, type='bool'),
            autorefresh = dict(required=False, default=True, type='bool', aliases=['refresh']),
            priority = dict(required=False, type='int'),
            enabled = dict(required=False, default=True, type='bool'),
            overwrite_multiple = dict(required=False, default=False, type='bool'),
            auto_import_keys = dict(required=False, default=False, type='bool'),
        ),
        supports_check_mode=False,
        required_one_of = [['state','runrefresh']],
    )

    repo = module.params['repo']
    alias = module.params['name']
    state = module.params['state']
    overwrite_multiple = module.params['overwrite_multiple']
    auto_import_keys = module.params['auto_import_keys']
    runrefresh = module.params['runrefresh']

    zypper_version = get_zypper_version(module)
    warnings = []  # collect warning messages for final output

    repodata = {
        'url': repo,
        'alias': alias,
        'name': module.params['description'],
        'priority': module.params['priority'],
    }
    # rewrite bools in the language that zypper lr -x provides for easier comparison
    if module.params['enabled']:
        repodata['enabled'] = '1'
    else:
        repodata['enabled'] = '0'
    if module.params['disable_gpg_check']:
        repodata['gpgcheck'] = '0'
    else:
        repodata['gpgcheck'] = '1'
    if module.params['autorefresh']:
        repodata['autorefresh'] = '1'
    else:
        repodata['autorefresh'] = '0'

    def exit_unchanged():
        module.exit_json(changed=False, repodata=repodata, state=state)

    # Check run-time module parameters
    if repo == '*' or alias == '*':
        if runrefresh:
            runrefreshrepo(module, auto_import_keys)
            module.exit_json(changed=False, runrefresh=True)
        else:
            module.fail_json(msg='repo=* can only be used with the runrefresh option.')

    if state == 'present' and not repo:
        module.fail_json(msg='Module option state=present requires repo')
    if state == 'absent' and not repo and not alias:
        module.fail_json(msg='Alias or repo parameter required when state=absent')

    if repo and repo.endswith('.repo'):
        if alias:
            module.fail_json(msg='Incompatible option: \'name\'. Do not use name when adding .repo files')
    else:
        if not alias and state == "present":
            module.fail_json(msg='Name required when adding non-repo files.')

    exists, mod, old_repos = repo_exists(module, repodata, overwrite_multiple)

    if repo:
        shortname = repo
    else:
        shortname = alias

    if state == 'present':
        if exists and not mod:
            if runrefresh:
                runrefreshrepo(module, auto_import_keys, shortname)
            exit_unchanged()
        rc, stdout, stderr = addmodify_repo(module, repodata, old_repos, zypper_version, warnings)
        if rc == 0 and (runrefresh or auto_import_keys):
            runrefreshrepo(module, auto_import_keys, shortname)
    elif state == 'absent':
        if not exists:
            exit_unchanged()
        rc, stdout, stderr = remove_repo(module, shortname)

    if rc == 0:
        module.exit_json(changed=True, repodata=repodata, state=state, warnings=warnings)
    else:
        module.fail_json(msg="Zypper failed with rc %s" % rc, rc=rc, stdout=stdout, stderr=stderr, repodata=repodata, state=state, warnings=warnings)

# import module snippets
from ansible.module_utils.basic import *

main()
<commit_msg>Add documention for enabled option in zypper_repository (#3083)<commit_after>#!/usr/bin/python
# encoding: utf-8

# (c) 2013, Matthias Vogelgesang <matthias.vogelgesang@gmail.com>
# (c) 2014, Justin Lecher <jlec@gentoo.org>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

DOCUMENTATION = '''
---
module: zypper_repository
author: "Matthias Vogelgesang (@matze)"
version_added: "1.4"
short_description: Add and remove Zypper repositories
description:
    - Add or remove Zypper repositories on SUSE and openSUSE
options:
    name:
        required: false
        default: none
        description:
            - A name for the repository. Not required when adding repofiles.
    repo:
        required: false
        default: none
        description:
            - URI of the repository or .repo file. Required when state=present.
    state:
        required: false
        choices: [ "absent", "present" ]
        default: "present"
        description:
            - A source string state.
    description:
        required: false
        default: none
        description:
            - A description of the repository
    disable_gpg_check:
        description:
            - Whether to disable GPG signature checking of
              all packages. Has an effect only if state is
              I(present).
            - Needs zypper version >= 1.6.2.
        required: false
        default: "no"
        choices: [ "yes", "no" ]
    autorefresh:
        description:
            - Enable autorefresh of the repository.
        required: false
        default: "yes"
        choices: [ "yes", "no" ]
        aliases: [ "refresh" ]
    priority:
        description:
            - Set priority of repository. Packages will always be installed
              from the repository with the smallest priority number.
            - Needs zypper version >= 1.12.25.
        required: false
        version_added: "2.1"
    overwrite_multiple:
        description:
            - Overwrite multiple repository entries, if repositories with both name and
              URL already exist.
        required: false
        default: "no"
        choices: [ "yes", "no" ]
        version_added: "2.1"
    auto_import_keys:
        description:
            - Automatically import the gpg signing key of the new or changed repository.
            - Has an effect only if state is I(present). Has no effect on existing (unchanged) repositories or in combination with I(absent).
            - Implies runrefresh.
        required: false
        default: "no"
        choices: ["yes", "no"]
        version_added: "2.2"
    runrefresh:
        description:
            - Refresh the package list of the given repository.
            - Can be used with repo=* to refresh all repositories.
        required: false
        default: "no"
        choices: ["yes", "no"]
        version_added: "2.2"
    enabled:
        description:
            - Set repository to enabled (or disabled).
        required: false
        default: "yes"
        choices: ["yes", "no"]
        version_added: "2.2"

requirements:
    - "zypper >= 1.0  # included in openSuSE >= 11.1 or SuSE Linux Enterprise Server/Desktop >= 11.0"
    - python-xml
'''

EXAMPLES = '''
# Add NVIDIA repository for graphics drivers
- zypper_repository: name=nvidia-repo repo='ftp://download.nvidia.com/opensuse/12.2' state=present

# Remove NVIDIA repository
- zypper_repository: name=nvidia-repo repo='ftp://download.nvidia.com/opensuse/12.2' state=absent

# Add python development repository
- zypper_repository: repo=http://download.opensuse.org/repositories/devel:/languages:/python/SLE_11_SP3/devel:languages:python.repo

# Refresh all repos
- zypper_repository: repo=* runrefresh=yes

# Add a repo and add it's gpg key
- zypper_repository: repo=http://download.opensuse.org/repositories/systemsmanagement/openSUSE_Leap_42.1/ auto_import_keys=yes

# Force refresh of a repository
- zypper_repository: repo=http://my_internal_ci_repo/repo name=my_ci_repo state=present runrefresh=yes
'''

REPO_OPTS = ['alias', 'name', 'priority', 'enabled', 'autorefresh', 'gpgcheck']

from distutils.version import LooseVersion

def _get_cmd(*args):
    """Combines the non-interactive zypper command with arguments/subcommands"""
    cmd = ['/usr/bin/zypper', '--quiet', '--non-interactive']
    cmd.extend(args)

    return cmd

def _parse_repos(module):
    """parses the output of zypper --xmlout repos and return a parse repo dictionary"""
    cmd = _get_cmd('--xmlout', 'repos')

    from xml.dom.minidom import parseString as parseXML
    rc, stdout, stderr = module.run_command(cmd, check_rc=False)
    if rc == 0:
        repos = []
        dom = parseXML(stdout)
        repo_list = dom.getElementsByTagName('repo')
        for repo in repo_list:
            opts = {}
            for o in REPO_OPTS:
                opts[o] = repo.getAttribute(o)
            opts['url'] = repo.getElementsByTagName('url')[0].firstChild.data
            # A repo can be uniquely identified by an alias + url
            repos.append(opts)
        return repos
    # exit code 6 is ZYPPER_EXIT_NO_REPOS (no repositories defined)
    elif rc == 6:
        return []
    else:
        module.fail_json(msg='Failed to execute "%s"' % " ".join(cmd), rc=rc, stdout=stdout, stderr=stderr)

def _repo_changes(realrepo, repocmp):
    "Check whether the 2 given repos have different settings."
    for k in repocmp:
        if repocmp[k] and k not in realrepo:
            return True

    for k, v in realrepo.items():
        if k in repocmp and repocmp[k]:
            valold = str(repocmp[k] or "")
            valnew = v or ""
            if k == "url":
                valold, valnew = valold.rstrip("/"), valnew.rstrip("/")
            if valold != valnew:
                return True
    return False

def repo_exists(module, repodata, overwrite_multiple):
    """Check whether the repository already exists.

        returns (exists, mod, old_repos)
            exists: whether a matching (name, URL) repo exists
            mod: whether there are changes compared to the existing repo
            old_repos: list of matching repos
    """
    existing_repos = _parse_repos(module)

    # look for repos that have matching alias or url to the one searched
    repos = []
    for kw in ['alias', 'url']:
        name = repodata[kw]
        for oldr in existing_repos:
            if repodata[kw] == oldr[kw] and oldr not in repos:
                repos.append(oldr)

    if len(repos) == 0:
        # Repo does not exist yet
        return (False, False, None)
    elif len(repos) == 1:
        # Found an existing repo, look for changes
        has_changes = _repo_changes(repos[0], repodata)
        return (True, has_changes, repos)
    elif len(repos) >= 2:
        if overwrite_multiple:
            # Found two repos and want to overwrite_multiple
            return (True, True, repos)
        else:
            errmsg = 'More than one repo matched "%s": "%s".' % (name, repos)
            errmsg += ' Use overwrite_multiple to allow more than one repo to be overwritten'
            module.fail_json(msg=errmsg)

def addmodify_repo(module, repodata, old_repos, zypper_version, warnings):
    "Adds the repo, removes old repos before, that would conflict."
    repo = repodata['url']
    cmd = _get_cmd('addrepo', '--check')
    if repodata['name']:
        cmd.extend(['--name', repodata['name']])

    # priority on addrepo available since 1.12.25
    # https://github.com/openSUSE/zypper/blob/b9b3cb6db76c47dc4c47e26f6a4d2d4a0d12b06d/package/zypper.changes#L327-L336
    if repodata['priority']:
        if zypper_version >= LooseVersion('1.12.25'):
            cmd.extend(['--priority', str(repodata['priority'])])
        else:
            warnings.append("Setting priority only available for zypper >= 1.12.25. Ignoring priority argument.")

    if repodata['enabled'] == '0':
        cmd.append('--disable')

    # gpgcheck available since 1.6.2
    # https://github.com/openSUSE/zypper/blob/b9b3cb6db76c47dc4c47e26f6a4d2d4a0d12b06d/package/zypper.changes#L2446-L2449
    # the default changed in the past, so don't assume a default here and show warning for old zypper versions
    if zypper_version >= LooseVersion('1.6.2'):
        if repodata['gpgcheck'] == '1':
            cmd.append('--gpgcheck')
        else:
            cmd.append('--no-gpgcheck')
    else:
        warnings.append("Enabling/disabling gpgcheck only available for zypper >= 1.6.2. Using zypper default value.")

    if repodata['autorefresh'] == '1':
        cmd.append('--refresh')

    cmd.append(repo)

    if not repo.endswith('.repo'):
        cmd.append(repodata['alias'])

    if old_repos is not None:
        for oldrepo in old_repos:
            remove_repo(module, oldrepo['url'])

    rc, stdout, stderr = module.run_command(cmd, check_rc=False)
    return rc, stdout, stderr

def remove_repo(module, repo):
    "Removes the repo."
    cmd = _get_cmd('removerepo', repo)

    rc, stdout, stderr = module.run_command(cmd, check_rc=True)
    return rc, stdout, stderr

def get_zypper_version(module):
    rc, stdout, stderr = module.run_command(['/usr/bin/zypper', '--version'])
    if rc != 0 or not stdout.startswith('zypper '):
        return LooseVersion('1.0')
    return LooseVersion(stdout.split()[1])

def runrefreshrepo(module, auto_import_keys=False, shortname=None):
    "Forces zypper to refresh repo metadata."
    cmd = _get_cmd('refresh', '--force')
    if auto_import_keys:
        cmd.append('--gpg-auto-import-keys')
    if shortname is not None:
        cmd.extend(['-r', shortname])

    rc, stdout, stderr = module.run_command(cmd, check_rc=True)
    return rc, stdout, stderr

def main():
    module = AnsibleModule(
        argument_spec=dict(
            name=dict(required=False),
            repo=dict(required=False),
            state=dict(choices=['present', 'absent'], default='present'),
            runrefresh=dict(required=False, default='no', type='bool'),
            description=dict(required=False),
            disable_gpg_check = dict(required=False, default=False, type='bool'),
            autorefresh = dict(required=False, default=True, type='bool', aliases=['refresh']),
            priority = dict(required=False, type='int'),
            enabled = dict(required=False, default=True, type='bool'),
            overwrite_multiple = dict(required=False, default=False, type='bool'),
            auto_import_keys = dict(required=False, default=False, type='bool'),
        ),
        supports_check_mode=False,
        required_one_of = [['state','runrefresh']],
    )

    repo = module.params['repo']
    alias = module.params['name']
    state = module.params['state']
    overwrite_multiple = module.params['overwrite_multiple']
    auto_import_keys = module.params['auto_import_keys']
    runrefresh = module.params['runrefresh']

    zypper_version = get_zypper_version(module)
    warnings = []  # collect warning messages for final output

    repodata = {
        'url': repo,
        'alias': alias,
        'name': module.params['description'],
        'priority': module.params['priority'],
    }
    # rewrite bools in the language that zypper lr -x provides for easier comparison
    if module.params['enabled']:
        repodata['enabled'] = '1'
    else:
        repodata['enabled'] = '0'
    if module.params['disable_gpg_check']:
        repodata['gpgcheck'] = '0'
    else:
        repodata['gpgcheck'] = '1'
    if module.params['autorefresh']:
        repodata['autorefresh'] = '1'
    else:
        repodata['autorefresh'] = '0'

    def exit_unchanged():
        module.exit_json(changed=False, repodata=repodata, state=state)

    # Check run-time module parameters
    if repo == '*' or alias == '*':
        if runrefresh:
            runrefreshrepo(module, auto_import_keys)
            module.exit_json(changed=False, runrefresh=True)
        else:
            module.fail_json(msg='repo=* can only be used with the runrefresh option.')

    if state == 'present' and not repo:
        module.fail_json(msg='Module option state=present requires repo')
    if state == 'absent' and not repo and not alias:
        module.fail_json(msg='Alias or repo parameter required when state=absent')

    if repo and repo.endswith('.repo'):
        if alias:
            module.fail_json(msg='Incompatible option: \'name\'. Do not use name when adding .repo files')
    else:
        if not alias and state == "present":
            module.fail_json(msg='Name required when adding non-repo files.')

    exists, mod, old_repos = repo_exists(module, repodata, overwrite_multiple)

    if repo:
        shortname = repo
    else:
        shortname = alias

    if state == 'present':
        if exists and not mod:
            if runrefresh:
                runrefreshrepo(module, auto_import_keys, shortname)
            exit_unchanged()
        rc, stdout, stderr = addmodify_repo(module, repodata, old_repos, zypper_version, warnings)
        if rc == 0 and (runrefresh or auto_import_keys):
            runrefreshrepo(module, auto_import_keys, shortname)
    elif state == 'absent':
        if not exists:
            exit_unchanged()
        rc, stdout, stderr = remove_repo(module, shortname)

    if rc == 0:
        module.exit_json(changed=True, repodata=repodata, state=state, warnings=warnings)
    else:
        module.fail_json(msg="Zypper failed with rc %s" % rc, rc=rc, stdout=stdout, stderr=stderr, repodata=repodata, state=state, warnings=warnings)

# import module snippets
from ansible.module_utils.basic import *

main()
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import os

def _verbs_dir():
    """Try to find the directory where the OpenFabrics verbs package is
    installed. Return None if not found."""
    try:
        # Try to locate Verbs by looking for a utility in the path
        ibv_devices = which("ibv_devices")
        # Run it (silently) to ensure it works
        ibv_devices(output=str, error=str)
        # Get path to executable
        path = ibv_devices.exe[0]
        # Remove executable name and "bin" directory
        path = os.path.dirname(path)
        path = os.path.dirname(path)
        # There's usually no "/include" on Unix; use "/usr/include" instead
        if path == "/":
            path = "/usr"
        return path
    except:
        return None

class Openmpi(AutotoolsPackage):
    """The Open MPI Project is an open source Message Passing Interface
       implementation that is developed and maintained by a consortium
       of academic, research, and industry partners. Open MPI is
       therefore able to combine the expertise, technologies, and
       resources from all across the High Performance Computing
       community in order to build the best MPI library available.
       Open MPI offers advantages for system and software vendors,
       application developers and computer science researchers.
    """

    homepage = "http://www.open-mpi.org"
    url = "http://www.open-mpi.org/software/ompi/v1.10/downloads/openmpi-1.10.1.tar.bz2"
    list_url = "http://www.open-mpi.org/software/ompi/"
    list_depth = 3

    version('2.0.2', 'ecd99aa436a1ca69ce936a96d6a3fa48')
    version('2.0.1', '6f78155bd7203039d2448390f3b51c96')
    version('2.0.0', 'cdacc800cb4ce690c1f1273cb6366674')
    version('1.10.3', 'e2fe4513200e2aaa1500b762342c674b')
    version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c')
    version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e')
    version('1.10.0', '280cf952de68369cebaca886c5ce0304')
    version('1.8.8', '0dab8e602372da1425e9242ae37faf8c')
    version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475')

    patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5")
    patch('llnl-platforms.patch', when="@1.6.5")
    patch('configure.patch', when="@1.10.0:1.10.1")
    patch('fix_multidef_pmi_class.patch', when="@2.0.0:2.0.1")

    # Fabrics
    variant('psm', default=False, description='Build support for the PSM library')
    variant('psm2', default=False,
            description='Build support for the Intel PSM2 library')
    variant('pmi', default=False,
            description='Build support for PMI-based launchers')
    variant('verbs', default=_verbs_dir() is not None,
            description='Build support for OpenFabrics verbs')
    variant('mxm', default=False, description='Build Mellanox Messaging support')

    # Schedulers
    # TODO: support for alps and loadleveler is missing
    variant('tm', default=False,
            description='Build TM (Torque, PBSPro, and compatible) support')
    variant('slurm', default=False,
            description='Build SLURM scheduler component')

    # Additional support options
    variant('java', default=False, description='Build Java support')
    variant('sqlite3', default=False, description='Build SQLite3 support')
    variant('vt', default=True, description='Build VampirTrace support')
    variant('thread_multiple', default=False,
            description='Enable MPI_THREAD_MULTIPLE support')

    # TODO: support for CUDA is missing

    provides('mpi@:2.2', when='@1.6.5')
    provides('mpi@:3.0', when='@1.7.5:')
    provides('mpi@:3.1', when='@2.0.0:')

    depends_on('hwloc')
    depends_on('jdk', when='+java')
    depends_on('sqlite', when='+sqlite3')

    def url_for_version(self, version):
        return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (
            version.up_to(2), version)

    @property
    def libs(self):
        query_parameters = self.spec.last_query.extra_parameters
        libraries = ['libmpi']

        if 'cxx' in query_parameters:
            libraries = ['libmpi_cxx'] + libraries

        return find_libraries(
            libraries, root=self.prefix, shared=True, recurse=True
        )

    def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
        spack_env.set('MPICC',  join_path(self.prefix.bin, 'mpicc'))
        spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpic++'))
        spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77'))
        spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90'))

        spack_env.set('OMPI_CC', spack_cc)
        spack_env.set('OMPI_CXX', spack_cxx)
        spack_env.set('OMPI_FC', spack_fc)
        spack_env.set('OMPI_F77', spack_f77)

    def setup_dependent_package(self, module, dependent_spec):
        self.spec.mpicc = join_path(self.prefix.bin, 'mpicc')
        self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++')
        self.spec.mpifc = join_path(self.prefix.bin, 'mpif90')
        self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77')
        self.spec.mpicxx_shared_libs = [
            join_path(self.prefix.lib, 'libmpi_cxx.{0}'.format(dso_suffix)),
            join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
        ]

    @property
    def verbs(self):
        # Up through version 1.6, this option was previously named
        # --with-openib
        if self.spec.satisfies('@:1.6'):
            return 'openib'
        # In version 1.7, it was renamed to be --with-verbs
        elif self.spec.satisfies('@1.7:'):
            return 'verbs'

    @run_before('autoreconf')
    def die_without_fortran(self):
        # Until we can pass variants such as +fortran through virtual
        # dependencies depends_on('mpi'), require Fortran compiler to
        # avoid delayed build errors in dependents.
        if (self.compiler.f77 is None) or (self.compiler.fc is None):
            raise InstallError(
                'OpenMPI requires both C and Fortran compilers!'
            )

    def configure_args(self):
        spec = self.spec

        config_args = [
            '--enable-shared',
            '--enable-static',
            '--enable-mpi-cxx',
            # Schedulers
            '--with-tm' if '+tm' in spec else '--without-tm',
            '--with-slurm' if '+slurm' in spec else '--without-slurm',
            # Fabrics
            '--with-psm' if '+psm' in spec else '--without-psm',
        ]

        # Intel PSM2 support
        if spec.satisfies('@1.10:'):
            if '+psm2' in spec:
                config_args.append('--with-psm2')
            else:
                config_args.append('--without-psm2')

        # PMI support
        if spec.satisfies('@1.5.5:'):
            if '+pmi' in spec:
                config_args.append('--with-pmi')
            else:
                config_args.append('--without-pmi')

        # Mellanox Messaging support
        if spec.satisfies('@1.5.4:'):
            if '+mxm' in spec:
                config_args.append('--with-mxm')
            else:
                config_args.append('--without-mxm')

        # OpenFabrics verbs support
        if '+verbs' in spec:
            path = _verbs_dir()
            if path is not None and path not in ('/usr', '/usr/local'):
                config_args.append('--with-{0}={1}'.format(self.verbs, path))
            else:
                config_args.append('--with-{0}'.format(self.verbs))
        else:
            config_args.append('--without-{0}'.format(self.verbs))

        # Hwloc support
        if spec.satisfies('@1.5.2:'):
            config_args.append('--with-hwloc={0}'.format(spec['hwloc'].prefix))

        # Java support
        if spec.satisfies('@1.7.4:'):
            if '+java' in spec:
                config_args.extend([
                    '--enable-java',
                    '--enable-mpi-java',
                    '--with-jdk-dir={0}'.format(spec['jdk'].prefix)
                ])
            else:
                config_args.extend([
                    '--disable-java',
                    '--disable-mpi-java'
                ])

        # SQLite3 support
        if spec.satisfies('@1.7.3:1.999'):
            if '+sqlite3' in spec:
                config_args.append('--with-sqlite3')
            else:
                config_args.append('--without-sqlite3')

        # VampirTrace support
        if spec.satisfies('@1.3:1.999'):
            if '+vt' not in spec:
                config_args.append('--enable-contrib-no-build=vt')

        # Multithreading support
        if spec.satisfies('@1.5.4:'):
            if '+thread_multiple' in spec:
                config_args.append('--enable-mpi-thread-multiple')
            else:
                config_args.append('--disable-mpi-thread-multiple')

        return config_args

    @run_after('install')
    def filter_compilers(self):
        """Run after install to make the MPI compilers use the
           compilers that Spack built the package with.

           If this isn't done, they'll have CC, CXX and FC set
           to Spack's generic cc, c++ and f90.  We want them to
           be bound to whatever compiler they were built with.
        """
        kwargs = {'ignore_absent': True, 'backup': False, 'string': False}
        wrapper_basepath = join_path(self.prefix, 'share', 'openmpi')

        wrappers = [
            ('mpicc-vt-wrapper-data.txt', self.compiler.cc),
            ('mpicc-wrapper-data.txt', self.compiler.cc),
            ('ortecc-wrapper-data.txt', self.compiler.cc),
            ('shmemcc-wrapper-data.txt', self.compiler.cc),
            ('mpic++-vt-wrapper-data.txt', self.compiler.cxx),
            ('mpic++-wrapper-data.txt', self.compiler.cxx),
            ('ortec++-wrapper-data.txt', self.compiler.cxx),
            ('mpifort-vt-wrapper-data.txt', self.compiler.fc),
            ('mpifort-wrapper-data.txt', self.compiler.fc),
            ('shmemfort-wrapper-data.txt', self.compiler.fc),
            ('mpif90-vt-wrapper-data.txt', self.compiler.fc),
            ('mpif90-wrapper-data.txt', self.compiler.fc),
            ('mpif77-vt-wrapper-data.txt',  self.compiler.f77),
            ('mpif77-wrapper-data.txt',  self.compiler.f77)
        ]

        for wrapper_name, compiler in wrappers:
            wrapper = join_path(wrapper_basepath, wrapper_name)
            if not os.path.islink(wrapper):
                # Substitute Spack compile wrappers for the real
                # underlying compiler
                match = 'compiler=.*'
                substitute = 'compiler={compiler}'.format(compiler=compiler)
                filter_file(match, substitute, wrapper, **kwargs)
                # Remove this linking flag if present
                # (it turns RPATH into RUNPATH)
                filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs)
<commit_msg>Open MPI: add new packages (#3519)<commit_after>##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import os

def _verbs_dir():
    """Try to find the directory where the OpenFabrics verbs package is
    installed. Return None if not found."""
    try:
        # Try to locate Verbs by looking for a utility in the path
        ibv_devices = which("ibv_devices")
        # Run it (silently) to ensure it works
        ibv_devices(output=str, error=str)
        # Get path to executable
        path = ibv_devices.exe[0]
        # Remove executable name and "bin" directory
        path = os.path.dirname(path)
        path = os.path.dirname(path)
        # There's usually no "/include" on Unix; use "/usr/include" instead
        if path == "/":
            path = "/usr"
        return path
    except:
        return None

class Openmpi(AutotoolsPackage):
    """The Open MPI Project is an open source Message Passing Interface
       implementation that is developed and maintained by a consortium
       of academic, research, and industry partners. Open MPI is
       therefore able to combine the expertise, technologies, and
       resources from all across the High Performance Computing
       community in order to build the best MPI library available.
       Open MPI offers advantages for system and software vendors,
       application developers and computer science researchers.
    """

    homepage = "http://www.open-mpi.org"
    url = "https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2"
    list_url = "http://www.open-mpi.org/software/ompi/"
    list_depth = 3

    version('2.1.0', '4838a5973115c44e14442c01d3f21d52')
    version('2.0.2', 'ecd99aa436a1ca69ce936a96d6a3fa48')
    version('2.0.1', '6f78155bd7203039d2448390f3b51c96')
    version('2.0.0', 'cdacc800cb4ce690c1f1273cb6366674')
    version('1.10.6', '2e65008c1867b1f47c32f9f814d41706')
    version('1.10.5', 'd32ba9530a869d9c1eae930882ea1834')
    version('1.10.4', '9d2375835c5bc5c184ecdeb76c7c78ac')
    version('1.10.3', 'e2fe4513200e2aaa1500b762342c674b')
    version('1.10.2', 'b2f43d9635d2d52826e5ef9feb97fd4c')
    version('1.10.1', 'f0fcd77ed345b7eafb431968124ba16e')
    version('1.10.0', '280cf952de68369cebaca886c5ce0304')
    version('1.8.8', '0dab8e602372da1425e9242ae37faf8c')
    version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475')

    patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5")
    patch('llnl-platforms.patch', when="@1.6.5")
    patch('configure.patch', when="@1.10.0:1.10.1")
    patch('fix_multidef_pmi_class.patch', when="@2.0.0:2.0.1")

    # Fabrics
    variant('psm', default=False, description='Build support for the PSM library')
    variant('psm2', default=False,
            description='Build support for the Intel PSM2 library')
    variant('pmi', default=False,
            description='Build support for PMI-based launchers')
    variant('verbs', default=_verbs_dir() is not None,
            description='Build support for OpenFabrics verbs')
    variant('mxm', default=False, description='Build Mellanox Messaging support')

    # Schedulers
    # TODO: support for alps and loadleveler is missing
    variant('tm', default=False,
            description='Build TM (Torque, PBSPro, and compatible) support')
    variant('slurm', default=False,
            description='Build SLURM scheduler component')

    # Additional support options
    variant('java', default=False, description='Build Java support')
    variant('sqlite3', default=False, description='Build SQLite3 support')
    variant('vt', default=True, description='Build VampirTrace support')
    variant('thread_multiple', default=False,
            description='Enable MPI_THREAD_MULTIPLE support')

    # TODO: support for CUDA is missing

    provides('mpi@:2.2', when='@1.6.5')
    provides('mpi@:3.0', when='@1.7.5:')
    provides('mpi@:3.1', when='@2.0.0:')

    depends_on('hwloc')
    depends_on('jdk', when='+java')
    depends_on('sqlite', when='+sqlite3')

    def url_for_version(self, version):
        return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (
            version.up_to(2), version)

    @property
    def libs(self):
        query_parameters = self.spec.last_query.extra_parameters
        libraries = ['libmpi']

        if 'cxx' in query_parameters:
            libraries = ['libmpi_cxx'] + libraries

        return find_libraries(
            libraries, root=self.prefix, shared=True, recurse=True
        )

    def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
        spack_env.set('MPICC',  join_path(self.prefix.bin, 'mpicc'))
        spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpic++'))
        spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77'))
        spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90'))

        spack_env.set('OMPI_CC', spack_cc)
        spack_env.set('OMPI_CXX', spack_cxx)
        spack_env.set('OMPI_FC', spack_fc)
        spack_env.set('OMPI_F77', spack_f77)

    def setup_dependent_package(self, module, dependent_spec):
        self.spec.mpicc = join_path(self.prefix.bin, 'mpicc')
        self.spec.mpicxx = join_path(self.prefix.bin, 'mpic++')
        self.spec.mpifc = join_path(self.prefix.bin, 'mpif90')
        self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77')
        self.spec.mpicxx_shared_libs = [
            join_path(self.prefix.lib, 'libmpi_cxx.{0}'.format(dso_suffix)),
            join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
        ]

    @property
    def verbs(self):
        # Up through version 1.6, this option was previously named
        # --with-openib
        if self.spec.satisfies('@:1.6'):
            return 'openib'
        # In version 1.7, it was renamed to be --with-verbs
        elif self.spec.satisfies('@1.7:'):
            return 'verbs'

    @run_before('autoreconf')
    def die_without_fortran(self):
        # Until we can pass variants such as +fortran through virtual
        # dependencies depends_on('mpi'), require Fortran compiler to
        # avoid delayed build errors in dependents.
        if (self.compiler.f77 is None) or (self.compiler.fc is None):
            raise InstallError(
                'OpenMPI requires both C and Fortran compilers!'
            )

    def configure_args(self):
        spec = self.spec

        config_args = [
            '--enable-shared',
            '--enable-static',
            '--enable-mpi-cxx',
            # Schedulers
            '--with-tm' if '+tm' in spec else '--without-tm',
            '--with-slurm' if '+slurm' in spec else '--without-slurm',
            # Fabrics
            '--with-psm' if '+psm' in spec else '--without-psm',
        ]

        # Intel PSM2 support
        if spec.satisfies('@1.10:'):
            if '+psm2' in spec:
                config_args.append('--with-psm2')
            else:
                config_args.append('--without-psm2')

        # PMI support
        if spec.satisfies('@1.5.5:'):
            if '+pmi' in spec:
                config_args.append('--with-pmi')
            else:
                config_args.append('--without-pmi')

        # Mellanox Messaging support
        if spec.satisfies('@1.5.4:'):
            if '+mxm' in spec:
                config_args.append('--with-mxm')
            else:
                config_args.append('--without-mxm')

        # OpenFabrics verbs support
        if '+verbs' in spec:
            path = _verbs_dir()
            if path is not None and path not in ('/usr', '/usr/local'):
                config_args.append('--with-{0}={1}'.format(self.verbs, path))
            else:
                config_args.append('--with-{0}'.format(self.verbs))
        else:
            config_args.append('--without-{0}'.format(self.verbs))

        # Hwloc support
        if spec.satisfies('@1.5.2:'):
            config_args.append('--with-hwloc={0}'.format(spec['hwloc'].prefix))

        # Java support
        if spec.satisfies('@1.7.4:'):
            if '+java' in spec:
                config_args.extend([
                    '--enable-java',
                    '--enable-mpi-java',
                    '--with-jdk-dir={0}'.format(spec['jdk'].prefix)
                ])
            else:
                config_args.extend([
                    '--disable-java',
                    '--disable-mpi-java'
                ])

        # SQLite3 support
        if spec.satisfies('@1.7.3:1.999'):
            if '+sqlite3' in spec:
                config_args.append('--with-sqlite3')
            else:
                config_args.append('--without-sqlite3')

        # VampirTrace support
        if spec.satisfies('@1.3:1.999'):
            if '+vt' not in spec:
                config_args.append('--enable-contrib-no-build=vt')

        # Multithreading support
        if spec.satisfies('@1.5.4:'):
            if '+thread_multiple' in spec:
                config_args.append('--enable-mpi-thread-multiple')
            else:
                config_args.append('--disable-mpi-thread-multiple')

        return config_args

    @run_after('install')
    def filter_compilers(self):
        """Run after install to make the MPI compilers use the
           compilers that Spack built the package with.

           If this isn't done, they'll have CC, CXX and FC set
           to Spack's generic cc, c++ and f90.  We want them to
           be bound to whatever compiler they were built with.
        """
        kwargs = {'ignore_absent': True, 'backup': False, 'string': False}
        wrapper_basepath = join_path(self.prefix, 'share', 'openmpi')

        wrappers = [
            ('mpicc-vt-wrapper-data.txt', self.compiler.cc),
            ('mpicc-wrapper-data.txt', self.compiler.cc),
            ('ortecc-wrapper-data.txt', self.compiler.cc),
            ('shmemcc-wrapper-data.txt', self.compiler.cc),
            ('mpic++-vt-wrapper-data.txt', self.compiler.cxx),
            ('mpic++-wrapper-data.txt', self.compiler.cxx),
            ('ortec++-wrapper-data.txt', self.compiler.cxx),
            ('mpifort-vt-wrapper-data.txt', self.compiler.fc),
            ('mpifort-wrapper-data.txt', self.compiler.fc),
            ('shmemfort-wrapper-data.txt', self.compiler.fc),
            ('mpif90-vt-wrapper-data.txt', self.compiler.fc),
            ('mpif90-wrapper-data.txt', self.compiler.fc),
            ('mpif77-vt-wrapper-data.txt',  self.compiler.f77),
            ('mpif77-wrapper-data.txt',  self.compiler.f77)
        ]

        for wrapper_name, compiler in wrappers:
            wrapper = join_path(wrapper_basepath, wrapper_name)
            if not os.path.islink(wrapper):
                # Substitute Spack compile wrappers for the real
                # underlying compiler
                match = 'compiler=.*'
                substitute = 'compiler={compiler}'.format(compiler=compiler)
                filter_file(match, substitute, wrapper, **kwargs)
                # Remove this linking flag if present
                # (it turns RPATH into RUNPATH)
                filter_file('-Wl,--enable-new-dtags', '', wrapper, **kwargs)
<|endoftext|>