<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (c) 2008-2017 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Netius System. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2017 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import os
import ctypes
import socket
import threading

import netius

CALLABLE_WORK = 1

class Thread(threading.Thread):

    def __init__(self, identifier, owner = None, *args, **kwargs):
        threading.Thread.__init__(self, *args, **kwargs)
        self.identifier = identifier
        self.owner = owner
        self._run = False

    @classmethod
    def available(self):
        return True

    def stop(self):
        self._run = False

    def run(self):
        threading.Thread.run(self)
        self._run = True
        while self._run: self.tick()

    def tick(self):
        self.owner.condition.acquire()
        while not self.owner.peek() and self._run:
            self.owner.condition.wait()
        try:
            if not self._run: return
            work = self.owner.pop()
        finally:
            self.owner.condition.release()
        self.execute(work)

    def execute(self, work):
        type = work[0]
        if type == CALLABLE_WORK: work[1]()
        else: raise netius.NotImplemented("Cannot execute type '%d'" % type)

class ThreadPool(object):

    def __init__(self, base = Thread, count = 32):
        self.base = base
        self.count = count
        self.instances = []
        self.queue = []
        self.condition = threading.Condition()
        self._built = False

    def start(self):
        self.build()
        for instance in self.instances:
            instance.start()

    def stop(self, join = True):
        for instance in self.instances: instance.stop()
        self.condition.acquire()
        try: self.condition.notify_all()
        finally: self.condition.release()
        if not join: return
        for instance in self.instances: instance.join()

    def build(self):
        if self._built: return
        for index in range(self.count):
            instance = self.base(index, owner = self)
            self.instances.append(instance)
        self._built = True

    def peek(self):
        if not self.queue: return None
        return self.queue[0]

    def pop(self, lock = True):
        lock and self.condition.acquire()
        try: value = self.queue.pop(0)
        finally: lock and self.condition.release()
        return value

    def push(self, work, lock = True):
        lock and self.condition.acquire()
        try:
            value = self.queue.append(work)
            self.condition.notify()
        finally:
            lock and self.condition.release()
        return value

    def push_callable(self, callable):
        work = (CALLABLE_WORK, callable)
        self.push(work)

class EventPool(ThreadPool):

    def __init__(self, base = Thread, count = 32):
        ThreadPool.__init__(self, base = base, count = count)
        self.events = []
        self.event_lock = threading.RLock()
        self._eventfd = None

    def stop(self, join = True):
        ThreadPool.stop(self, join = join)
        if self._eventfd: self._eventfd.close()

    def push_event(self, event):
        self.event_lock.acquire()
        try: self.events.append(event)
        finally: self.event_lock.release()
        self.notify()

    def pop_event(self):
        self.event_lock.acquire()
        try: event = self.events.pop(0)
        finally: self.event_lock.release()
        return event

    def pop_all(self, denotify = False):
        self.event_lock.acquire()
        try:
            events = list(self.events)
            del self.events[:]
            if events and denotify: self.denotify()
        finally:
            self.event_lock.release()
        return events

    def notify(self):
        if not self._eventfd: return
        self._eventfd.notify()

    def denotify(self):
        if not self._eventfd: return
        self._eventfd.denotify()

    def eventfd(self):
        if self._eventfd: return self._eventfd
        if UnixEventFile.available():
            self._eventfd = UnixEventFile()
        elif PipeEventFile.available():
            self._eventfd = PipeEventFile()
        else:
            self._eventfd = SocketEventFile()
        return self._eventfd

class EventFile(object):

    def __init__(self, *args, **kwargs):
        self._rfileno = None
        self._wfileno = None
        self.closed = False

    def close(self):
        self.closed = True

    def fileno(self):
        return self.rfileno()

    def rfileno(self):
        return self._rfileno

    def wfileno(self):
        return self._wfileno

    def notify(self):
        raise netius.NotImplemented("Missing implementation")

    def denotify(self):
        raise netius.NotImplemented("Missing implementation")

class UnixEventFile(EventFile):

    _LIBC = None

    def __init__(self, *args, **kwargs):
        EventFile.__init__(self, *args, **kwargs)
        cls = self.__class__
        init_val = kwargs.get("init_val", 0)
        flags = kwargs.get("flags", 0)
        libc = cls.libc()
        self._rfileno = libc.eventfd(init_val, flags)
        self._wfileno = self._rfileno

    @classmethod
    def available(cls):
        if not os.name == "posix": return False
        return True if cls.libc() else False

    @classmethod
    def libc(cls):
        if cls._LIBC: return cls._LIBC
        try: cls._LIBC = ctypes.cdll.LoadLibrary("libc.so.6")
        except: return None
        return cls._LIBC

    def close(self):
        EventFile.close(self)
        os.close(self._wfileno)

    def notify(self):
        self._write(1)

    def denotify(self):
        self._read()

    def _read(self, length = 8):
        return os.read(self._rfileno, length)

    def _write(self, value):
        os.write(self._wfileno, ctypes.c_ulonglong(value))

class PipeEventFile(EventFile):

    def __init__(self, *args, **kwargs):
        import fcntl
        EventFile.__init__(self, *args, **kwargs)
        self._rfileno, self._wfileno = os.pipe()
        fcntl.fcntl(self._rfileno, fcntl.F_SETFL, os.O_NONBLOCK) #@UndefinedVariable
        fcntl.fcntl(self._wfileno, fcntl.F_SETFL, os.O_NONBLOCK) #@UndefinedVariable
        self._read_file = os.fdopen(self._rfileno, "rb", 0)
        self._write_file = os.fdopen(self._wfileno, "wb", 0)
        self._lock = threading.RLock()
        self._count = 0

    @classmethod
    def available(cls):
        if not os.name == "posix": return False
        if not hasattr(os, "pipe"): return False
        return True

    def close(self):
        EventFile.close(self)
        self._read_file.close()
        self._write_file.close()

    def notify(self):
        self._lock.acquire()
        try:
            self._write(b"1")
            self._count += 1
        finally:
            self._lock.release()

    def denotify(self):
        self._lock.acquire()
        try:
            data = self._read()
            self._count -= len(data)
        finally:
            self._lock.release()

    def _read(self, length = 4096):
        return self._read_file.read(length)

    def _write(self, data):
        self._write_file.write(data)

class SocketEventFile(EventFile):

    def __init__(self, *args, **kwargs):
        EventFile.__init__(self, *args, **kwargs)
        temp_socket = socket.socket()
        temp_socket.bind(("127.0.0.1", 0))
        temp_socket.listen(1)
        hostname = temp_socket.getsockname()
        self._read_socket = socket.create_connection(hostname)
        self._write_socket, _port = temp_socket.accept()
        self._rfileno = self._read_socket.fileno()
        self._wfileno = self._write_socket.fileno()
        self._lock = threading.RLock()
        self._count = 0
        temp_socket.close()

    def close(self):
        EventFile.close(self)
        self._read_socket.close()
        self._write_socket.close()

    def notify(self):
        self._lock.acquire()
        try:
            self._write(b"1")
            self._count += 1
        finally:
            self._lock.release()

    def denotify(self):
        self._lock.acquire()
        try:
            if self._count == 0: return
            data = self._read()
            self._count -= len(data)
        finally:
            self._lock.release()

    def _read(self, length = 4096):
        return self._read_socket.recv(length)

    def _write(self, data):
        self._write_socket.send(data)
<commit_msg>new validation<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (c) 2008-2017 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the Apache License as published by the Apache
# Foundation, either version 2.0 of the License, or (at your option) any
# later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# Apache License for more details.
#
# You should have received a copy of the Apache License along with
# Hive Netius System. If not, see <http://www.apache.org/licenses/>.

__author__ = "João Magalhães <joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2017 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "Apache License, Version 2.0"
""" The license for the module """

import os
import ctypes
import socket
import threading

import netius

CALLABLE_WORK = 1

class Thread(threading.Thread):

    def __init__(self, identifier, owner = None, *args, **kwargs):
        threading.Thread.__init__(self, *args, **kwargs)
        self.identifier = identifier
        self.owner = owner
        self._run = False

    @classmethod
    def available(self):
        return True

    def stop(self):
        self._run = False

    def run(self):
        threading.Thread.run(self)
        self._run = True
        while self._run: self.tick()

    def tick(self):
        self.owner.condition.acquire()
        while not self.owner.peek() and self._run:
            self.owner.condition.wait()
        try:
            if not self._run: return
            work = self.owner.pop()
        finally:
            self.owner.condition.release()
        self.execute(work)

    def execute(self, work):
        type = work[0]
        if type == CALLABLE_WORK: work[1]()
        else: raise netius.NotImplemented("Cannot execute type '%d'" % type)

class ThreadPool(object):

    def __init__(self, base = Thread, count = 32):
        self.base = base
        self.count = count
        self.instances = []
        self.queue = []
        self.condition = threading.Condition()
        self._built = False

    def start(self):
        self.build()
        for instance in self.instances:
            instance.start()

    def stop(self, join = True):
        for instance in self.instances: instance.stop()
        self.condition.acquire()
        try: self.condition.notify_all()
        finally: self.condition.release()
        if not join: return
        for instance in self.instances: instance.join()

    def build(self):
        if self._built: return
        for index in range(self.count):
            instance = self.base(index, owner = self)
            self.instances.append(instance)
        self._built = True

    def peek(self):
        if not self.queue: return None
        return self.queue[0]

    def pop(self, lock = True):
        lock and self.condition.acquire()
        try: value = self.queue.pop(0)
        finally: lock and self.condition.release()
        return value

    def push(self, work, lock = True):
        lock and self.condition.acquire()
        try:
            value = self.queue.append(work)
            self.condition.notify()
        finally:
            lock and self.condition.release()
        return value

    def push_callable(self, callable):
        work = (CALLABLE_WORK, callable)
        self.push(work)

class EventPool(ThreadPool):

    def __init__(self, base = Thread, count = 32):
        ThreadPool.__init__(self, base = base, count = count)
        self.events = []
        self.event_lock = threading.RLock()
        self._eventfd = None

    def stop(self, join = True):
        ThreadPool.stop(self, join = join)
        if not self._eventfd: return
        self._eventfd.close()
        self._eventfd = None

    def push_event(self, event):
        self.event_lock.acquire()
        try: self.events.append(event)
        finally: self.event_lock.release()
        self.notify()

    def pop_event(self):
        self.event_lock.acquire()
        try: event = self.events.pop(0)
        finally: self.event_lock.release()
        return event

    def pop_all(self, denotify = False):
        self.event_lock.acquire()
        try:
            events = list(self.events)
            del self.events[:]
            if events and denotify: self.denotify()
        finally:
            self.event_lock.release()
        return events

    def notify(self):
        if not self._eventfd: return
        self._eventfd.notify()

    def denotify(self):
        if not self._eventfd: return
        self._eventfd.denotify()

    def eventfd(self):
        if self._eventfd: return self._eventfd
        if UnixEventFile.available():
            self._eventfd = UnixEventFile()
        elif PipeEventFile.available():
            self._eventfd = PipeEventFile()
        else:
            self._eventfd = SocketEventFile()
        return self._eventfd

class EventFile(object):

    def __init__(self, *args, **kwargs):
        self._rfileno = None
        self._wfileno = None
        self.closed = False

    def close(self):
        self.closed = True

    def fileno(self):
        return self.rfileno()

    def rfileno(self):
        return self._rfileno

    def wfileno(self):
        return self._wfileno

    def notify(self):
        raise netius.NotImplemented("Missing implementation")

    def denotify(self):
        raise netius.NotImplemented("Missing implementation")

class UnixEventFile(EventFile):

    _LIBC = None

    def __init__(self, *args, **kwargs):
        EventFile.__init__(self, *args, **kwargs)
        cls = self.__class__
        init_val = kwargs.get("init_val", 0)
        flags = kwargs.get("flags", 0)
        libc = cls.libc()
        self._rfileno = libc.eventfd(init_val, flags)
        self._wfileno = self._rfileno

    @classmethod
    def available(cls):
        if not os.name == "posix": return False
        return True if cls.libc() else False

    @classmethod
    def libc(cls):
        if cls._LIBC: return cls._LIBC
        try: cls._LIBC = ctypes.cdll.LoadLibrary("libc.so.6")
        except: return None
        return cls._LIBC

    def close(self):
        EventFile.close(self)
        os.close(self._wfileno)

    def notify(self):
        self._write(1)

    def denotify(self):
        self._read()

    def _read(self, length = 8):
        return os.read(self._rfileno, length)

    def _write(self, value):
        os.write(self._wfileno, ctypes.c_ulonglong(value))

class PipeEventFile(EventFile):

    def __init__(self, *args, **kwargs):
        import fcntl
        EventFile.__init__(self, *args, **kwargs)
        self._rfileno, self._wfileno = os.pipe()
        fcntl.fcntl(self._rfileno, fcntl.F_SETFL, os.O_NONBLOCK) #@UndefinedVariable
        fcntl.fcntl(self._wfileno, fcntl.F_SETFL, os.O_NONBLOCK) #@UndefinedVariable
        self._read_file = os.fdopen(self._rfileno, "rb", 0)
        self._write_file = os.fdopen(self._wfileno, "wb", 0)
        self._lock = threading.RLock()
        self._count = 0

    @classmethod
    def available(cls):
        if not os.name == "posix": return False
        if not hasattr(os, "pipe"): return False
        return True

    def close(self):
        EventFile.close(self)
        self._read_file.close()
        self._write_file.close()

    def notify(self):
        self._lock.acquire()
        try:
            self._write(b"1")
            self._count += 1
        finally:
            self._lock.release()

    def denotify(self):
        self._lock.acquire()
        try:
            data = self._read()
            self._count -= len(data)
        finally:
            self._lock.release()

    def _read(self, length = 4096):
        return self._read_file.read(length)

    def _write(self, data):
        self._write_file.write(data)

class SocketEventFile(EventFile):

    def __init__(self, *args, **kwargs):
        EventFile.__init__(self, *args, **kwargs)
        temp_socket = socket.socket()
        temp_socket.bind(("127.0.0.1", 0))
        temp_socket.listen(1)
        hostname = temp_socket.getsockname()
        self._read_socket = socket.create_connection(hostname)
        self._write_socket, _port = temp_socket.accept()
        self._rfileno = self._read_socket.fileno()
        self._wfileno = self._write_socket.fileno()
        self._lock = threading.RLock()
        self._count = 0
        temp_socket.close()

    def close(self):
        EventFile.close(self)
        self._read_socket.close()
        self._write_socket.close()

    def notify(self):
        self._lock.acquire()
        try:
            self._write(b"1")
            self._count += 1
        finally:
            self._lock.release()

    def denotify(self):
        self._lock.acquire()
        try:
            if self._count == 0: return
            data = self._read()
            self._count -= len(data)
        finally:
            self._lock.release()

    def _read(self, length = 4096):
        return self._read_socket.recv(length)

    def _write(self, data):
        self._write_socket.send(data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Bibclassify keyword extractor command line entry point.
"""

__revision__ = "$Id$"

import getopt
import os
import sys
import time

try:
    from bibclassifylib import get_regular_expressions, \
        get_keywords_from_text, check_ontology
    from bibclassify_text_extractor import text_lines_from_local_file, \
        text_lines_from_url, is_pdf
except ImportError, err:
    print >> sys.stderr, "Error: %s" % err
    sys.exit(1)

_OPTIONS = {}

def display_help():
    """Prints the help message for this module."""
    print >> sys.stdout, """Usage: bibclassify [OPTION]... [FILE/URL]...
  or:  bibclassify [OPTION]... [DIRECTORY]...
Searches keywords in FILEs and/or files in DIRECTORY(ies). If a directory is
specified, BibClassify will generate keywords for all PDF documents contained
in the directory.

  -h, --help                display this help and exit
  -V, --version             output version information and exit
  -v, --verbose LEVEL       sets the verbose to LEVEL (=0)
  -k, --ontology FILE       sets the FILE to read the ontology from
  -o, --output-mode TYPE    changes the output format to TYPE (text, marcxml or
                              html) (=text)
  -s, --spires              outputs keywords in the SPIRES format
  -n, --keywords-number INT sets the number of keywords displayed (=20), use 0
                              to set no limit
  -m, --matching-mode TYPE  changes the search mode to TYPE (full or partial)
                              (=full)
  --detect-author-keywords  detect keywords that are explicitely written in the
                              document
  --rebuild-cache           ignores the existing cache and regenerates it
  --no-cache                don't cache the ontology

Backward compatibility (using these options is discouraged):
  -q                        equivalent to -s
  -f FILE URL               sets the file to read the keywords from

Example:
    python bibclassifycli.py -k etc/HEP.rdf http://arxiv.org/pdf/0808.1825
    python bibclassifycli.py -k etc/HEP.rdf article.pdf
    python bibclassifycli.py -k etc/HEP.rdf directory/"""
    sys.exit(0)

def main():
    """Main function """
    read_options(sys.argv[1:])

    # Ontology check
    if _OPTIONS["check_ontology"]:
        print >> sys.stdout, ("Checking ontology file %s" %
            _OPTIONS["ontology_file"])
        check_ontology(_OPTIONS["ontology_file"])
    # End of ontology check.

    # Initialize cache
    get_regular_expressions(_OPTIONS["ontology_file"],
                            _OPTIONS["rebuild_cache"],
                            _OPTIONS["no_cache"])

    sources = {}
    for entry in _OPTIONS["text_files"]:
        text_lines = None
        if os.path.isdir(entry):
            for filename in os.listdir(entry):
                if (os.path.isfile(entry + filename) and
                    is_pdf(entry + filename)):
                    text_lines = text_lines_from_local_file(entry + filename)
                    sources[filename] = text_lines
        elif os.path.isfile(entry):
            text_lines = text_lines_from_local_file(entry)
            sources[os.path.basename(entry)] = text_lines
        else:
            # Treat as a URL.
            text_lines = text_lines_from_url(entry)
            sources[entry.split("/")[-1]] = text_lines

    # For each identified source, check the keywords and output them.
    for source, text_lines in sources.iteritems():
        if _OPTIONS["output_mode"] == "text":
            print >> sys.stdout, source
        print >> sys.stdout, get_keywords_from_text(text_lines,
            output_mode=_OPTIONS["output_mode"],
            output_limit=_OPTIONS["output_limit"],
            spires=_OPTIONS["spires"],
            match_mode=_OPTIONS["match_mode"],
            with_author_keywords=_OPTIONS["with_author_keywords"])

def read_options(options_string):
    """Reads the options, test if the specified values are consistent and
    populates the options dictionary."""
    global _OPTIONS
    _OPTIONS = {}
    _OPTIONS["spires"] = False
    _OPTIONS["output_limit"] = 20
    _OPTIONS["text_files"] = []
    _OPTIONS["ontology_file"] = ""
    _OPTIONS["output_mode"] = "text"
    _OPTIONS["verbose"] = 0
    _OPTIONS["match_mode"] = "full"
    _OPTIONS["output_prefix"] = None
    _OPTIONS["rebuild_cache"] = False
    _OPTIONS["no_cache"] = False
    _OPTIONS["check_ontology"] = False
    _OPTIONS["with_author_keywords"] = False

    output_modes = ("html", "text", "marcxml")
    modes = ("full", "partial")

    try:
        long_flags = ["ontology=", "output-mode=", "verbose=", "spires",
                      "keywords-number=", "matching-mode=", "help", "version",
                      "file", "rebuild-cache", "no-limit", "no-cache",
                      "check-ontology", "detect-author-keywords"]
        short_flags = "f:k:o:n:m:v:sqhV"
        opts, args = getopt.gnu_getopt(options_string, short_flags, long_flags)
    except getopt.GetoptError, err1:
        print >> sys.stderr, "Options problem: %s" % err1
        usage()

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            display_help()
        elif opt in ("-V", "--version"):
            try:
                from invenio.config import CFG_VERSION
                print >> sys.stdout, ("CDS Invenio/%s bibclassify/%s" %
                    (CFG_VERSION, CFG_VERSION))
            except ImportError:
                print >> sys.stdout, "CDS Invenio bibclassify/standalone"
            sys.exit(1)
        elif opt in ("-v", "--verbose"):
            _OPTIONS["verbose"] = arg
        elif opt in ("-k", "--ontology"):
            _OPTIONS["ontology_file"] = arg
        elif opt in ("-o", "--output-mode"):
            _OPTIONS["output_mode"] = arg.lower()
        elif opt in ("-m", "--matching-mode"):
            _OPTIONS["match_mode"] = arg.lower()
        # -q for backward compatibility
        elif opt in ("-s", "--spires", "-q"):
            _OPTIONS["spires"] = True
        elif opt in ("-n", "--nkeywords"):
            _OPTIONS["output_limit"] = arg
        elif opt == "--rebuild-cache":
            _OPTIONS["rebuild_cache"] = True
        elif opt == "--no-cache":
            _OPTIONS["no_cache"] = True
        elif opt == "--write-to-file":
            _OPTIONS["output_prefix"] = arg
        # -f for compatibility reasons
        elif opt in ("-f", "--file"):
            _OPTIONS["text_files"].append(arg)
        elif opt == "--check-ontology":
            _OPTIONS["check_ontology"] = True
        elif opt == "--detect-author-keywords":
            _OPTIONS["with_author_keywords"] = True

    if not opts and not args:
        display_help()

    _OPTIONS["text_files"] += args

    # Test if the options are consistent.
    if not args:
        if not _OPTIONS["check_ontology"] and not _OPTIONS["text_files"]:
            print >> sys.stderr, "ERROR: please specify a file or directory."
            usage()
    if not _OPTIONS["ontology_file"]:
        print >> sys.stderr, "ERROR: please specify an ontology file (-k)."
        usage()
    if _OPTIONS["output_mode"] not in output_modes:
        print >> sys.stderr, ("ERROR: output (-o) should be TEXT, MARCXML or "
            "HTML.")
        usage()
    if _OPTIONS["match_mode"] not in modes:
        print >> sys.stderr, "ERROR: mode (-m) should be FULL or PARTIAL."
        usage()
    try:
        _OPTIONS["output_limit"] = int(_OPTIONS["output_limit"])
        if _OPTIONS["output_limit"] < 0:
            print >> sys.stderr, ("ERROR: output limit must be a positive "
                "integer.")
    except ValueError:
        print >> sys.stderr, ("ERROR: output limit must be a positive "
            "integer.")
        usage()

def usage():
    """Displays usage (single line) and exit."""
    # TODO: write usage
    display_help()
    sys.exit(1)

def version():
    """Display BibClassify version and exit."""
    # TODO
    display_help()
    sys.exit(0)

def write_message(msg, stream=sys.stdout, verbose=1):
    """Write message and flush output stream (may be sys.stdout or sys.stderr).
    Useful for debugging stuff. Copied from bibtask.py."""
    if msg and _OPTIONS["verbose"] >= verbose:
        if stream == sys.stdout or stream == sys.stderr:
            stream.write(time.strftime("%Y-%m-%d %H:%M:%S --> ",
                                       time.localtime()))
            try:
                stream.write("%s\n" % msg)
            except UnicodeEncodeError:
                stream.write("%s\n" % msg.encode('ascii', 'backslashreplace'))
                stream.flush()
        else:
            sys.stderr.write("Unknown stream %s. [must be sys.stdout or "
                             "sys.stderr]\n" % stream)

if __name__ == '__main__':
    main()

<commit_msg>Added the --check-ontology option to the help message.<commit_after># -*- coding: utf-8 -*-
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Bibclassify keyword extractor command line entry point.
"""

__revision__ = "$Id$"

import getopt
import os
import sys
import time

try:
    from bibclassifylib import get_regular_expressions, \
        get_keywords_from_text, check_ontology
    from bibclassify_text_extractor import text_lines_from_local_file, \
        text_lines_from_url, is_pdf
except ImportError, err:
    print >> sys.stderr, "Error: %s" % err
    sys.exit(1)

_OPTIONS = {}

def display_help():
    """Prints the help message for this module."""
    print >> sys.stdout, """Usage: bibclassify [OPTION]... [FILE/URL]...
  or:  bibclassify [OPTION]... [DIRECTORY]...
Searches keywords in FILEs and/or files in DIRECTORY(ies). If a directory is
specified, BibClassify will generate keywords for all PDF documents contained
in the directory.

  -h, --help                display this help and exit
  -V, --version             output version information and exit
  -v, --verbose LEVEL       sets the verbose to LEVEL (=0)
  -k, --ontology FILE       sets the FILE to read the ontology from
  -o, --output-mode TYPE    changes the output format to TYPE (text, marcxml or
                              html) (=text)
  -s, --spires              outputs keywords in the SPIRES format
  -n, --keywords-number INT sets the number of keywords displayed (=20), use 0
                              to set no limit
  -m, --matching-mode TYPE  changes the search mode to TYPE (full or partial)
                              (=full)
  --detect-author-keywords  detect keywords that are explicitely written in the
                              document
  --check-ontology          checks the ontology and reports warnings and errors
  --rebuild-cache           ignores the existing cache and regenerates it
  --no-cache                don't cache the ontology

Backward compatibility (using these options is discouraged):
  -q                        equivalent to -s
  -f FILE URL               sets the file to read the keywords from

Example:
    python bibclassifycli.py -k etc/HEP.rdf http://arxiv.org/pdf/0808.1825
    python bibclassifycli.py -k etc/HEP.rdf article.pdf
    python bibclassifycli.py -k etc/HEP.rdf directory/"""
    sys.exit(0)

def main():
    """Main function """
    read_options(sys.argv[1:])

    # Ontology check
    if _OPTIONS["check_ontology"]:
        print >> sys.stdout, ("Checking ontology file %s" %
            _OPTIONS["ontology_file"])
        check_ontology(_OPTIONS["ontology_file"])
    # End of ontology check.

    # Initialize cache
    get_regular_expressions(_OPTIONS["ontology_file"],
                            _OPTIONS["rebuild_cache"],
                            _OPTIONS["no_cache"])

    sources = {}
    for entry in _OPTIONS["text_files"]:
        text_lines = None
        if os.path.isdir(entry):
            for filename in os.listdir(entry):
                if (os.path.isfile(entry + filename) and
                    is_pdf(entry + filename)):
                    text_lines = text_lines_from_local_file(entry + filename)
                    sources[filename] = text_lines
        elif os.path.isfile(entry):
            text_lines = text_lines_from_local_file(entry)
            sources[os.path.basename(entry)] = text_lines
        else:
            # Treat as a URL.
            text_lines = text_lines_from_url(entry)
            sources[entry.split("/")[-1]] = text_lines

    # For each identified source, check the keywords and output them.
    for source, text_lines in sources.iteritems():
        if _OPTIONS["output_mode"] == "text":
            print >> sys.stdout, source
        print >> sys.stdout, get_keywords_from_text(text_lines,
            output_mode=_OPTIONS["output_mode"],
            output_limit=_OPTIONS["output_limit"],
            spires=_OPTIONS["spires"],
            match_mode=_OPTIONS["match_mode"],
            with_author_keywords=_OPTIONS["with_author_keywords"])

def read_options(options_string):
    """Reads the options, test if the specified values are consistent and
    populates the options dictionary."""
    global _OPTIONS
    _OPTIONS = {}
    _OPTIONS["spires"] = False
    _OPTIONS["output_limit"] = 20
    _OPTIONS["text_files"] = []
    _OPTIONS["ontology_file"] = ""
    _OPTIONS["output_mode"] = "text"
    _OPTIONS["verbose"] = 0
    _OPTIONS["match_mode"] = "full"
    _OPTIONS["output_prefix"] = None
    _OPTIONS["rebuild_cache"] = False
    _OPTIONS["no_cache"] = False
    _OPTIONS["check_ontology"] = False
    _OPTIONS["with_author_keywords"] = False

    output_modes = ("html", "text", "marcxml")
    modes = ("full", "partial")

    try:
        long_flags = ["ontology=", "output-mode=", "verbose=", "spires",
                      "keywords-number=", "matching-mode=", "help", "version",
                      "file", "rebuild-cache", "no-limit", "no-cache",
                      "check-ontology", "detect-author-keywords"]
        short_flags = "f:k:o:n:m:v:sqhV"
        opts, args = getopt.gnu_getopt(options_string, short_flags, long_flags)
    except getopt.GetoptError, err1:
        print >> sys.stderr, "Options problem: %s" % err1
        usage()

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            display_help()
        elif opt in ("-V", "--version"):
            try:
                from invenio.config import CFG_VERSION
                print >> sys.stdout, ("CDS Invenio/%s bibclassify/%s" %
                    (CFG_VERSION, CFG_VERSION))
            except ImportError:
                print >> sys.stdout, "CDS Invenio bibclassify/standalone"
            sys.exit(1)
        elif opt in ("-v", "--verbose"):
            _OPTIONS["verbose"] = arg
        elif opt in ("-k", "--ontology"):
            _OPTIONS["ontology_file"] = arg
        elif opt in ("-o", "--output-mode"):
            _OPTIONS["output_mode"] = arg.lower()
        elif opt in ("-m", "--matching-mode"):
            _OPTIONS["match_mode"] = arg.lower()
        # -q for backward compatibility
        elif opt in ("-s", "--spires", "-q"):
            _OPTIONS["spires"] = True
        elif opt in ("-n", "--nkeywords"):
            _OPTIONS["output_limit"] = arg
        elif opt == "--rebuild-cache":
            _OPTIONS["rebuild_cache"] = True
        elif opt == "--no-cache":
            _OPTIONS["no_cache"] = True
        elif opt == "--write-to-file":
            _OPTIONS["output_prefix"] = arg
        # -f for compatibility reasons
        elif opt in ("-f", "--file"):
            _OPTIONS["text_files"].append(arg)
        elif opt == "--check-ontology":
            _OPTIONS["check_ontology"] = True
        elif opt == "--detect-author-keywords":
            _OPTIONS["with_author_keywords"] = True

    if not opts and not args:
        display_help()

    _OPTIONS["text_files"] += args

    # Test if the options are consistent.
    if not args:
        if not _OPTIONS["check_ontology"] and not _OPTIONS["text_files"]:
            print >> sys.stderr, "ERROR: please specify a file or directory."
            usage()
    if not _OPTIONS["ontology_file"]:
        print >> sys.stderr, "ERROR: please specify an ontology file (-k)."
        usage()
    if _OPTIONS["output_mode"] not in output_modes:
        print >> sys.stderr, ("ERROR: output (-o) should be TEXT, MARCXML or "
            "HTML.")
        usage()
    if _OPTIONS["match_mode"] not in modes:
        print >> sys.stderr, "ERROR: mode (-m) should be FULL or PARTIAL."
        usage()
    try:
        _OPTIONS["output_limit"] = int(_OPTIONS["output_limit"])
        if _OPTIONS["output_limit"] < 0:
            print >> sys.stderr, ("ERROR: output limit must be a positive "
                "integer.")
    except ValueError:
        print >> sys.stderr, ("ERROR: output limit must be a positive "
            "integer.")
        usage()

def usage():
    """Displays usage (single line) and exit."""
    # TODO: write usage
    display_help()
    sys.exit(1)

def version():
    """Display BibClassify version and exit."""
    # TODO
    display_help()
    sys.exit(0)

def write_message(msg, stream=sys.stdout, verbose=1):
    """Write message and flush output stream (may be sys.stdout or sys.stderr).
    Useful for debugging stuff. Copied from bibtask.py."""
    if msg and _OPTIONS["verbose"] >= verbose:
        if stream == sys.stdout or stream == sys.stderr:
            stream.write(time.strftime("%Y-%m-%d %H:%M:%S --> ",
                                       time.localtime()))
            try:
                stream.write("%s\n" % msg)
            except UnicodeEncodeError:
                stream.write("%s\n" % msg.encode('ascii', 'backslashreplace'))
                stream.flush()
        else:
            sys.stderr.write("Unknown stream %s. [must be sys.stdout or "
                             "sys.stderr]\n" % stream)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import argparse
import logging
import os

from ceph_deploy import hosts
from ceph_deploy.cliutil import priority
from ceph_deploy.lib.remoto import process, rsync

LOG = logging.getLogger(__name__)

def install(args):
    # XXX This whole dance is because --stable is getting deprecated
    if args.stable is not None:
        LOG.warning('the --stable flag is deprecated, use --release instead')
        args.release = args.stable
    if args.version_kind == 'stable':
        version = args.release
    else:
        version = getattr(args, args.version_kind)
    # XXX Tango ends here.

    version_str = args.version_kind

    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
    )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)
        distro = hosts.get(hostname, username=args.username)
        LOG.info(
            'Distro info: %s %s %s',
            distro.name,
            distro.release,
            distro.codename
        )
        rlogger = logging.getLogger(hostname)
        rlogger.info('installing ceph on %s' % hostname)

        cd_conf = getattr(args, 'cd_conf', None)

        # custom repo arguments
        repo_url = os.environ.get('CEPH_DEPLOY_REPO_URL') or args.repo_url
        gpg_url = os.environ.get('CEPH_DEPLOY_GPG_URL') or args.gpg_url
        gpg_fallback = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'

        if gpg_url is None and repo_url:
            LOG.warning('--gpg-url was not used, will fallback')
            LOG.warning('using GPG fallback: %s', gpg_fallback)
            gpg_url = gpg_fallback

        if args.local_mirror:
            rsync(hostname, args.local_mirror, '/opt/ceph-deploy/repo', distro.conn.logger, sudo=True)
            repo_url = 'file:///opt/ceph-deploy/repo'
            gpg_url = 'file:///opt/ceph-deploy/repo/release.asc'

        if repo_url:  # triggers using a custom repository
            # the user used a custom repo url, this should override anything
            # we can detect from the configuration, so warn about it
            if cd_conf:
                if cd_conf.get_default_repo():
                    rlogger.warning('a default repo was found but it was \
                        overridden on the CLI')
                if args.release in cd_conf.get_repos():
                    rlogger.warning('a custom repo was found but it was \
                        overridden on the CLI')

            rlogger.info('using custom repository location: %s', repo_url)
            distro.mirror_install(
                distro,
                repo_url,
                gpg_url,
                args.adjust_repos
            )

        # Detect and install custom repos here if needed
        elif should_use_custom_repo(args, cd_conf, repo_url):
            LOG.info('detected valid custom repositories from config file')
            custom_repo(distro, args, cd_conf, rlogger)

        else:  # otherwise a normal installation
            distro.install(
                distro,
                args.version_kind,
                version,
                args.adjust_repos
            )

        # Check the ceph version we just installed
        hosts.common.ceph_version(distro.conn)
        distro.conn.exit()

def should_use_custom_repo(args, cd_conf, repo_url):
    """
    A boolean to determine the logic needed to proceed with a custom repo
    installation instead of cramming everything nect to the logic operator.
    """
    if repo_url:
        # repo_url signals a CLI override, return False immediately
        return False
    if cd_conf:
        if cd_conf.has_repos:
            has_valid_release = args.release in cd_conf.get_repos()
            has_default_repo = cd_conf.get_default_repo()
            if has_valid_release or has_default_repo:
                return True
    return False

def custom_repo(distro, args, cd_conf, rlogger):
    """
    A custom repo install helper that will go through config checks to retrieve
    repos (and any extra repos defined) and install those

    ``cd_conf`` is the object built from argparse that holds the flags and
    information needed to determine what metadata from the configuration to be
    used.
    """
    default_repo = cd_conf.get_default_repo()
    if args.release in cd_conf.get_repos():
        LOG.info('will use repository from conf: %s' % args.release)
        default_repo = args.release
    elif default_repo:
        LOG.info('will use default repository: %s' % default_repo)

    # At this point we know there is a cd_conf and that it has custom
    # repos make sure we were able to detect and actual repo
    if not default_repo:
        LOG.warning('a ceph-deploy config was found with repos \
            but could not default to one')
    else:
        options = dict(cd_conf.items(default_repo))
        options['install_ceph'] = True
        extra_repos = cd_conf.get_list(default_repo, 'extra-repos')
        rlogger.info('adding custom repository file')
        try:
            distro.repo_install(
                distro,
                default_repo,
                options.pop('baseurl'),
                options.pop('gpgkey'),
                **options
            )
        except KeyError as err:
            raise RuntimeError('missing required key: %s in config section: %s' % (err, default_repo))

        for xrepo in extra_repos:
            rlogger.info('adding extra repo file: %s.repo' % xrepo)
            options = dict(cd_conf.items(xrepo))
            try:
                distro.repo_install(
                    distro,
                    xrepo,
                    options.pop('baseurl'),
                    options.pop('gpgkey'),
                    **options
                )
            except KeyError as err:
                raise RuntimeError('missing required key: %s in config section: %s' % (err, xrepo))

def uninstall(args):
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        distro = hosts.get(hostname, username=args.username)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('uninstalling ceph on %s' % hostname)
        distro.uninstall(distro.conn)
        distro.conn.exit()

def purge(args):
    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        distro = hosts.get(hostname, username=args.username)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('purging host ... %s' % hostname)
        distro.uninstall(distro.conn, purge=True)
        distro.conn.exit()

def purgedata(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    installed_hosts = []
    for hostname in args.host:
        distro = hosts.get(hostname, username=args.username)
        ceph_is_installed = distro.conn.remote_module.which('ceph')
        if ceph_is_installed:
            installed_hosts.append(hostname)
        distro.conn.exit()

    if installed_hosts:
        LOG.error("ceph is still installed on: %s", installed_hosts)
        raise RuntimeError("refusing to purge data while ceph is still installed")

    for hostname in args.host:
        distro = hosts.get(hostname, username=args.username)
        LOG.info(
            'Distro info: %s %s %s',
            distro.name,
            distro.release,
            distro.codename
        )

        rlogger = logging.getLogger(hostname)
        rlogger.info('purging data on %s' % hostname)

        # Try to remove the contents of /var/lib/ceph first, don't worry
        # about errors here, we deal with them later on
        process.check(
            distro.conn,
            [
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ]
        )

        # If we failed in the previous call, then we probably have OSDs
        # still mounted, so we unmount them here
        if distro.conn.remote_module.path_exists('/var/lib/ceph'):
            rlogger.warning(
                'OSDs may still be mounted, trying to unmount them'
            )
            process.run(
                distro.conn,
                [
                    'find', '/var/lib/ceph',
                    '-mindepth', '1',
                    '-maxdepth', '2',
                    '-type', 'd',
                    '-exec', 'umount', '{}', ';',
                ]
            )

            # And now we try again to remove the contents, since OSDs should be
            # unmounted, but this time we do check for errors
            process.run(
                distro.conn,
                [
                    'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ]
            )

        process.run(
            distro.conn,
            [
                'rm', '-rf', '--one-file-system', '--', '/etc/ceph/',
            ]
        )

        distro.conn.exit()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        if self.dest == 'release':
            self.dest = 'stable'
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    # XXX deprecated in favor of release
    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        metavar='CODENAME',
        help='[DEPRECATED] install a release known as CODENAME\
                (done by default) (default: %(default)s)',
    )

    version.add_argument(
        '--release',
        nargs='?',
        action=StoreVersion,
        metavar='CODENAME',
        help='install a release known as CODENAME\
                (done by default) (default: %(default)s)',
    )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
    )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch\
                or tag (default: %(default)s)',
    )

    version.add_argument(
        '--adjust-repos',
        dest='adjust_repos',
        action='store_true',
        help='install packages modifying source repos',
    )

    version.add_argument(
        '--no-adjust-repos',
        dest='adjust_repos',
        action='store_false',
        help='install packages without modifying source repos',
    )

    version.set_defaults(
        func=install,
        stable=None,  # XXX deprecated in favor of release
        release='emperor',
        dev='master',
        version_kind='stable',
        adjust_repos=True,
    )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
    )

    parser.add_argument(
        '--local-mirror',
        nargs='?',
        const='PATH',
        default=None,
        help='Fetch packages and push them to hosts for a local repo mirror',
    )

    parser.add_argument(
        '--repo-url',
        nargs='?',
        dest='repo_url',
        help='specify a repo URL that mirrors/contains ceph packages',
    )

    parser.add_argument(
        '--gpg-url',
        nargs='?',
        dest='gpg_url',
        help='specify a GPG key URL to be used with custom repos\
                (defaults to ceph.com)'
    )

    parser.set_defaults(
        func=install,
    )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purgedata,
        )
<commit_msg>when purging/uninstalling mention note about librados and librbd<commit_after>import argparse
import logging
import os

from ceph_deploy import hosts
from ceph_deploy.cliutil import priority
from ceph_deploy.lib.remoto import process, rsync

LOG = logging.getLogger(__name__)

def install(args):
    # XXX This whole dance is because --stable is getting deprecated
    if args.stable is not None:
        LOG.warning('the --stable flag is deprecated, use --release instead')
        args.release = args.stable
    if args.version_kind == 'stable':
        version = args.release
    else:
        version = getattr(args, args.version_kind)
    # XXX Tango ends here.

    version_str = args.version_kind

    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
    )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)
        distro = hosts.get(hostname, username=args.username)
        LOG.info(
            'Distro info: %s %s %s',
            distro.name,
            distro.release,
            distro.codename
        )
        rlogger = logging.getLogger(hostname)
        rlogger.info('installing ceph on %s' % hostname)

        cd_conf = getattr(args, 'cd_conf', None)

        # custom repo arguments
        repo_url = os.environ.get('CEPH_DEPLOY_REPO_URL') or args.repo_url
        gpg_url = os.environ.get('CEPH_DEPLOY_GPG_URL') or args.gpg_url
        gpg_fallback = 'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/release.asc'

        if gpg_url is None and repo_url:
            LOG.warning('--gpg-url was not used, will fallback')
            LOG.warning('using GPG fallback: %s', gpg_fallback)
            gpg_url = gpg_fallback

        if args.local_mirror:
            rsync(hostname, args.local_mirror, '/opt/ceph-deploy/repo', distro.conn.logger, sudo=True)
            repo_url = 'file:///opt/ceph-deploy/repo'
            gpg_url = 'file:///opt/ceph-deploy/repo/release.asc'

        if repo_url:  # triggers using a custom repository
            # the user used a custom repo url, this should override anything
            # we can detect from the configuration, so warn about it
            if cd_conf:
                if cd_conf.get_default_repo():
                    rlogger.warning('a default repo was found but it was \
                        overridden on the CLI')
                if args.release in cd_conf.get_repos():
                    rlogger.warning('a custom repo was found but it was \
                        overridden on the CLI')

            rlogger.info('using custom repository location: %s', repo_url)
            distro.mirror_install(
                distro,
                repo_url,
                gpg_url,
                args.adjust_repos
            )

        # Detect and install custom repos here if needed
        elif should_use_custom_repo(args, cd_conf, repo_url):
            LOG.info('detected valid custom repositories from config file')
            custom_repo(distro, args, cd_conf, rlogger)

        else:  # otherwise a normal installation
            distro.install(
                distro,
                args.version_kind,
                version,
                args.adjust_repos
            )

        # Check the ceph version we just installed
        hosts.common.ceph_version(distro.conn)
        distro.conn.exit()

def should_use_custom_repo(args, cd_conf, repo_url):
    """
    A boolean to determine the logic needed to proceed with a custom repo
    installation instead of cramming everything nect to the logic operator.
    """
    if repo_url:
        # repo_url signals a CLI override, return False immediately
        return False
    if cd_conf:
        if cd_conf.has_repos:
            has_valid_release = args.release in cd_conf.get_repos()
            has_default_repo = cd_conf.get_default_repo()
            if has_valid_release or has_default_repo:
                return True
    return False

def custom_repo(distro, args, cd_conf, rlogger):
    """
    A custom repo install helper that will go through config checks to retrieve
    repos (and any extra repos defined) and install those

    ``cd_conf`` is the object built from argparse that holds the flags and
    information needed to determine what metadata from the configuration to be
    used.
    """
    default_repo = cd_conf.get_default_repo()
    if args.release in cd_conf.get_repos():
        LOG.info('will use repository from conf: %s' % args.release)
        default_repo = args.release
    elif default_repo:
        LOG.info('will use default repository: %s' % default_repo)

    # At this point we know there is a cd_conf and that it has custom
    # repos make sure we were able to detect and actual repo
    if not default_repo:
        LOG.warning('a ceph-deploy config was found with repos \
            but could not default to one')
    else:
        options = dict(cd_conf.items(default_repo))
        options['install_ceph'] = True
        extra_repos = cd_conf.get_list(default_repo, 'extra-repos')
        rlogger.info('adding custom repository file')
        try:
            distro.repo_install(
                distro,
                default_repo,
                options.pop('baseurl'),
                options.pop('gpgkey'),
                **options
            )
        except KeyError as err:
            raise RuntimeError('missing required key: %s in config section: %s' % (err, default_repo))

        for xrepo in extra_repos:
            rlogger.info('adding extra repo file: %s.repo' % xrepo)
            options = dict(cd_conf.items(xrepo))
            try:
                distro.repo_install(
                    distro,
                    xrepo,
                    options.pop('baseurl'),
                    options.pop('gpgkey'),
                    **options
                )
            except KeyError as err:
                raise RuntimeError('missing required key: %s in config section: %s' % (err, xrepo))

def uninstall(args):
    LOG.info('note that some dependencies *will not* be removed because they can cause issues with qemu-kvm')
    LOG.info('like: librbd1 and librados2')
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        distro = hosts.get(hostname, username=args.username)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('uninstalling ceph on %s' % hostname)
        distro.uninstall(distro.conn)
        distro.conn.exit()

def purge(args):
    LOG.info('note that some dependencies *will not* be removed because they can cause issues with qemu-kvm')
    LOG.info('like: librbd1 and librados2')

    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        distro = hosts.get(hostname, username=args.username)
        LOG.info('Distro info: %s %s %s', distro.name, distro.release, distro.codename)
        rlogger = logging.getLogger(hostname)
        rlogger.info('purging host ... %s' % hostname)
        distro.uninstall(distro.conn, purge=True)
        distro.conn.exit()

def purgedata(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    installed_hosts = []
    for hostname in args.host:
        distro = hosts.get(hostname, username=args.username)
        ceph_is_installed = distro.conn.remote_module.which('ceph')
        if ceph_is_installed:
            installed_hosts.append(hostname)
        distro.conn.exit()

    if installed_hosts:
        LOG.error("ceph is still installed on: %s", installed_hosts)
        raise RuntimeError("refusing to purge data while ceph is still installed")

    for hostname in args.host:
        distro = hosts.get(hostname, username=args.username)
        LOG.info(
            'Distro info: %s %s %s',
            distro.name,
            distro.release,
            distro.codename
        )

        rlogger = logging.getLogger(hostname)
        rlogger.info('purging data on %s' % hostname)

        # Try to remove the contents of /var/lib/ceph first, don't worry
        # about errors here, we deal with them later on
        process.check(
            distro.conn,
            [
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ]
        )

        # If we failed in the previous call, then we probably have OSDs
        # still mounted, so we unmount them here
        if distro.conn.remote_module.path_exists('/var/lib/ceph'):
            rlogger.warning(
                'OSDs may still be mounted, trying to unmount them'
            )
            process.run(
                distro.conn,
                [
                    'find', '/var/lib/ceph',
                    '-mindepth', '1',
                    '-maxdepth', '2',
                    '-type', 'd',
                    '-exec', 'umount', '{}', ';',
                ]
            )

            # And now we try again to remove the contents, since OSDs should be
            # unmounted, but this time we do check for errors
            process.run(
                distro.conn,
                [
                    'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ]
            )

        process.run(
            distro.conn,
            [
                'rm', '-rf', '--one-file-system', '--', '/etc/ceph/',
            ]
        )

        distro.conn.exit()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        if self.dest == 'release':
            self.dest = 'stable'
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    # XXX deprecated in favor of release
    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        metavar='CODENAME',
        help='[DEPRECATED] install a release known as CODENAME\
                (done by default) (default: %(default)s)',
    )

    version.add_argument(
        '--release',
        nargs='?',
        action=StoreVersion,
        metavar='CODENAME',
        help='install a release known as CODENAME\
                (done by default) (default: %(default)s)',
    )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
    )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch\
                or tag (default: %(default)s)',
    )

    version.add_argument(
        '--adjust-repos',
        dest='adjust_repos',
        action='store_true',
        help='install packages modifying source repos',
    )

    version.add_argument(
        '--no-adjust-repos',
        dest='adjust_repos',
        action='store_false',
        help='install packages without modifying source repos',
    )

    version.set_defaults(
        func=install,
        stable=None,  # XXX deprecated in favor of release
        release='emperor',
        dev='master',
        version_kind='stable',
        adjust_repos=True,
    )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
    )

    parser.add_argument(
        '--local-mirror',
        nargs='?',
        const='PATH',
        default=None,
        help='Fetch packages and push them to hosts for a local repo mirror',
    )

    parser.add_argument(
        '--repo-url',
        nargs='?',
        dest='repo_url',
        help='specify a repo URL that mirrors/contains ceph packages',
    )

    parser.add_argument(
        '--gpg-url',
        nargs='?',
        dest='gpg_url',
        help='specify a GPG key URL to be used with custom repos\
                (defaults to ceph.com)'
    )

    parser.set_defaults(
        func=install,
    )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purgedata,
        )
<|endoftext|>


<commit_before>import argparse
import logging

from . import exc
from . import lsb
from .cliutil import priority
from .sudo_pushy import get_transport

LOG = logging.getLogger(__name__)

def install_centos(release, codename, version_kind, version):
    import platform
    import subprocess

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    subprocess.check_call(
        args='su -c \'rpm --import "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc"\''.format(key=key),
        shell=True,
        )

    if version_kind == 'stable':
        url = 'http://ceph.com/rpm-{version}/'.format(
        version=version,
        )
    elif version_kind == 'testing':
        url = 'http://ceph.com/rpm-testing/'
    elif version_kind == 'dev':
        url = 'http://gitbuilder.ceph.com/ceph-rpm-centos{release}-{machine}-basic/ref/{version}/'.format(
            release=release.split(".",1)[0],
            machine=platform.machine(),
            version=version,
            )

    subprocess.call(
        args=['rpm', '-Uvh','--quiet', '{url}noarch/ceph-release-1-0.el6.noarch.rpm'.format(
            url=url
            )]
        )

    subprocess.check_call(
        args=[
            'yum',
            '-y',
            '-q',
            'install',
            'ceph',
            'ceph-common',
            'ceph-fs-common',
            ],
        )

def uninstall_centos(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def install_debian(release, codename, version_kind, version):
    import platform
    import subprocess

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    subprocess.check_call(
        args='wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'.format(key=key),
        shell=True,
        )

    if version_kind == 'stable':
        url = 'http://ceph.com/debian-{version}/'.format(
            version=version,
            )
    elif version_kind == 'testing':
        url = 'http://ceph.com/debian-testing/'
    elif version_kind == 'dev':
        url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
            codename=codename,
            machine=platform.machine(),
            version=version,
            )
    else:
        raise RuntimeError('Unknown version kind: %r' % version_kind)

    with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
        f.write('deb {url} {codename} main\n'.format(
                url=url,
                codename=codename,
                ))

    subprocess.check_call(
        args=[
            'apt-get',
            '-q',
            'update',
            ],
        )

    # TODO this does not downgrade -- should it?
    subprocess.check_call(
        args=[
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'DEBIAN_PRIORITY=critical',
            'apt-get',
            '-q',
            '-o', 'Dpkg::Options::=--force-confnew',
            'install',
            '--no-install-recommends',
            '--assume-yes',
            '--',
            'ceph',
            'ceph-mds',
            'ceph-common',
            'ceph-fs-common',
            # ceph only recommends gdisk, make sure we actually have
            # it; only really needed for osds, but minimal collateral
            'gdisk',
            ],
        )

def purge_data_any():
    import subprocess

    subprocess.call(args=[
            'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ])
    if os.path.exists('/var/lib/ceph'):
        subprocess.check_call(args=[
                'find', '/var/lib/ceph',
                '-mindepth', '1',
                '-maxdepth', '2',
                '-type', 'd',
                '-exec', 'umount', '{}', ';',
                ])
        subprocess.check_call(args=[
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ])

def install(args):
    version = getattr(args, args.version_kind)
    version_str = args.version_kind
    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s release %s codename %s', distro, release, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Installing on host %s ...', hostname)
            install_r = sudo.compile(install_debian)
        elif (distro == 'CentOS'):
            LOG.debug('Installing on host %s ...', hostname)
            install_r = sudo.compile(install_centos)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        install_r(
            release=release,
            codename=codename,
            version_kind=args.version_kind,
            version=version,
            )

def uninstall(args):
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Uninstalling on host %s ...', hostname)
            uninstall_r = sudo.compile(uninstall_debian)
        elif (distro == 'centos'):
            LOG.debug('Uninstalling on host %s ...', hostname)
            uninstall_r = sudo.compile(uninstall_centos)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        uninstall_r()

def purge(args):
    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Purging host %s ...', hostname)
            purge_r = sudo.compile(uninstall_debian)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        purge_r(arg_purge=True)

def purge_data(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        # TODO username
        sudo = args.pushy(get_transport(hostname))

        LOG.debug('Purging data from host %s ...', hostname)
        purge_data_any_r = sudo.compile(purge_data_any)
        purge_data_any_r()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        choices=[
            'argonaut',
            'bobtail',
            'cuttlefish',
            ],
        metavar='CODENAME',
        help='install a release known as CODENAME (done by default) (default: %(default)s)',
        )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
        )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch or tag (default: %(default)s)',
        )

    version.set_defaults(
        func=install,
        stable='bobtail',
        dev='master',
        version_kind='stable',
        )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
        )
    parser.set_defaults(
        func=install,
        )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purge_data,
        )
<commit_msg>install.py:  Use centos install for redhat.<commit_after>import argparse
import logging

from . import exc
from . import lsb
from .cliutil import priority
from .sudo_pushy import get_transport

LOG = logging.getLogger(__name__)

def install_centos(release, codename, version_kind, version):
    import platform
    import subprocess

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    subprocess.check_call(
        args='su -c \'rpm --import "https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc"\''.format(key=key),
        shell=True,
        )

    if version_kind == 'stable':
        url = 'http://ceph.com/rpm-{version}/'.format(
        version=version,
        )
    elif version_kind == 'testing':
        url = 'http://ceph.com/rpm-testing/'
    elif version_kind == 'dev':
        url = 'http://gitbuilder.ceph.com/ceph-rpm-centos{release}-{machine}-basic/ref/{version}/'.format(
            release=release.split(".",1)[0],
            machine=platform.machine(),
            version=version,
            )

    subprocess.call(
        args=['rpm', '-Uvh','--quiet', '{url}noarch/ceph-release-1-0.el6.noarch.rpm'.format(
            url=url
            )]
        )

    subprocess.check_call(
        args=[
            'yum',
            '-y',
            '-q',
            'install',
            'ceph',
            'ceph-common',
            'ceph-fs-common',
            ],
        )

def uninstall_centos(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'yum',
        '-q',
        '-y',
        'remove',
        ]

    args.extend(packages)
    subprocess.check_call(args=args)

def uninstall_debian(arg_purge=False):
    import subprocess

    packages = [
        'ceph',
        'ceph-mds',
        'ceph-common',
        'ceph-fs-common',
        ]
    args = [
        'apt-get',
        '-q',
        'remove',
        '-f',
        '-y',
        '--force-yes',
        ]
    if arg_purge:
        args.append('--purge')
    args.append('--')
    args.extend(packages)
    subprocess.check_call(args=args)

def install_debian(release, codename, version_kind, version):
    import platform
    import subprocess

    if version_kind in ['stable', 'testing']:
        key = 'release'
    else:
        key = 'autobuild'

    subprocess.check_call(
        args='wget -q -O- \'https://ceph.com/git/?p=ceph.git;a=blob_plain;f=keys/{key}.asc\' | apt-key add -'.format(key=key),
        shell=True,
        )

    if version_kind == 'stable':
        url = 'http://ceph.com/debian-{version}/'.format(
            version=version,
            )
    elif version_kind == 'testing':
        url = 'http://ceph.com/debian-testing/'
    elif version_kind == 'dev':
        url = 'http://gitbuilder.ceph.com/ceph-deb-{codename}-{machine}-basic/ref/{version}'.format(
            codename=codename,
            machine=platform.machine(),
            version=version,
            )
    else:
        raise RuntimeError('Unknown version kind: %r' % version_kind)

    with file('/etc/apt/sources.list.d/ceph.list', 'w') as f:
        f.write('deb {url} {codename} main\n'.format(
                url=url,
                codename=codename,
                ))

    subprocess.check_call(
        args=[
            'apt-get',
            '-q',
            'update',
            ],
        )

    # TODO this does not downgrade -- should it?
    subprocess.check_call(
        args=[
            'env',
            'DEBIAN_FRONTEND=noninteractive',
            'DEBIAN_PRIORITY=critical',
            'apt-get',
            '-q',
            '-o', 'Dpkg::Options::=--force-confnew',
            'install',
            '--no-install-recommends',
            '--assume-yes',
            '--',
            'ceph',
            'ceph-mds',
            'ceph-common',
            'ceph-fs-common',
            # ceph only recommends gdisk, make sure we actually have
            # it; only really needed for osds, but minimal collateral
            'gdisk',
            ],
        )

def purge_data_any():
    import subprocess

    subprocess.call(args=[
            'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
            ])
    if os.path.exists('/var/lib/ceph'):
        subprocess.check_call(args=[
                'find', '/var/lib/ceph',
                '-mindepth', '1',
                '-maxdepth', '2',
                '-type', 'd',
                '-exec', 'umount', '{}', ';',
                ])
        subprocess.check_call(args=[
                'rm', '-rf', '--one-file-system', '--', '/var/lib/ceph',
                ])

def install(args):
    version = getattr(args, args.version_kind)
    version_str = args.version_kind
    if version:
        version_str += ' version {version}'.format(version=version)
    LOG.debug(
        'Installing %s on cluster %s hosts %s',
        version_str,
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s release %s codename %s', distro, release, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Installing on host %s ...', hostname)
            install_r = sudo.compile(install_debian)
        elif (distro == 'CentOS') or distro.startswith('RedHat'):
            LOG.debug('Installing on host %s ...', hostname)
            install_r = sudo.compile(install_centos)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        install_r(
            release=release,
            codename=codename,
            version_kind=args.version_kind,
            version=version,
            )

def uninstall(args):
    LOG.debug(
        'Uninstalling on cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Uninstalling on host %s ...', hostname)
            uninstall_r = sudo.compile(uninstall_debian)
        elif (distro == 'centos'):
            LOG.debug('Uninstalling on host %s ...', hostname)
            uninstall_r = sudo.compile(uninstall_centos)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        uninstall_r()

def purge(args):
    LOG.debug(
        'Purging from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        LOG.debug('Detecting platform for host %s ...', hostname)

        # TODO username
        sudo = args.pushy(get_transport(hostname))
        (distro, release, codename) = lsb.get_lsb_release(sudo)
        LOG.debug('Distro %s codename %s', distro, codename)

        if (distro == 'Debian' or distro == 'Ubuntu'):
            LOG.debug('Purging host %s ...', hostname)
            purge_r = sudo.compile(uninstall_debian)
        else:
            raise exc.UnsupportedPlatform(distro=distro, codename=codename)

        purge_r(arg_purge=True)

def purge_data(args):
    LOG.debug(
        'Purging data from cluster %s hosts %s',
        args.cluster,
        ' '.join(args.host),
        )

    for hostname in args.host:
        # TODO username
        sudo = args.pushy(get_transport(hostname))

        LOG.debug('Purging data from host %s ...', hostname)
        purge_data_any_r = sudo.compile(purge_data_any)
        purge_data_any_r()

class StoreVersion(argparse.Action):
    """
    Like ``"store"`` but also remember which one of the exclusive
    options was set.

    There are three kinds of versions: stable, testing and dev.
    This sets ``version_kind`` to be the right one of the above.

    This kludge essentially lets us differentiate explicitly set
    values from defaults.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        setattr(namespace, self.dest, values)
        namespace.version_kind = self.dest

@priority(20)
def make(parser):
    """
    Install Ceph packages on remote hosts.
    """

    version = parser.add_mutually_exclusive_group()

    version.add_argument(
        '--stable',
        nargs='?',
        action=StoreVersion,
        choices=[
            'argonaut',
            'bobtail',
            'cuttlefish',
            ],
        metavar='CODENAME',
        help='install a release known as CODENAME (done by default) (default: %(default)s)',
        )

    version.add_argument(
        '--testing',
        nargs=0,
        action=StoreVersion,
        help='install the latest development release',
        )

    version.add_argument(
        '--dev',
        nargs='?',
        action=StoreVersion,
        const='master',
        metavar='BRANCH_OR_TAG',
        help='install a bleeding edge build from Git branch or tag (default: %(default)s)',
        )

    version.set_defaults(
        func=install,
        stable='bobtail',
        dev='master',
        version_kind='stable',
        )

    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to install on',
        )
    parser.set_defaults(
        func=install,
        )

@priority(80)
def make_uninstall(parser):
    """
    Remove Ceph packages from remote hosts.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to uninstall Ceph from',
        )
    parser.set_defaults(
        func=uninstall,
        )

@priority(80)
def make_purge(parser):
    """
    Remove Ceph packages from remote hosts and purge all data.
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph from',
        )
    parser.set_defaults(
        func=purge,
        )

@priority(80)
def make_purge_data(parser):
    """
    Purge (delete, destroy, discard, shred) any Ceph data from /var/lib/ceph
    """
    parser.add_argument(
        'host',
        metavar='HOST',
        nargs='+',
        help='hosts to purge Ceph data from',
        )
    parser.set_defaults(
        func=purge_data,
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import sys
import os

import json

import collections

from shapely.geometry.point import BaseGeometry
from shapely.geometry import shape, mapping
from shapely.ops import cascaded_union

import util

class Feature(object):
    """単一の feature を扱うためのクラス
    """
    def __init__(self, data):
        """feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.load(data)

    def load(self, data):
        """data を元に feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.geometry = data['geometry']
        self._properties = util.rec_decode(data['properties'])
        self._attributes = util.rec_decode({
            k: v for k, v in data.items()
            if k not in set(('geometry', 'properties', 'type'))
        })
        return self

    def dump(self):
        """このインスタンスを表す、json.dumpsなどでダンプ可能なオブジェクトを返す
        """
        return dict({u'type': u'Feature',
                     u'geometry': util.rec_decode(mapping(self._geometry)),
                     u'properties': self._properties}, **self._attributes)

    @property
    def properties(self):
        """このインスタンスの properties オブジェクト自体を返す
        """
        return self._properties

    @properties.setter
    def properties(self, x):
        """このインスタンスの properties を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('properties must be an instance of Mapping')
        self._properties = rDecode(x)
        return self

    @property
    def geometry(self):
        """このインスタンスの geometry オブジェクト自体を返す
        """
        return self._geometry

    @geometry.setter
    def geometry(self, x):
        """このインスタンの geometry を設定する

        :param x: shape か shape に変換可能な Mapping オブジェクト
        """
        if isinstance(x, BaseGeometry):
            self._geometry = x
        elif util.is_map(x):
            self._geometry = shape(x)
        else:
            raise Exception('geometry must be an instance of shape')
        return self

    @property
    def attributes(self):
        """このインスタンスのその他の属性の Mapping オブジェクトを返す
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """このインスタンスのその他の属性を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('attributes must be an instance of Mapping')
        self._attributes = rDecode(x)
        return self

    def property(self, *x):
        """properties を set/get する。
        property('id') id 属性の値を取得する
        property('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]):
                    self.property(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.property(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._properties:
                return self._properties[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.safe_decode(x[1])
        if v is None:
            if k in self._properties:
                del self._properties[k]
            return self
        self._properties[k] = v
        return self

    def attr(self, *x):
        """attributes を set/get する。
        attr('id') id 属性の値を取得する
        attr('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]):
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.safe_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self

    def isMatch(self, condition):
        """このインスタンスの properties が condition に一致しているかどうか返す

        :param condition: 一つの引数（propertiesが渡される）を取る関数
        """
        return condition(self._properties)

    def join(self, d):
        """d のキーヴァリューの組をこのインスタンスの properties に追加する

        :param d: Mapping オブジェクト
        """
        self._properties = dict(self._properties, **d)
        return self
<commit_msg>fixed bug<commit_after># -*- coding: utf-8 -*-

import sys
import os

import json

import collections

from shapely.geometry.point import BaseGeometry
from shapely.geometry import shape, mapping
from shapely.ops import cascaded_union

import util

class Feature(object):
    """単一の feature を扱うためのクラス
    """
    def __init__(self, data):
        """feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.load(data)

    def load(self, data):
        """data を元に feature を構成する

        :param data: 'geometry' と 'properties' を属性に持った Mapping オブジェクト
        """
        self.geometry = data['geometry']
        self._properties = util.rec_decode(data['properties'])
        self._attributes = util.rec_decode({
            k: v for k, v in data.items()
            if k not in set(('geometry', 'properties', 'type'))
        })
        return self

    def dump(self):
        """このインスタンスを表す、json.dumpsなどでダンプ可能なオブジェクトを返す
        """
        return dict({u'type': u'Feature',
                     u'geometry': util.rec_decode(mapping(self._geometry)),
                     u'properties': self._properties}, **self._attributes)

    @property
    def properties(self):
        """このインスタンスの properties オブジェクト自体を返す
        """
        return self._properties

    @properties.setter
    def properties(self, x):
        """このインスタンスの properties を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('properties must be an instance of Mapping')
        self._properties = rDecode(x)
        return self

    @property
    def geometry(self):
        """このインスタンスの geometry オブジェクト自体を返す
        """
        return self._geometry

    @geometry.setter
    def geometry(self, x):
        """このインスタンの geometry を設定する

        :param x: shape か shape に変換可能な Mapping オブジェクト
        """
        if isinstance(x, BaseGeometry):
            self._geometry = x
        elif util.is_map(x):
            self._geometry = shape(x)
        else:
            raise Exception('geometry must be an instance of shape')
        return self

    @property
    def attributes(self):
        """このインスタンスのその他の属性の Mapping オブジェクトを返す
        """
        return self._attributes

    @attributes.setter
    def attributes(self, x):
        """このインスタンスのその他の属性を設定する

        :param x: Mapping オブジェクト
        """
        if not util.is_map(x):
            raise Exception('attributes must be an instance of Mapping')
        self._attributes = rDecode(x)
        return self

    def property(self, *x):
        """properties を set/get する。
        property('id') id 属性の値を取得する
        property('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]):
                    self.property(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.property(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.property(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._properties:
                return self._properties[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.safe_decode(x[1])
        if v is None:
            if k in self._properties:
                del self._properties[k]
            return self
        self._properties[k] = v
        return self

    def attr(self, *x):
        """attributes を set/get する。
        attr('id') id 属性の値を取得する
        attr('id', 'a123') id 属性の値を a123 に設定する

        :param x: 単一の値、list, dict, set, tupleまたはキーとヴァリューのペア
        """
        if len(x) == 0:
            return self
        if len(x) == 1:
            if util.is_map(x[0]):
                for k, v in util.rec_decode(x[0]):
                    self.attr(k, v)
                return self
            if isinstance(x[0], collections.Set):
                return {k: self.attr(k) for k in util.rec_decode(x[0])}
            if util.is_array(x[0]):
                return util.cons_array(
                    (self.attr(k) for k in util.rec_decode(x[0])),
                    x[0].__class__, tuple)
            k = util.safe_decode(x[0])
            if not util.is_string(x[0]):
                k = unicode(x[0])
            if k in self._attributes:
                return self._attributes[k]
            return None
        k = util.safe_decode(x[0])
        if not util.is_string(x[0]):
            k = unicode(x[0])
        v = util.safe_decode(x[1])
        if v is None:
            if k in self._attributes:
                del self._attributes[k]
            return self
        self._attributes[k] = v
        return self
<|endoftext|>