<commit_before># -*- coding: utf-8 -*-
# kqueue_observer.py: kqueue-based observer implementation for BSD systems.
#
# Copyright (C) 2010 Gora Khargosh <gora.khargosh@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import stat
import sys
import errno
import os.path

from watchdog.utils import has_attribute

try:
    # Python 3k
    from queue import Queue, Empty as QueueEmpty
except ImportError:
    from Queue import Queue, Empty as QueueEmpty
import select
if not has_attribute(select, 'kqueue') or sys.version_info < (2,7,0):
    import select_backport as select

from threading import Thread, Lock as ThreadedLock, Event as ThreadedEvent
from watchdog.utils import get_walker
from watchdog.dirsnapshot import DirectorySnapshot
from watchdog.decorator_utils import synchronized
from watchdog.observers import DaemonThread
from watchdog.observers.polling_observer import PollingObserver
from watchdog.events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

import logging
logging.basicConfig(level=logging.DEBUG)

# Maximum number of events to process.
MAX_EVENTS = 104896

# Mac OS X file system performance guidelines:
# --------------------------------------------
# http://developer.apple.com/library/ios/#documentation/Performance/Conceptual/FileSystem/Articles/TrackingChanges.html#//apple_ref/doc/uid/20001993-CJBJFIDD
# http://www.mlsite.net/blog/?p=2312
#
# Specifically:
# -------------
# When you only want to track changes on a file or directory, be sure to
# open it# using the O_EVTONLY flag. This flag prevents the file or
# directory from being marked as open or in use. This is important
# if you are tracking files on a removable volume and the user tries to
# unmount the volume. With this flag in place, the system knows it can
# dismiss the volume. If you had opened the files or directories without
# this flag, the volume would be marked as busy and would not be unmounted.
O_EVTONLY = 0x8000

# Flags pre-calculated that we will use for the kevent filter, flags, and
# fflags attributes.
if sys.platform == 'darwin':
    WATCHDOG_OS_OPEN_FLAGS = O_EVTONLY
else:
    WATCHDOG_OS_OPEN_FLAGS = os.O_RDONLY | os.O_NONBLOCK
WATCHDOG_KQ_FILTER = select.KQ_FILTER_VNODE
WATCHDOG_KQ_EV_FLAGS = select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR
WATCHDOG_KQ_FFLAGS = \
    select.KQ_NOTE_DELETE | \
    select.KQ_NOTE_WRITE  | \
    select.KQ_NOTE_EXTEND | \
    select.KQ_NOTE_ATTRIB | \
    select.KQ_NOTE_LINK   | \
    select.KQ_NOTE_RENAME | \
    select.KQ_NOTE_REVOKE

def create_kevent_for_path(path):
    """Creates a kevent for the given path."""
    fd = os.open(path, WATCHDOG_OS_OPEN_FLAGS)
    kev = select.kevent(fd,
                        filter=WATCHDOG_KQ_FILTER,
                        flags=WATCHDOG_KQ_EV_FLAGS,
                        fflags=WATCHDOG_KQ_FFLAGS)
    return kev, fd

# Flag tests.
def is_deleted(kev):
    """Determines whether the given kevent represents deletion."""
    return kev.fflags & select.KQ_NOTE_DELETE

def is_modified(kev):
    """Determines whether the given kevent represents modification."""
    fflags = kev.fflags
    return (fflags & select.KQ_NOTE_EXTEND) or (fflags & select.KQ_NOTE_WRITE)

def is_attrib_modified(kev):
    """Determines whether the given kevent represents attribute modification."""
    return kev.fflags & select.KQ_NOTE_ATTRIB

def is_renamed(kev):
    """Determines whether the given kevent represents movement."""
    return kev.fflags & select.KQ_NOTE_RENAME

class _FileSystemObject(object):
    """Handy structure to store relevant information in."""
    def __init__(self, fd, kev, path, is_directory):
        self.fd = fd
        self.path = path
        self.kev = kev
        self.is_directory = is_directory

class _KqueueEventEmitter(DaemonThread):
    def __init__(self, path, out_event_queue, recursive, interval=1):
        DaemonThread.__init__(self, interval)
        self.path = os.path.abspath(os.path.realpath(path))
        self.out_event_queue = out_event_queue
        self.is_recursive = recursive
        self.kq = select.kqueue()
        self.kevent_list = list()
        self.fso_table = dict()
        self.descriptor_list = set()
        self.dir_snapshot = None

    @synchronized()
    def register_dir_tree(self, path, recursive):
        """Registers event handling for an entire directory tree."""
        path = os.path.abspath(os.path.realpath(path)).rstrip(os.path.sep)

        def walker_callback(path, stat_info, self=self):
            self.register_path(path, stat.S_ISDIR(stat_info.st_mode))
        self.dir_snapshot = DirectorySnapshot(path, recursive, walker_callback)

    @synchronized()
    def unregister_path(self, path):
        """Bookkeeping method that unregisters watching a given path."""
        path = path.rstrip(os.path.sep)
        if path in self.fso_table:
            fso = self.fso_table[path]
            self.kevent_list.remove(fso.kev)
            del self.fso_table[path]
            del self.fso_table[fso.fd]
            try:
                os.close(fso.fd)
            except OSError, e:
                logging.warn(e)
            self.descriptor_list.remove(fso.fd)

    @synchronized()
    def register_path(self, path, is_directory=False):
        """Bookkeeping method that registers watching a given path."""
        path = path.rstrip(os.path.sep)
        if not path in self.fso_table:
            # If we haven't registered a kevent for this path already,
            # add a new kevent for the path.
            kev, fd = create_kevent_for_path(path)
            self.kevent_list.append(kev)
            fso = _FileSystemObject(fd, kev, path, is_directory)
            self.descriptor_list.add(fd)
            self.fso_table[fd] = fso
            self.fso_table[path] = fso

    def __process_kevents_except_movement(self, event_list, out_event_queue):
        """Process only basic kevents. Movement and directory modifications
        need to be further processed."""
        files_renamed = set()
        dirs_renamed = set()
        dirs_modified = set()

        for kev in event_list:
            fso = self.fso_table[kev.ident]
            src_path = fso.path

            if is_deleted(kev):
                if fso.is_directory:
                    event = DirDeletedEvent(src_path=src_path)
                else:
                    event = FileDeletedEvent(src_path=src_path)
                out_event_queue.put((self.path, event))
                self.unregister_path(src_path)
            elif is_attrib_modified(kev):
                if fso.is_directory:
                    event = DirModifiedEvent(src_path=src_path)
                else:
                    event = FileModifiedEvent(src_path=src_path)
                out_event_queue.put((self.path, event))
            elif is_modified(kev):
                if fso.is_directory:
                    dirs_modified.add(src_path)
                else:
                    out_event_queue.put((self.path, FileModifiedEvent(src_path=src_path)))
            elif is_renamed(kev):
                if fso.is_directory:
                    dirs_renamed.add(src_path)
                else:
                    files_renamed.add(src_path)

        return files_renamed, dirs_renamed, dirs_modified

    def __process_kevent_file_renames(self, out_event_queue, \
                                          ref_dir_snapshot, \
                                          new_dir_snapshot, \
                                          files_renamed):
        """Process kevent-hinted file renames. These may be deletes too
        relative to the watched directory."""
        for path_renamed in files_renamed:
            # These are kqueue-hinted renames. We classify them into
            # either moved if the new path is found or deleted.
            ref_stat_info = ref_dir_snapshot.stat_info(path_renamed)
            try:
                path = new_dir_snapshot.path_for_inode(ref_stat_info.st_ino)
                out_event_queue.put((self.path, FileMovedEvent(src_path=path_renamed, dest_path=path)))
                self.unregister_path(path_renamed)
                self.register_path(path, is_directory=False)
            except KeyError:
                # We could not find the new name.
                out_event_queue.put((self.path, FileDeletedEvent(src_path=path_renamed)))
                self.unregister_path(path_renamed)

    def __process_kevent_dir_renames(self, out_event_queue, \
                                         ref_dir_snapshot, \
                                         new_dir_snapshot, \
                                         dirs_renamed):
        """Process kevent-hinted directory renames. These may be deletes
        too relative to the watched directory."""
        for path_renamed in dirs_renamed:
            # These are kqueue-hinted renames. We classify them into
            # either moved if the new path is found or deleted.
            ref_stat_info = ref_dir_snapshot.stat_info(path_renamed)
            try:
                path = new_dir_snapshot.path_for_inode(ref_stat_info.st_ino)
                path = path.rstrip(os.path.sep)

                # If we're in recursive mode, we fire move events for
                # the entire contents of the moved directory.
                if self.is_recursive:
                    dir_path_renamed = path_renamed.rstrip(os.path.sep)
                    for root, directories, filenames in os.walk(path):
                        for directory_path in directories:
                            full_path = os.path.join(root, directory_path)
                            renamed_path = full_path.replace(path, dir_path_renamed)
                            out_event_queue.put((self.path, DirMovedEvent(src_path=renamed_path, dest_path=full_path)))
                            self.unregister_path(renamed_path)
                            self.register_path(full_path, is_directory=True)
                        for filename in filenames:
                            full_path = os.path.join(root, filename)
                            renamed_path = full_path.replace(path, dir_path_renamed)
                            out_event_queue.put((self.path, FileMovedEvent(src_path=renamed_path, dest_path=full_path)))
                            self.unregister_path(renamed_path)
                            self.register_path(full_path, is_directory=False)

                # Fire the directory moved events after firing moved
                # events for its children file system objects.
                out_event_queue.put((self.path, DirMovedEvent(src_path=path_renamed, dest_path=path)))
                self.unregister_path(path_renamed)
                self.register_path(path, is_directory=True)
            except KeyError:
                # We could not find the new name.
                out_event_queue.put((self.path, DirDeletedEvent(src_path=path_renamed)))
                self.unregister_path(path)

    def __process_kevent_dir_modifications(self, out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_modified):
        """Process kevent-hinted directory modifications. Created
        files/directories are also detected here."""
        for dir_modified in dirs_modified:
            out_event_queue.put((self.path, DirModifiedEvent(src_path=dir_modified)))
            # Don't need to register here. It's already registered.
            #self.register_path(dir_modified, is_directory=True)
        diff = new_dir_snapshot - ref_dir_snapshot
        for file_created in diff.files_created:
            out_event_queue.put((self.path, FileCreatedEvent(src_path=file_created)))
            self.register_path(file_created, is_directory=False)
        for dir_created in diff.dirs_created:
            out_event_queue.put((self.path, DirCreatedEvent(src_path=dir_created)))
            self.register_path(dir_created, is_directory=True)

    @synchronized()
    def process_events(self, out_event_queue):
        """Blocking call to kqueue.control that enlists events and then
    processes them classifying them into various events defined in watchdog.events."""
        event_list = self.kq.control(list(self.kevent_list), MAX_EVENTS)
        files_renamed, dirs_renamed, dirs_modified = \
            self.__process_kevents_except_movement(event_list,
                                                   out_event_queue)

        # Take a fresh snapshot of the directory and update saved snapshot.
        new_dir_snapshot = DirectorySnapshot(self.path, self.is_recursive)
        ref_dir_snapshot = self.dir_snapshot
        self.dir_snapshot = new_dir_snapshot

        # Process events for renames and directories modified.
        if files_renamed or dirs_renamed or dirs_modified:
            self.__process_kevent_file_renames(out_event_queue, ref_dir_snapshot, new_dir_snapshot, files_renamed)
            self.__process_kevent_dir_renames(out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_renamed)

            if dirs_modified:
                self.__process_kevent_dir_modifications(out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_modified)

    def run(self):
        self.register_dir_tree(self.path, self.is_recursive)
        while not self.is_stopped:
            try:
                #if not os.path.exists(self.path):
                #    self.stop()
                #    continue
                self.process_events(self.out_event_queue)
            except OSError, e:
                if e.errno == errno.EBADF:
                    # select.kqueue seems to be blowing up on the first
                    # call to kqueue.control with this error.
                    logging.debug(e)
                    continue
                else:
                    raise

        # Close all open file descriptors
        for fd in self.descriptor_list:
            try:
                os.close(fd)
            except OSError, e:
                logging.warn(e)
        self.kq.close()

class KqueueObserver(PollingObserver):
    """BSD/OS X kqueue-based observer implementation."""
    def _create_event_emitter(self, path, recursive):
        return _KqueueEventEmitter(path=path,
                                   interval=self.interval,
                                   out_event_queue=self.event_queue,
                                   recursive=recursive)

<commit_msg>Handle very fast file movement in kqueue_observer.py<commit_after># -*- coding: utf-8 -*-
# kqueue_observer.py: kqueue-based observer implementation for BSD systems.
#
# Copyright (C) 2010 Gora Khargosh <gora.khargosh@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import os
import stat
import sys
import errno
import os.path

from watchdog.utils import has_attribute

try:
    # Python 3k
    from queue import Queue, Empty as QueueEmpty
except ImportError:
    from Queue import Queue, Empty as QueueEmpty
import select
if not has_attribute(select, 'kqueue') or sys.version_info < (2,7,0):
    import select_backport as select

from threading import Thread, Lock as ThreadedLock, Event as ThreadedEvent
from watchdog.utils import get_walker
from watchdog.dirsnapshot import DirectorySnapshot
from watchdog.decorator_utils import synchronized
from watchdog.observers import DaemonThread
from watchdog.observers.polling_observer import PollingObserver
from watchdog.events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

import logging
logging.basicConfig(level=logging.DEBUG)

# Maximum number of events to process.
MAX_EVENTS = 104896

# Mac OS X file system performance guidelines:
# --------------------------------------------
# http://developer.apple.com/library/ios/#documentation/Performance/Conceptual/FileSystem/Articles/TrackingChanges.html#//apple_ref/doc/uid/20001993-CJBJFIDD
# http://www.mlsite.net/blog/?p=2312
#
# Specifically:
# -------------
# When you only want to track changes on a file or directory, be sure to
# open it# using the O_EVTONLY flag. This flag prevents the file or
# directory from being marked as open or in use. This is important
# if you are tracking files on a removable volume and the user tries to
# unmount the volume. With this flag in place, the system knows it can
# dismiss the volume. If you had opened the files or directories without
# this flag, the volume would be marked as busy and would not be unmounted.
O_EVTONLY = 0x8000

# Flags pre-calculated that we will use for the kevent filter, flags, and
# fflags attributes.
if sys.platform == 'darwin':
    WATCHDOG_OS_OPEN_FLAGS = O_EVTONLY
else:
    WATCHDOG_OS_OPEN_FLAGS = os.O_RDONLY | os.O_NONBLOCK
WATCHDOG_KQ_FILTER = select.KQ_FILTER_VNODE
WATCHDOG_KQ_EV_FLAGS = select.KQ_EV_ADD | select.KQ_EV_ENABLE | select.KQ_EV_CLEAR
WATCHDOG_KQ_FFLAGS = \
    select.KQ_NOTE_DELETE | \
    select.KQ_NOTE_WRITE  | \
    select.KQ_NOTE_EXTEND | \
    select.KQ_NOTE_ATTRIB | \
    select.KQ_NOTE_LINK   | \
    select.KQ_NOTE_RENAME | \
    select.KQ_NOTE_REVOKE

def create_kevent_for_path(path):
    """Creates a kevent for the given path."""
    fd = os.open(path, WATCHDOG_OS_OPEN_FLAGS)
    kev = select.kevent(fd,
                        filter=WATCHDOG_KQ_FILTER,
                        flags=WATCHDOG_KQ_EV_FLAGS,
                        fflags=WATCHDOG_KQ_FFLAGS)
    return kev, fd

# Flag tests.
def is_deleted(kev):
    """Determines whether the given kevent represents deletion."""
    return kev.fflags & select.KQ_NOTE_DELETE

def is_modified(kev):
    """Determines whether the given kevent represents modification."""
    fflags = kev.fflags
    return (fflags & select.KQ_NOTE_EXTEND) or (fflags & select.KQ_NOTE_WRITE)

def is_attrib_modified(kev):
    """Determines whether the given kevent represents attribute modification."""
    return kev.fflags & select.KQ_NOTE_ATTRIB

def is_renamed(kev):
    """Determines whether the given kevent represents movement."""
    return kev.fflags & select.KQ_NOTE_RENAME

class _FileSystemObject(object):
    """Handy structure to store relevant information in."""
    def __init__(self, fd, kev, path, is_directory):
        self.fd = fd
        self.path = path
        self.kev = kev
        self.is_directory = is_directory

class _KqueueEventEmitter(DaemonThread):
    def __init__(self, path, out_event_queue, recursive, interval=1):
        DaemonThread.__init__(self, interval)
        self.path = os.path.abspath(os.path.realpath(path))
        self.out_event_queue = out_event_queue
        self.is_recursive = recursive
        self.kq = select.kqueue()
        self.kevent_list = list()
        self.fso_table = dict()
        self.descriptor_list = set()
        self.dir_snapshot = None

    @synchronized()
    def register_dir_tree(self, path, recursive):
        """Registers event handling for an entire directory tree."""
        path = os.path.abspath(os.path.realpath(path)).rstrip(os.path.sep)

        def walker_callback(path, stat_info, self=self):
            self.register_path(path, stat.S_ISDIR(stat_info.st_mode))
        self.dir_snapshot = DirectorySnapshot(path, recursive, walker_callback)

    @synchronized()
    def unregister_path(self, path):
        """Bookkeeping method that unregisters watching a given path."""
        path = path.rstrip(os.path.sep)
        if path in self.fso_table:
            fso = self.fso_table[path]
            self.kevent_list.remove(fso.kev)
            del self.fso_table[path]
            del self.fso_table[fso.fd]
            try:
                os.close(fso.fd)
            except OSError, e:
                logging.warn(e)
            self.descriptor_list.remove(fso.fd)

    @synchronized()
    def register_path(self, path, is_directory=False):
        """Bookkeeping method that registers watching a given path."""
        path = path.rstrip(os.path.sep)
        if not path in self.fso_table and os.path.exists(path):
            try:
                # If we haven't registered a kevent for this path already,
                # add a new kevent for the path.
                kev, fd = create_kevent_for_path(path)
                self.kevent_list.append(kev)
                fso = _FileSystemObject(fd, kev, path, is_directory)
                self.descriptor_list.add(fd)
                self.fso_table[fd] = fso
                self.fso_table[path] = fso
            except OSError, e:
                if e.errno == errno.ENOENT:
                    # No such file or directory.
                    # Possibly a temporary file we can ignore.
                    logging.warn(e)

    def __process_kevents_except_movement(self, event_list, out_event_queue):
        """Process only basic kevents. Movement and directory modifications
        need to be further processed."""
        files_renamed = set()
        dirs_renamed = set()
        dirs_modified = set()

        for kev in event_list:
            fso = self.fso_table[kev.ident]
            src_path = fso.path

            if is_deleted(kev):
                if fso.is_directory:
                    event = DirDeletedEvent(src_path=src_path)
                else:
                    event = FileDeletedEvent(src_path=src_path)
                out_event_queue.put((self.path, event))
                self.unregister_path(src_path)
            elif is_attrib_modified(kev):
                if fso.is_directory:
                    event = DirModifiedEvent(src_path=src_path)
                else:
                    event = FileModifiedEvent(src_path=src_path)
                out_event_queue.put((self.path, event))
            elif is_modified(kev):
                if fso.is_directory:
                    dirs_modified.add(src_path)
                else:
                    out_event_queue.put((self.path, FileModifiedEvent(src_path=src_path)))
            elif is_renamed(kev):
                if fso.is_directory:
                    dirs_renamed.add(src_path)
                else:
                    files_renamed.add(src_path)

        return files_renamed, dirs_renamed, dirs_modified

    def __process_kevent_file_renames(self, out_event_queue, \
                                          ref_dir_snapshot, \
                                          new_dir_snapshot, \
                                          files_renamed):
        """Process kevent-hinted file renames. These may be deletes too
        relative to the watched directory."""
        for path_renamed in files_renamed:
            # These are kqueue-hinted renames. We classify them into
            # either moved if the new path is found or deleted.
            try:
                ref_stat_info = ref_dir_snapshot.stat_info(path_renamed)
                path = new_dir_snapshot.path_for_inode(ref_stat_info.st_ino)
                out_event_queue.put((self.path, FileMovedEvent(src_path=path_renamed, dest_path=path)))
                self.unregister_path(path_renamed)
                self.register_path(path, is_directory=False)
            except KeyError:
                # We could not find the new name.
                out_event_queue.put((self.path, FileDeletedEvent(src_path=path_renamed)))
                self.unregister_path(path_renamed)

    def __process_kevent_dir_renames(self, out_event_queue, \
                                         ref_dir_snapshot, \
                                         new_dir_snapshot, \
                                         dirs_renamed):
        """Process kevent-hinted directory renames. These may be deletes
        too relative to the watched directory."""
        for path_renamed in dirs_renamed:
            # These are kqueue-hinted renames. We classify them into
            # either moved if the new path is found or deleted.
            try:
                ref_stat_info = ref_dir_snapshot.stat_info(path_renamed)
                path = new_dir_snapshot.path_for_inode(ref_stat_info.st_ino)
                path = path.rstrip(os.path.sep)

                # If we're in recursive mode, we fire move events for
                # the entire contents of the moved directory.
                if self.is_recursive:
                    dir_path_renamed = path_renamed.rstrip(os.path.sep)
                    for root, directories, filenames in os.walk(path):
                        for directory_path in directories:
                            full_path = os.path.join(root, directory_path)
                            renamed_path = full_path.replace(path, dir_path_renamed)
                            out_event_queue.put((self.path, DirMovedEvent(src_path=renamed_path, dest_path=full_path)))
                            self.unregister_path(renamed_path)
                            self.register_path(full_path, is_directory=True)
                        for filename in filenames:
                            full_path = os.path.join(root, filename)
                            renamed_path = full_path.replace(path, dir_path_renamed)
                            out_event_queue.put((self.path, FileMovedEvent(src_path=renamed_path, dest_path=full_path)))
                            self.unregister_path(renamed_path)
                            self.register_path(full_path, is_directory=False)

                # Fire the directory moved events after firing moved
                # events for its children file system objects.
                out_event_queue.put((self.path, DirMovedEvent(src_path=path_renamed, dest_path=path)))
                self.unregister_path(path_renamed)
                self.register_path(path, is_directory=True)
            except KeyError:
                # We could not find the new name.
                out_event_queue.put((self.path, DirDeletedEvent(src_path=path_renamed)))
                self.unregister_path(path)

    def __process_kevent_dir_modifications(self, out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_modified):
        """Process kevent-hinted directory modifications. Created
        files/directories are also detected here."""
        for dir_modified in dirs_modified:
            out_event_queue.put((self.path, DirModifiedEvent(src_path=dir_modified)))
            # Don't need to register here. It's already registered.
            #self.register_path(dir_modified, is_directory=True)
        diff = new_dir_snapshot - ref_dir_snapshot
        for file_created in diff.files_created:
            out_event_queue.put((self.path, FileCreatedEvent(src_path=file_created)))
            self.register_path(file_created, is_directory=False)
        for dir_created in diff.dirs_created:
            out_event_queue.put((self.path, DirCreatedEvent(src_path=dir_created)))
            self.register_path(dir_created, is_directory=True)

    @synchronized()
    def process_events(self, out_event_queue):
        """Blocking call to kqueue.control that enlists events and then
    processes them classifying them into various events defined in watchdog.events."""
        event_list = self.kq.control(list(self.kevent_list), MAX_EVENTS)
        files_renamed, dirs_renamed, dirs_modified = \
            self.__process_kevents_except_movement(event_list,
                                                   out_event_queue)

        # Take a fresh snapshot of the directory and update saved snapshot.
        new_dir_snapshot = DirectorySnapshot(self.path, self.is_recursive)
        ref_dir_snapshot = self.dir_snapshot
        self.dir_snapshot = new_dir_snapshot

        # Process events for renames and directories modified.
        if files_renamed or dirs_renamed or dirs_modified:
            self.__process_kevent_file_renames(out_event_queue, ref_dir_snapshot, new_dir_snapshot, files_renamed)
            self.__process_kevent_dir_renames(out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_renamed)

            if dirs_modified:
                self.__process_kevent_dir_modifications(out_event_queue, ref_dir_snapshot, new_dir_snapshot, dirs_modified)

    def run(self):
        self.register_dir_tree(self.path, self.is_recursive)
        while not self.is_stopped:
            try:
                #if not os.path.exists(self.path):
                #    self.stop()
                #    continue
                self.process_events(self.out_event_queue)
            except OSError, e:
                if e.errno == errno.EBADF:
                    # select.kqueue seems to be blowing up on the first
                    # call to kqueue.control with this error.
                    logging.debug(e)
                    continue
                else:
                    raise

        # Close all open file descriptors
        for fd in self.descriptor_list:
            try:
                os.close(fd)
            except OSError, e:
                logging.warn(e)
        self.kq.close()

class KqueueObserver(PollingObserver):
    """BSD/OS X kqueue-based observer implementation."""
    def _create_event_emitter(self, path, recursive):
        return _KqueueEventEmitter(path=path,
                                   interval=self.interval,
                                   out_event_queue=self.event_queue,
                                   recursive=recursive)

<|endoftext|>


<commit_before># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

__version__ = "1.2.0.1"
<commit_msg>Bump version to 1.2.0.2.dev0<commit_after># Copyright (c) 2015, The MITRE Corporation. All rights reserved.
# See LICENSE.txt for complete terms.

__version__ = "1.2.0.2.dev0"
<|endoftext|>


<commit_before>from google.appengine.api import memcache

from mlabns.db import model
from mlabns.util import constants
from mlabns.util import distance
from mlabns.util import message

import logging
import math
import random

class ResolverBase:
    """Resolver base class."""

    def get_candidates(self, query):
        """Find candidates for server selection.

        Args:
            query: A LookupQuery instance.

        Returns:
            A list of SliverTool entities that match the requirements
            specified in the 'query'.
        """
        candidates = []
        if query.address_family is not None:
            candidates = self._get_candidates(query, query.address_family)
        # If no candidates with this address family and if this address family
        # was not user-defined, try the other address family.
        if len(candidates) == 0 and \
            query.address_family != query.user_defined_af:
            if query.address_family == message.ADDRESS_FAMILY_IPv4:
                candidates = self._get_candidates(query,
                                                  message.ADDRESS_FAMILY_IPv6)
            elif query.address_family == message.ADDRESS_FAMILY_IPv6:
                candidates = self._get_candidates(query,
                                                  message.ADDRESS_FAMILY_IPv4)
        if (query.sieves):
            for sieve in query.sieves:
                candidates = sieve.sieve(candidates)
        return candidates

    def _get_candidates(self, query, address_family):
        """Returns a (possibly empty) list of available candidates."""

        # First try to get the sliver tools from the memcache.
        sliver_tools = memcache.get(
            query.tool_id, namespace=constants.MEMCACHE_NAMESPACE_TOOLS)
        if sliver_tools is not None:
            logging.info('Sliver tools found in memcache (%s results).',
                         len(sliver_tools))
            candidates = []
            for sliver_tool in sliver_tools:
                if (address_family == message.ADDRESS_FAMILY_IPv4 and
                    sliver_tool.status_ipv4 == message.STATUS_ONLINE) or \
                    (address_family == message.ADDRESS_FAMILY_IPv6 and
                    sliver_tool.status_ipv6 == message.STATUS_ONLINE):
                    candidates.append(sliver_tool)
            return candidates
        logging.info('Sliver tools not found in memcache.')

        # Get the sliver tools from datastore.
        status_field = 'status_' + address_family
        candidates = model.SliverTool.gql(
            'WHERE tool_id = :tool_id '
            'AND ' + status_field + ' = :status',
            tool_id=query.tool_id,
            status=message.STATUS_ONLINE)
        logging.info('Found (%s candidates)', candidates.count())
        return candidates.fetch(constants.MAX_FETCHED_RESULTS)

    def _get_candidates_from_sites(self, query, address_family, site_id_list):
        """Returns a (possibly empty) list of available candidates."""

        # First try to get the sliver tools from the cache.
        sliver_tools = memcache.get(
            query.tool_id, namespace=constants.MEMCACHE_NAMESPACE_TOOLS)
        if sliver_tools is not None:
            logging.info('Sliver tools found in memcache (%s results).',
                         len(sliver_tools))
            candidates = []
            for sliver_tool in sliver_tools:
                if sliver_tool.site_id in site_id_list and \
                    ((address_family == message.ADDRESS_FAMILY_IPv4 and
                    sliver_tool.status_ipv4 == message.STATUS_ONLINE) or
                    (address_family == message.ADDRESS_FAMILY_IPv6 and
                    sliver_tool.status_ipv6 == message.STATUS_ONLINE)):
                    candidates.append(sliver_tool)
            return candidates
        logging.info('Sliver tools not found in memcache.')

        # Get the sliver tools from datastore.
        status_field = 'status_' + address_family
        candidates = model.SliverTool.gql(
            'WHERE tool_id = :tool_id '
            'AND ' + status_field + ' = :status '
            'AND site_id in :site_id_list',
            tool_id=query.tool_id,
            status=message.STATUS_ONLINE,
            site_id_list=site_id_list)
        logging.info('Found (%s candidates)', candidates.count())
        return candidates.fetch(constants.MAX_FETCHED_RESULTS)

    def answer_query(self, query):
        """Selects a random sliver tool among the available candidates.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity if any available, None otherwise.
        """
        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        return random.choice(candidates)

class GeoResolver(ResolverBase):
    """Chooses the server geographically closest to the client."""

    def answer_query(self, query):
        """Selects the geographically closest SliverTool.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity in case of success, or None if there is no
            SliverTool available that matches the query.
        """
        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        if (query.latitude is None) or (query.longitude is None):
            logging.warning('No latide/longitude, return a random sliver tool.')
            return random.choice(candidates)

        min_distance = float('+inf')
        closest_sliver_tools = []
        distances = {}

        # Compute for each SliverTool the distance and add keep in the
        # 'closest_sliver_tools' list only the SliverTools whose distance is
        # less or equal than the current minimum.
        for sliver_tool in candidates:
            # Check if we already computed the distance of this site.
            if distances.has_key(sliver_tool.site_id):
                current_distance = distances[sliver_tool.site_id]
            else:
                current_distance = distance.distance(
                    query.latitude,
                    query.longitude,
                    sliver_tool.latitude,
                    sliver_tool.longitude)

                distances[sliver_tool.site_id] = current_distance

            # Update the min distance and add the SliverTool to the list.
            if current_distance < min_distance:
                min_distance = current_distance
                closest_sliver_tools = [sliver_tool]
            elif current_distance == min_distance:
                closest_sliver_tools.append(sliver_tool)

        # Add the min_distance to the query so it can be logged later. Round to
        # the next highest kilometre radius to remove precision.
        query.distance = math.ceil(min_distance)

        # Choose randomly among candidates with the same, minimum distance.
        return random.choice(closest_sliver_tools)

class MetroResolver(ResolverBase):
    """Implements the metro policy."""

    def _get_candidates(self, query, address_family):
        # TODO(claudiu) Test whether the following query is better.
        # sites = model.Site.gql("WHERE metro = :metro", metro=query.metro)
        sites = model.Site.all().filter("metro =", query.metro).fetch(
            constants.MAX_FETCHED_RESULTS)
        logging.info(
            'Found %s results for metro %s.', len(sites), query.metro)
        if len(sites) == 0:
            logging.info('No results found for metro %s.', query.metro)
            return []

        site_id_list = []
        for site in sites:
            site_id_list.append(site.site_id)

        return self._get_candidates_from_sites(
            query, address_family, site_id_list)

class RandomResolver(ResolverBase):
    """Returns a server chosen randomly."""
    pass

class CountryResolver(ResolverBase):
    """Returns a server in a specified country."""

    def answer_query(self, query):
        """Returns a SliverTool in a specified country.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity if available, None otherwise.
        """
        if query.user_defined_country is None:
            return None

        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        country_candidates = []
        for candidate in candidates:
            if candidate.country == query.user_defined_country:
                country_candidates.append(candidate)

        if len(country_candidates) == 0:
            return None
        return random.choice(country_candidates)

def new_resolver(policy):
    if policy == message.POLICY_GEO:
        return GeoResolver()
    elif policy == message.POLICY_METRO:
        return MetroResolver()
    elif policy == message.POLICY_RANDOM:
        return RandomResolver()
    elif policy == message.POLICY_COUNTRY:
        return CountryResolver()
    else:
        return RandomResolver()
<commit_msg>Update resolver to return lists.<commit_after>from google.appengine.api import memcache

from mlabns.db import model
from mlabns.util import constants
from mlabns.util import distance
from mlabns.util import message

import logging
import math
import random

class ResolverBase:
    """Resolver base class."""

    def get_candidates(self, query):
        """Find candidates for server selection.

        Args:
            query: A LookupQuery instance.

        Returns:
            A list of SliverTool entities that match the requirements
            specified in the 'query'.
        """
        candidates = []
        if query.address_family is not None:
            candidates = self._get_candidates(query, query.address_family)
        # If no candidates with this address family and if this address family
        # was not user-defined, try the other address family.
        if len(candidates) == 0 and \
            query.address_family != query.user_defined_af:
            if query.address_family == message.ADDRESS_FAMILY_IPv4:
                candidates = self._get_candidates(query,
                                                  message.ADDRESS_FAMILY_IPv6)
            elif query.address_family == message.ADDRESS_FAMILY_IPv6:
                candidates = self._get_candidates(query,
                                                  message.ADDRESS_FAMILY_IPv4)
        if (query.sieves):
            for sieve in query.sieves:
                candidates = sieve.sieve(candidates)
        return candidates

    def _get_candidates(self, query, address_family):
        """Returns a (possibly empty) list of available candidates."""

        # First try to get the sliver tools from the memcache.
        sliver_tools = memcache.get(
            query.tool_id, namespace=constants.MEMCACHE_NAMESPACE_TOOLS)
        if sliver_tools is not None:
            logging.info('Sliver tools found in memcache (%s results).',
                         len(sliver_tools))
            candidates = []
            for sliver_tool in sliver_tools:
                if (address_family == message.ADDRESS_FAMILY_IPv4 and
                    sliver_tool.status_ipv4 == message.STATUS_ONLINE) or \
                    (address_family == message.ADDRESS_FAMILY_IPv6 and
                    sliver_tool.status_ipv6 == message.STATUS_ONLINE):
                    candidates.append(sliver_tool)
            return candidates
        logging.info('Sliver tools not found in memcache.')

        # Get the sliver tools from datastore.
        status_field = 'status_' + address_family
        candidates = model.SliverTool.gql(
            'WHERE tool_id = :tool_id '
            'AND ' + status_field + ' = :status',
            tool_id=query.tool_id,
            status=message.STATUS_ONLINE)
        logging.info('Found (%s candidates)', candidates.count())
        return candidates.fetch(constants.MAX_FETCHED_RESULTS)

    def _get_candidates_from_sites(self, query, address_family, site_id_list):
        """Returns a (possibly empty) list of available candidates."""

        # First try to get the sliver tools from the cache.
        sliver_tools = memcache.get(
            query.tool_id, namespace=constants.MEMCACHE_NAMESPACE_TOOLS)
        if sliver_tools is not None:
            logging.info('Sliver tools found in memcache (%s results).',
                         len(sliver_tools))
            candidates = []
            for sliver_tool in sliver_tools:
                if sliver_tool.site_id in site_id_list and \
                    ((address_family == message.ADDRESS_FAMILY_IPv4 and
                    sliver_tool.status_ipv4 == message.STATUS_ONLINE) or
                    (address_family == message.ADDRESS_FAMILY_IPv6 and
                    sliver_tool.status_ipv6 == message.STATUS_ONLINE)):
                    candidates.append(sliver_tool)
            return candidates
        logging.info('Sliver tools not found in memcache.')

        # Get the sliver tools from datastore.
        status_field = 'status_' + address_family
        candidates = model.SliverTool.gql(
            'WHERE tool_id = :tool_id '
            'AND ' + status_field + ' = :status '
            'AND site_id in :site_id_list',
            tool_id=query.tool_id,
            status=message.STATUS_ONLINE,
            site_id_list=site_id_list)
        logging.info('Found (%s candidates)', candidates.count())
        return candidates.fetch(constants.MAX_FETCHED_RESULTS)

    def answer_query(self, query):
        """Selects a random sliver tool among the available candidates.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity if any available, None otherwise.
        """
        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        return [random.choice(candidates)]

class GeoResolver(ResolverBase):
    """Chooses the server geographically closest to the client."""

    def answer_query(self, query):
        """Selects the geographically closest SliverTool.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity in case of success, or None if there is no
            SliverTool available that matches the query.
        """
        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        if (query.latitude is None) or (query.longitude is None):
            logging.warning('No latide/longitude, return a random sliver tool.')
            return random.choice(candidates)

        min_distance = float('+inf')
        closest_sliver_tools = []
        distances = {}

        # Compute for each SliverTool the distance and add keep in the
        # 'closest_sliver_tools' list only the SliverTools whose distance is
        # less or equal than the current minimum.
        for sliver_tool in candidates:
            # Check if we already computed the distance of this site.
            if distances.has_key(sliver_tool.site_id):
                current_distance = distances[sliver_tool.site_id]
            else:
                current_distance = distance.distance(
                    query.latitude,
                    query.longitude,
                    sliver_tool.latitude,
                    sliver_tool.longitude)

                distances[sliver_tool.site_id] = current_distance

            # Update the min distance and add the SliverTool to the list.
            if current_distance < min_distance:
                min_distance = current_distance
                closest_sliver_tools = [sliver_tool]
            elif current_distance == min_distance:
                closest_sliver_tools.append(sliver_tool)

        # Add the min_distance to the query so it can be logged later. Round to
        # the next highest kilometre radius to remove precision.
        query.distance = math.ceil(min_distance)

        # Choose randomly among candidates with the same, minimum distance.
        return [random.choice(closest_sliver_tools)]

class MetroResolver(ResolverBase):
    """Implements the metro policy."""

    def _get_candidates(self, query, address_family):
        # TODO(claudiu) Test whether the following query is better.
        # sites = model.Site.gql("WHERE metro = :metro", metro=query.metro)
        sites = model.Site.all().filter("metro =", query.metro).fetch(
            constants.MAX_FETCHED_RESULTS)
        logging.info(
            'Found %s results for metro %s.', len(sites), query.metro)
        if len(sites) == 0:
            logging.info('No results found for metro %s.', query.metro)
            return []

        site_id_list = []
        for site in sites:
            site_id_list.append(site.site_id)

        return self._get_candidates_from_sites(
            query, address_family, site_id_list)

class RandomResolver(ResolverBase):
    """Returns a server chosen randomly."""
    pass

class CountryResolver(ResolverBase):
    """Returns a server in a specified country."""

    def answer_query(self, query):
        """Returns a SliverTool in a specified country.

        Args:
            query: A LookupQuery instance.

        Returns:
            A SliverTool entity if available, None otherwise.
        """
        if query.user_defined_country is None:
            return None

        candidates = self.get_candidates(query)
        if len(candidates) == 0:
            logging.error('No results found for %s.', query.tool_id)
            return None

        country_candidates = []
        for candidate in candidates:
            if candidate.country == query.user_defined_country:
                country_candidates.append(candidate)

        if len(country_candidates) == 0:
            return None
        return [random.choice(country_candidates)]

def new_resolver(policy):
    if policy == message.POLICY_GEO:
        return GeoResolver()
    elif policy == message.POLICY_METRO:
        return MetroResolver()
    elif policy == message.POLICY_RANDOM:
        return RandomResolver()
    elif policy == message.POLICY_COUNTRY:
        return CountryResolver()
    else:
        return RandomResolver()
<|endoftext|>


<commit_before>import configparser
import os.path
import sqlite3

import datetime
from distutils.dir_util import mkpath

LOG_FILE = '/var/log/messages.1'
BASE_DIR = '/var/tmp/data'
CONFIG   = '/path/to/config.ini'

def main():
    now = datetime.datetime.now()
    dir_path = '%d/%02d/%02d/%02d' % (now.year, now.month, now.day, now.hour)
    db_file = '%d.db' % (now.minute)
    db_path = '%s/%s/%s' % (BASE_DIR, dir_path, db_file)

    dir_path = '%s/%s' %(BASE_DIR, dir_path)
    mkpath(dir_path)

    if not os.path.exists(db_path):
        conn = sqlite3.connect(db_path)
        conn.execute("CREATE VIRTUAL TABLE SYSLOG USING FTS3(LOGS)");
        conn.close()

    conn = sqlite3.connect(db_path)
    conn.execute("PRAGMA SYNCHRONOUS = OFF");
    conn.execute("PRAGMA JOURNAL_MODE = MEMORY");

    with open(LOG_FILE) as fh:
        lines = [[line] for line in fh]

        try:
            conn.executemany('INSERT INTO SYSLOG VALUES( ? )', lines)
        except sqlite3.Error as e:
            print(e)
        else:
            conn.commit()

if __name__ == '__main__':
    config = configparser.ConfigParser()
    config.read(CONFIG)

    LOG_FILE = config['path']['log-file']
    BASE_DIR = config['path']['base-dir']

    main()
<commit_msg>add utf8 error handing<commit_after>import configparser
import os.path
import sqlite3

import datetime
from distutils.dir_util import mkpath

LOG_FILE = '/var/log/messages.1'
BASE_DIR = '/var/tmp/data'
CONFIG   = '/path/to/config.ini'

def main():
    now = datetime.datetime.now()
    dir_path = '%d/%02d/%02d/%02d' % (now.year, now.month, now.day, now.hour)
    db_file = '%d.db' % (now.minute)
    db_path = '%s/%s/%s' % (BASE_DIR, dir_path, db_file)

    dir_path = '%s/%s' %(BASE_DIR, dir_path)
    mkpath(dir_path)

    if not os.path.exists(db_path):
        conn = sqlite3.connect(db_path)
        conn.execute("CREATE VIRTUAL TABLE SYSLOG USING FTS3(LOGS)");
        conn.close()

    conn = sqlite3.connect(db_path)
    conn.execute("PRAGMA SYNCHRONOUS = OFF");
    conn.execute("PRAGMA JOURNAL_MODE = MEMORY");

    with open(LOG_FILE, "r", encoding="utf-8", errors='ignore') as fh:
        lines = [[line] for line in fh]

        try:
            conn.executemany('INSERT INTO SYSLOG VALUES( ? )', lines)
        except sqlite3.Error as e:
            print(e)
        else:
            conn.commit()

if __name__ == '__main__':
    config = configparser.ConfigParser()
    config.read(CONFIG)

    LOG_FILE = config['path']['log-file']
    BASE_DIR = config['path']['base-dir']

    main()
<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Eric
Credit: Me!

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
# Anoushka Alavilli
# Credits: http://stackoverflow.com/questions/4481724/convert-a-list-of-characters-into-a-string
text= input("Please enter a string of text (the bigger the better): ")
print ("You entered " + (str(text)) + ". Now jumble it: ")

    # separates each letter
listtext= list(str(text))
#print (listtext)
#print (listtext[4::-1])
#print (str(listtext[4::-1]))
print (''.join(listtext[4::-1]))

<commit_msg>Updated from Brython Server: 9/30/2015 9:20:13 PM<commit_after>"""
stringjumble.py
Author: Eric
Credit: Me!

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
# Anoushka Alavilli
# Credits: http://stackoverflow.com/questions/4481724/convert-a-list-of-characters-into-a-string
text= input("Please enter a string of text (the bigger the better): ")
print ("You entered " + (str(text)) + ". Now jumble it: ")

    # separates each letter
#listtext= list(str(text))
#print (listtext)
#print (listtext[4::-1])
#print (str(listtext[4::-1]))
print (''.join(listtext[4::-1]))

<|endoftext|>


<commit_before>"""
stringjumble.py
Author: Avery Wallis
Credit: Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))
l=(int(len(string)))
print('You entered "'+ string + '". Now jumble it:')
the = list(string)
spil = string.split(' ')
print()

for x in range(1,l+1):
    print(string[-x],end="")
print()

f=list(reversed(spil))
for word in f:
    print(word,end=" ")
print("")
print()

for word in spil:
    g=0
    while g< len(word):
        r=str(word[len(word)-g-1])
        print("{0}".format(r), end="")
        g=g+1
    print("",end=" ")<commit_msg>Updated from Brython Server: 2/12/2016 2:48:48 PM<commit_after>"""
stringjumble.py
Author: Avery Wallis
Credit: Ethan, Daniel

Assignment:

The purpose of this challenge is to gain proficiency with
manipulating lists.

Write and submit a Python program that accepts a string from
the user and prints it back in three different ways:

* With all letters in reverse.
* With words in reverse order, but letters within each word in
  the correct order.
* With all words in correct order, but letters reversed within
  the words.

Output of your program should look like this:

Please enter a string of text (the bigger the better): There are a few techniques or tricks that you may find handy
You entered "There are a few techniques or tricks that you may find handy". Now jumble it:
ydnah dnif yam uoy taht skcirt ro seuqinhcet wef a era erehT
handy find may you that tricks or techniques few a are There
erehT era a wef seuqinhcet ro skcirt taht uoy yam dnif ydnah
"""
string= str(input("Please enter a string of text (the bigger the better): "))
l=(int(len(string)))
print('You entered "'+ string + '". Now jumble it:')
the = list(string)
spil = string.split(' ')
print()

for x in range(1,l+1):
    print(string[-x],end="")
print()

f=list(reversed(spil))
for word in f:
    print(word,end=" ")
print("")
print()
for word in spil:
    g=0
    while g< len(word):
        r=str(word[len(word)-g-1])
        print("{0}".format(r), end="")
        g=g+1
    print("",end=" ")<|endoftext|>


<commit_before>__version_info__ = {
    'major': 0,
    'minor': 7,
    'micro': 1,
    'releaselevel': 'final',
    'serial': 1
}

def get_version(short=False):
    assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final')
    vers = ["%(major)i.%(minor)i" % __version_info__, ]
    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final' and not short:
        vers.append('%s%i' % (__version_info__['releaselevel'][0], __version_info__['serial']))
    return ''.join(vers)

__version__ = get_version()
<commit_msg>0.7.2 Bump<commit_after>__version_info__ = {
    'major': 0,
    'minor': 7,
    'micro': 2,
    'releaselevel': 'beta',
    'serial': 1
}

def get_version(short=False):
    assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final')
    vers = ["%(major)i.%(minor)i" % __version_info__, ]
    if __version_info__['micro']:
        vers.append(".%(micro)i" % __version_info__)
    if __version_info__['releaselevel'] != 'final' and not short:
        vers.append('%s%i' % (__version_info__['releaselevel'][0], __version_info__['serial']))
    return ''.join(vers)

__version__ = get_version()
<|endoftext|>


<commit_before># Author: Enric Tejedor CERN  01/2019

################################################################################
# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers.                      #
# All rights reserved.                                                         #
#                                                                              #
# For the licensing terms see $ROOTSYS/LICENSE.                                #
# For the list of contributors see $ROOTSYS/README/CREDITS.                    #
################################################################################

from ROOT import pythonization
from cppyy.gbl import TIter

from ._generic import add_len

# Python-list-like methods

def remove_pyz(self, o):
	# Parameters:
    # self: collection
    # o: object to remove from the collection
	res = self.Remove(o)

	if not res:
		raise ValueError('list.remove(x): x not in list')

def extend_pyz(self, c):
	# Parameters:
    # self: collection
    # c: collection to extend self with
    lenc = c.GetEntries()
    it = TIter(c)
    for i in range(lenc):
    	self.Add(it.Next())

def count_pyz(self, o):
	# Parameters:
    # self: collection
    # o: object to be counted in the collection
	n = 0

	it = TIter(self)
	obj = it.Next()
	while obj:
		if obj == o:
			n += 1
		obj = it.Next()

	return n

# Python operators

def add_pyz(self, c):
	# Parameters:
    # - self: first collection to be added
    # - c: second collection to be added
    # Returns:
    # - self + c
	res = self.__class__()
	extend_pyz(res, self)
	extend_pyz(res, c)
	return res

def mul_pyz(self, n):
	# Parameters:
    # - self: collection to be multiplied
    # - n: factor to multiply the collection by
    # Returns:
    # - self * n
	res = self.__class__()
	for _ in range(n):
		extend_pyz(res, self)
	return res

def imul_pyz(self, n):
	# Parameters:
    # - self: collection to be multiplied (in place)
    # - n: factor to multiply the collection by
    # Returns:
    # - self *= n
	for _ in range(n - 1):
		extend_pyz(self, self)
	return self

@pythonization()
def pythonize_tcollection(klass, name):
    # Parameters:
    # klass: class to be pythonized
    # name: string containing the name of the class

    if name == 'TCollection':
        # Support `len(c)` as `c.GetEntries()`
        add_len(klass, 'GetEntries')

        # Add Python lists methods
        klass.append = klass.Add
        klass.remove = remove_pyz
        klass.extend = extend_pyz
        klass.count = count_pyz

        # Define Python operators
        klass.__add__ = add_pyz
        klass.__mul__ = mul_pyz
        klass.__rmul__ = mul_pyz
        klass.__imul__ = imul_pyz
<commit_msg>[Exp PyROOT] Improve docs of TCollection pythonisations<commit_after># Author: Enric Tejedor CERN  01/2019

################################################################################
# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers.                      #
# All rights reserved.                                                         #
#                                                                              #
# For the licensing terms see $ROOTSYS/LICENSE.                                #
# For the list of contributors see $ROOTSYS/README/CREDITS.                    #
################################################################################

from ROOT import pythonization
from cppyy.gbl import TIter

from ._generic import add_len

# Python-list-like methods

def remove_pyz(self, o):
	# Parameters:
    # - self: collection
    # - o: object to remove from the collection
	res = self.Remove(o)

	if not res:
		raise ValueError('list.remove(x): x not in list')

def extend_pyz(self, c):
	# Parameters:
    # - self: collection
    # - c: collection to extend self with
    lenc = c.GetEntries()
    it = TIter(c)
    for i in range(lenc):
    	self.Add(it.Next())

def count_pyz(self, o):
	# Parameters:
    # - self: collection
    # - o: object to be counted in the collection
    # Returns:
    # - Number of occurrences of the object in the collection
	n = 0

	it = TIter(self)
	obj = it.Next()
	while obj:
		if obj == o:
			n += 1
		obj = it.Next()

	return n

# Python operators

def add_pyz(self, c):
	# Parameters:
    # - self: first collection to be added
    # - c: second collection to be added
    # Returns:
    # - self + c
	res = self.__class__()
	extend_pyz(res, self)
	extend_pyz(res, c)
	return res

def mul_pyz(self, n):
	# Parameters:
    # - self: collection to be multiplied
    # - n: factor to multiply the collection by
    # Returns:
    # - self * n
	res = self.__class__()
	for _ in range(n):
		extend_pyz(res, self)
	return res

def imul_pyz(self, n):
	# Parameters:
    # - self: collection to be multiplied (in place)
    # - n: factor to multiply the collection by
    # Returns:
    # - self *= n
	for _ in range(n - 1):
		extend_pyz(self, self)
	return self

@pythonization()
def pythonize_tcollection(klass, name):
    # Parameters:
    # klass: class to be pythonized
    # name: string containing the name of the class

    if name == 'TCollection':
        # Support `len(c)` as `c.GetEntries()`
        add_len(klass, 'GetEntries')

        # Add Python lists methods
        klass.append = klass.Add
        klass.remove = remove_pyz
        klass.extend = extend_pyz
        klass.count = count_pyz

        # Define Python operators
        klass.__add__ = add_pyz
        klass.__mul__ = mul_pyz
        klass.__rmul__ = mul_pyz
        klass.__imul__ = imul_pyz
<|endoftext|>


<commit_before># Copyright 2014 Red Hat
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log as logging
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import decorators
from tempest import test

from ceilometer.tests.tempest.service import client

CONF = config.CONF

LOG = logging.getLogger(__name__)

class ClientManager(client.Manager):

    load_clients = [
        'telemetry_client',
        'container_client',
        'object_client',
    ]

class TestObjectStorageTelemetry(test.BaseTestCase):
    """Test that swift uses the ceilometer middleware.

     * create container.
     * upload a file to the created container.
     * retrieve the file from the created container.
     * wait for notifications from ceilometer.
    """

    credentials = ['primary']
    client_manager = ClientManager

    @classmethod
    def skip_checks(cls):
        super(TestObjectStorageTelemetry, cls).skip_checks()
        if ("gnocchi" in CONF.service_available and
                CONF.service_available.gnocchi):
            skip_msg = ("%s skipped as gnocchi is enabled" %
                        cls.__name__)
            raise cls.skipException(skip_msg)
        if not CONF.service_available.swift:
            skip_msg = ("%s skipped as swift is not available" %
                        cls.__name__)
            raise cls.skipException(skip_msg)
        if not CONF.service_available.ceilometer:
            skip_msg = ("%s skipped as ceilometer is not available" %
                        cls.__name__)
            raise cls.skipException(skip_msg)

    @classmethod
    def setup_credentials(cls):
        cls.set_network_resources()
        super(TestObjectStorageTelemetry, cls).setup_credentials()

    @classmethod
    def setup_clients(cls):
        super(TestObjectStorageTelemetry, cls).setup_clients()
        cls.telemetry_client = cls.os_primary.telemetry_client
        cls.container_client = cls.os_primary.container_client
        cls.object_client = cls.os_primary.object_client

    def _confirm_notifications(self, container_name, obj_name):
        # NOTE: Loop seeking for appropriate notifications about the containers
        # and objects sent to swift.

        def _check_samples():
            # NOTE: Return True only if we have notifications about some
            # containers and some objects and the notifications are about
            # the expected containers and objects.
            # Otherwise returning False will case _check_samples to be
            # called again.
            results = self.telemetry_client.list_samples(
                'storage.objects.incoming.bytes')
            LOG.debug('got samples %s', results)

            # Extract container info from samples.
            containers, objects = [], []
            for sample in results:
                meta = sample['resource_metadata']
                if meta.get('container') and meta['container'] != 'None':
                    containers.append(meta['container'])
                elif (meta.get('target.metadata:container') and
                      meta['target.metadata:container'] != 'None'):
                    containers.append(meta['target.metadata:container'])

                if meta.get('object') and meta['object'] != 'None':
                    objects.append(meta['object'])
                elif (meta.get('target.metadata:object') and
                      meta['target.metadata:object'] != 'None'):
                    objects.append(meta['target.metadata:object'])

            return (container_name in containers and obj_name in objects)

        self.assertTrue(
            test_utils.call_until_true(_check_samples,
                                       CONF.telemetry.notification_wait,
                                       CONF.telemetry.notification_sleep),
            'Correct notifications were not received after '
            '%s seconds.' % CONF.telemetry.notification_wait)

    def create_container(self):
        name = data_utils.rand_name('swift-scenario-container')
        self.container_client.create_container(name)
        # look for the container to assure it is created
        self.container_client.list_container_contents(name)
        LOG.debug('Container %s created' % (name))
        self.addCleanup(self.container_client.delete_container,
                        name)
        return name

    def upload_object_to_container(self, container_name):
        obj_name = data_utils.rand_name('swift-scenario-object')
        obj_data = data_utils.arbitrary_string()
        self.object_client.create_object(container_name, obj_name, obj_data)
        self.addCleanup(self.object_client.delete_object,
                        container_name,
                        obj_name)
        return obj_name

    @decorators.idempotent_id('6d6b88e5-3e38-41bc-b34a-79f713a6cb85')
    @test.services('object_storage')
    def test_swift_middleware_notifies(self):
        container_name = self.create_container()
        obj_name = self.upload_object_to_container(container_name)
        self._confirm_notifications(container_name, obj_name)
<commit_msg>handle new tempest swift interface<commit_after># Copyright 2014 Red Hat
#
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_log import log as logging
from tempest import config
from tempest.lib.common.utils import data_utils
from tempest.lib.common.utils import test_utils
from tempest.lib import decorators
from tempest import test

from ceilometer.tests.tempest.service import client

CONF = config.CONF

LOG = logging.getLogger(__name__)

class ClientManager(client.Manager):

    load_clients = [
        'telemetry_client',
        'container_client',
        'object_client',
    ]

class TestObjectStorageTelemetry(test.BaseTestCase):
    """Test that swift uses the ceilometer middleware.

     * create container.
     * upload a file to the created container.
     * retrieve the file from the created container.
     * wait for notifications from ceilometer.
    """

    credentials = ['primary']
    client_manager = ClientManager

    @classmethod
    def skip_checks(cls):
        super(TestObjectStorageTelemetry, cls).skip_checks()
        if ("gnocchi" in CONF.service_available and
                CONF.service_available.gnocchi):
            skip_msg = ("%s skipped as gnocchi is enabled" %
                        cls.__name__)
            raise cls.skipException(skip_msg)
        if not CONF.service_available.swift:
            skip_msg = ("%s skipped as swift is not available" %
                        cls.__name__)
            raise cls.skipException(skip_msg)
        if not CONF.service_available.ceilometer:
            skip_msg = ("%s skipped as ceilometer is not available" %
                        cls.__name__)
            raise cls.skipException(skip_msg)

    @classmethod
    def setup_credentials(cls):
        cls.set_network_resources()
        super(TestObjectStorageTelemetry, cls).setup_credentials()

    @classmethod
    def setup_clients(cls):
        super(TestObjectStorageTelemetry, cls).setup_clients()
        cls.telemetry_client = cls.os_primary.telemetry_client
        cls.container_client = cls.os_primary.container_client
        cls.object_client = cls.os_primary.object_client

    def _confirm_notifications(self, container_name, obj_name):
        # NOTE: Loop seeking for appropriate notifications about the containers
        # and objects sent to swift.

        def _check_samples():
            # NOTE: Return True only if we have notifications about some
            # containers and some objects and the notifications are about
            # the expected containers and objects.
            # Otherwise returning False will case _check_samples to be
            # called again.
            results = self.telemetry_client.list_samples(
                'storage.objects.incoming.bytes')
            LOG.debug('got samples %s', results)

            # Extract container info from samples.
            containers, objects = [], []
            for sample in results:
                meta = sample['resource_metadata']
                if meta.get('container') and meta['container'] != 'None':
                    containers.append(meta['container'])
                elif (meta.get('target.metadata:container') and
                      meta['target.metadata:container'] != 'None'):
                    containers.append(meta['target.metadata:container'])

                if meta.get('object') and meta['object'] != 'None':
                    objects.append(meta['object'])
                elif (meta.get('target.metadata:object') and
                      meta['target.metadata:object'] != 'None'):
                    objects.append(meta['target.metadata:object'])

            return (container_name in containers and obj_name in objects)

        self.assertTrue(
            test_utils.call_until_true(_check_samples,
                                       CONF.telemetry.notification_wait,
                                       CONF.telemetry.notification_sleep),
            'Correct notifications were not received after '
            '%s seconds.' % CONF.telemetry.notification_wait)

    def create_container(self):
        name = data_utils.rand_name('swift-scenario-container')
        self.container_client.create_container(name)
        # look for the container to assure it is created
        self.container_client.list_container_objects(name)
        LOG.debug('Container %s created' % (name))
        self.addCleanup(self.container_client.delete_container,
                        name)
        return name

    def upload_object_to_container(self, container_name):
        obj_name = data_utils.rand_name('swift-scenario-object')
        obj_data = data_utils.arbitrary_string()
        self.object_client.create_object(container_name, obj_name, obj_data)
        self.addCleanup(self.object_client.delete_object,
                        container_name,
                        obj_name)
        return obj_name

    @decorators.idempotent_id('6d6b88e5-3e38-41bc-b34a-79f713a6cb85')
    @test.services('object_storage')
    def test_swift_middleware_notifies(self):
        container_name = self.create_container()
        obj_name = self.upload_object_to_container(container_name)
        self._confirm_notifications(container_name, obj_name)
<|endoftext|>


<commit_before>import copy
from os.path import join
from types import DictType

from .vendor.etcaetera.config import Config as EtcConfig
from .vendor.etcaetera.adapter import File, Defaults, Overrides, Env

def noop(s):
    """
    No-op 'formatter' for etcaetera adapters.

    For when we do not want auto upper/lower casing (currently, always).
    """
    return s

class DataProxy(object):
    """
    Helper class implementing nested dict+attr access for `.Config`.
    """

    # Attributes which get proxied through to inner etc.Config obj.
    _proxies = tuple("""
        clear
        get
        has_key
        items
        iteritems
        iterkeys
        itervalues
        keys
        pop
        popitem
        setdefault
        update
        values
    """.split()) + tuple("__{0}__".format(x) for x in """
        cmp
        contains
        iter
        sizeof
    """.split())

    # Alt constructor used so we aren't getting in the way of Config's real
    # __init__().
    @classmethod
    def from_data(cls, data):
        obj = cls()
        obj.config = data
        return obj

    def __getattr__(self, key):
        try:
            return self._get(key)
        except KeyError:
            # Proxy most special vars to config for dict procotol.
            if key in self._proxies:
                return getattr(self.config, key)
            # Otherwise, raise useful AttributeError to follow getattr proto.
            err = "No attribute or config key found for {0!r}".format(key)
            attrs = [x for x in dir(self.__class__) if not x.startswith('_')]
            err += "\n\nValid real attributes: {0!r}".format(attrs)
            err += "\n\nValid keys: {0!r}".format(self.config.keys())
            raise AttributeError(err)

    def __hasattr__(self, key):
        return key in self.config or key in self._proxies

    def __iter__(self):
        # For some reason Python is ignoring our __hasattr__ when determining
        # whether we support __iter__. BOO
        return iter(self.config)

    def __eq__(self, other):
        # Can't proxy __eq__ because the RHS will always be an obj of the
        # current class, not the proxied-to class, and that causes
        # NotImplemented.
        return self.config == other.config

    def __len__(self):
        # Can't proxy __len__ either apparently? ugh
        return len(self.config)

    def __setitem__(self, key, value):
        # ... or __setitem__? thanks for nothing Python >:(
        self.config[key] = value

    def __delitem__(self, key):
        # OK this is really getting annoying
        del self.config[key]

    def __getitem__(self, key):
        return self._get(key)

    def _get(self, key):
        value = self.config[key]
        if isinstance(value, DictType):
            value = DataProxy.from_data(value)
        return value

    # TODO: copy()?

class Config(DataProxy):
    """
    Invoke's primary configuration handling class.

    See :doc:`/concepts/configuration` for details on the configuration system
    this class implements, including the :ref:`configuration hierarchy
    <config-hierarchy>`.

    This class lightly wraps ``etcaetera.config.Config``, allowing for another
    level of configurability (re: which files are loaded and in what order) as
    well as convenient access to configuration values, which may be accessed
    using dict syntax::

        config['foo']

    or attribute syntax::

        config.foo

    .. warning::
        Any "real" attributes (methods, etc) on `Config` take precedence over
        settings values - so if you e.g. have a top level setting named
        ``load``, you *must* use dict syntax to access it.

    Nesting works the same way - dict config values are turned into objects
    which honor both the dictionary protocol and the attribute-access method::

       config['foo']['bar']
       config.foo.bar

    This class implements the entire dictionary protocol.
    """

    def __init__(self, overrides=None, global_prefix=None, user_prefix=None, project_home=None, runtime_path=None, adapters=None):
        """
        Creates a new config object, but does not load any configuration data.

        .. note::
            To load configuration data, call `~.Config.load` after
            initialization.

        :param dict overrides:
            A dict containing override-level config data. Default: ``{}``.

        :param str global_prefix:
            Path & partial filename for the global config file location. Should
            include everything but the dot & file extension.

            Default: ``/etc/invoke`` (e.g. ``/etc/invoke.yaml`` or
            ``/etc/invoke.json``).

        :param str user_prefix:
            Like ``global_prefix`` but for the per-user config file.

            Default: ``~/.invoke`` (e.g. ``~/.invoke.yaml``).

        :param str project_home:
            Optional directory path location of the currently loaded
            `.Collection` (as loaded by `.Loader`). When non-empty, will
            trigger seeking of per-project config files in this location +
            ``invoke.(yaml|json|py)``.

        :param str runtime_path:
            Optional file path to a runtime configuration file.

            Used to fill the penultimate slot in the config hierarchy. Should
            be a full file path to an existing file, not a directory path, or a
            prefix.

        :param iterable adapters:
            An iterable of `Adapters` to use instead of the default
            :ref:`hierarchy <config-hierarchy>`.

            If this option is given, ``global_prefix`` and ``user_prefix`` will
            be ignored.

        .. _Adapters: http://etcaetera.readthedocs.org/en/0.4.0/howto.html#adapters
        """
        # Setup
        if overrides is None:
            overrides = {}
        if global_prefix is None:
            global_prefix = '/etc/invoke'
        if user_prefix is None:
            user_prefix = '~/.invoke'
        c = EtcConfig(formatter=noop)
        # Explicit adapter set
        if adapters is not None:
            c.register(*adapters)
        # The Hierarchy
        else:
            # Level 1 is Defaults, set via kwargs or client calling
            # set_defaults(). Normally comes from task collection tree.
            # Levels 2-4: global, user, & project config files
            prefixes = [global_prefix, user_prefix]
            if project_home is not None:
                prefixes.append(join(project_home, "invoke"))
            for prefix in prefixes:
                c.register(File("{0}.yaml".format(prefix)))
                c.register(File("{0}.json".format(prefix)))
                py = File("{0}.py".format(prefix), python_uppercase=False)
                c.register(py)
            # Level 5: environment variables. See `load()` - must be done as
            # late as possible to 'see' all other defined keys.
            # Level 6: Runtime config file
            if runtime_path is not None:
                # Give python_uppercase in case it's a .py. Is a safe no-op
                # otherwise.
                c.register(File(runtime_path, python_uppercase=False))
            # Level 7 is Overrides, typically runtime flag values
            c.register(Overrides(overrides, formatter=noop))
        # Assign to member
        self.config = c

    def load(self, defaults=None):
        """
        Performs loading and merging of all config sources.

        See :ref:`config-hierarchy` for details on load order and file
        locations.

        :param dict defaults:
            A dict containing defaults-level config data. Default: ``{}``.
        """
        # Pull in defaults (we do so at this level because typically we won't
        # know about collection-level default values until closer to runtime.
        if defaults is None:
            defaults = {}
        # Reinforce 'noop' here, Defaults calls load() in init()
        self.config.register(Defaults(defaults, formatter=noop))
        # Now that we have all other sources defined, we can load the Env
        # adapter primed with all known config keys.
        self.config.register(Env(*self._env_keys()))
        # TODO: fix up register order?? Probably just need to actually-register
        # Env prior to registering the runtime config file...
        return self.config.load()

    def clone(self):
        """
        Return a copy of this configuration object.

        The new object will be identical in terms of data, but will be a
        distinct object with no shared mutable state.
        """
        # New config w/ formatter preserved
        c = EtcConfig(formatter=self.config.formatter)
        # New adapters, ditto + deepcopy internal data
        adapters = []
        for old in self.config.adapters:
            c.register(_clone_adapter(old))
        # Then deepcopy loaded data in case already loaded (we don't
        # necessarily know at this point if loading has occurred).
        c.update(copy.deepcopy(dict(self.config)))
        # All set
        new = Config()
        new.config = c
        return new

    def _env_keys(self):
        """
        Return list of valid configuration keys for use in an ``Env`` adapter.
        """
        return []

def _clone_adapter(old):
    if isinstance(old, (Defaults, Overrides)):
        new = old.__class__(formatter=old.formatter)
    elif isinstance(old, File):
        new = File(
            filepath=old.filepath,
            python_uppercase=old.python_uppercase,
            formatter=old.formatter,
        )
    elif isinstance(old, Env):
        new = Env(*old.keys)
        new.formatter = old.formatter # not available as init kwarg
    new.data = copy.deepcopy(old.data)
    return new
<commit_msg>Keys load now, need to lowercase for consistency<commit_after>import copy
from os.path import join
from types import DictType

from .vendor.etcaetera.config import Config as EtcConfig
from .vendor.etcaetera.adapter import File, Defaults, Overrides, Env

def noop(s):
    """
    No-op 'formatter' for etcaetera adapters.

    For when we do not want auto upper/lower casing (currently, always).
    """
    return s

class DataProxy(object):
    """
    Helper class implementing nested dict+attr access for `.Config`.
    """

    # Attributes which get proxied through to inner etc.Config obj.
    _proxies = tuple("""
        clear
        get
        has_key
        items
        iteritems
        iterkeys
        itervalues
        keys
        pop
        popitem
        setdefault
        update
        values
    """.split()) + tuple("__{0}__".format(x) for x in """
        cmp
        contains
        iter
        sizeof
    """.split())

    # Alt constructor used so we aren't getting in the way of Config's real
    # __init__().
    @classmethod
    def from_data(cls, data):
        obj = cls()
        obj.config = data
        return obj

    def __getattr__(self, key):
        try:
            return self._get(key)
        except KeyError:
            # Proxy most special vars to config for dict procotol.
            if key in self._proxies:
                return getattr(self.config, key)
            # Otherwise, raise useful AttributeError to follow getattr proto.
            err = "No attribute or config key found for {0!r}".format(key)
            attrs = [x for x in dir(self.__class__) if not x.startswith('_')]
            err += "\n\nValid real attributes: {0!r}".format(attrs)
            err += "\n\nValid keys: {0!r}".format(self.config.keys())
            raise AttributeError(err)

    def __hasattr__(self, key):
        return key in self.config or key in self._proxies

    def __iter__(self):
        # For some reason Python is ignoring our __hasattr__ when determining
        # whether we support __iter__. BOO
        return iter(self.config)

    def __eq__(self, other):
        # Can't proxy __eq__ because the RHS will always be an obj of the
        # current class, not the proxied-to class, and that causes
        # NotImplemented.
        return self.config == other.config

    def __len__(self):
        # Can't proxy __len__ either apparently? ugh
        return len(self.config)

    def __setitem__(self, key, value):
        # ... or __setitem__? thanks for nothing Python >:(
        self.config[key] = value

    def __delitem__(self, key):
        # OK this is really getting annoying
        del self.config[key]

    def __getitem__(self, key):
        return self._get(key)

    def _get(self, key):
        value = self.config[key]
        if isinstance(value, DictType):
            value = DataProxy.from_data(value)
        return value

    # TODO: copy()?

class Config(DataProxy):
    """
    Invoke's primary configuration handling class.

    See :doc:`/concepts/configuration` for details on the configuration system
    this class implements, including the :ref:`configuration hierarchy
    <config-hierarchy>`.

    This class lightly wraps ``etcaetera.config.Config``, allowing for another
    level of configurability (re: which files are loaded and in what order) as
    well as convenient access to configuration values, which may be accessed
    using dict syntax::

        config['foo']

    or attribute syntax::

        config.foo

    .. warning::
        Any "real" attributes (methods, etc) on `Config` take precedence over
        settings values - so if you e.g. have a top level setting named
        ``load``, you *must* use dict syntax to access it.

    Nesting works the same way - dict config values are turned into objects
    which honor both the dictionary protocol and the attribute-access method::

       config['foo']['bar']
       config.foo.bar

    This class implements the entire dictionary protocol.
    """

    def __init__(self, overrides=None, global_prefix=None, user_prefix=None, project_home=None, runtime_path=None, adapters=None):
        """
        Creates a new config object, but does not load any configuration data.

        .. note::
            To load configuration data, call `~.Config.load` after
            initialization.

        :param dict overrides:
            A dict containing override-level config data. Default: ``{}``.

        :param str global_prefix:
            Path & partial filename for the global config file location. Should
            include everything but the dot & file extension.

            Default: ``/etc/invoke`` (e.g. ``/etc/invoke.yaml`` or
            ``/etc/invoke.json``).

        :param str user_prefix:
            Like ``global_prefix`` but for the per-user config file.

            Default: ``~/.invoke`` (e.g. ``~/.invoke.yaml``).

        :param str project_home:
            Optional directory path location of the currently loaded
            `.Collection` (as loaded by `.Loader`). When non-empty, will
            trigger seeking of per-project config files in this location +
            ``invoke.(yaml|json|py)``.

        :param str runtime_path:
            Optional file path to a runtime configuration file.

            Used to fill the penultimate slot in the config hierarchy. Should
            be a full file path to an existing file, not a directory path, or a
            prefix.

        :param iterable adapters:
            An iterable of `Adapters` to use instead of the default
            :ref:`hierarchy <config-hierarchy>`.

            If this option is given, ``global_prefix`` and ``user_prefix`` will
            be ignored.

        .. _Adapters: http://etcaetera.readthedocs.org/en/0.4.0/howto.html#adapters
        """
        # Setup
        if overrides is None:
            overrides = {}
        if global_prefix is None:
            global_prefix = '/etc/invoke'
        if user_prefix is None:
            user_prefix = '~/.invoke'
        c = EtcConfig(formatter=noop)
        # Explicit adapter set
        if adapters is not None:
            c.register(*adapters)
        # The Hierarchy
        else:
            # Level 1 is Defaults, set via kwargs or client calling
            # set_defaults(). Normally comes from task collection tree.
            # Levels 2-4: global, user, & project config files
            prefixes = [global_prefix, user_prefix]
            if project_home is not None:
                prefixes.append(join(project_home, "invoke"))
            for prefix in prefixes:
                c.register(File("{0}.yaml".format(prefix)))
                c.register(File("{0}.json".format(prefix)))
                py = File("{0}.py".format(prefix), python_uppercase=False)
                c.register(py)
            # Level 5: environment variables. See `load()` - must be done as
            # late as possible to 'see' all other defined keys.
            # Level 6: Runtime config file
            if runtime_path is not None:
                # Give python_uppercase in case it's a .py. Is a safe no-op
                # otherwise.
                c.register(File(runtime_path, python_uppercase=False))
            # Level 7 is Overrides, typically runtime flag values
            c.register(Overrides(overrides, formatter=noop))
        # Assign to member
        self.config = c

    def load(self, defaults=None):
        """
        Performs loading and merging of all config sources.

        See :ref:`config-hierarchy` for details on load order and file
        locations.

        :param dict defaults:
            A dict containing defaults-level config data. Default: ``{}``.
        """
        # Pull in defaults (we do so at this level because typically we won't
        # know about collection-level default values until closer to runtime.
        if defaults is None:
            defaults = {}
        # Reinforce 'noop' here, Defaults calls load() in init()
        self.config.register(Defaults(defaults, formatter=noop))
        # Now that we have all other sources defined, we can load the Env
        # adapter primed with all known config keys.
        env = Env(*self._env_keys())
        self.config.register(env)
        # TODO: fix up register order?? Probably just need to actually-register
        # Env prior to registering the runtime config file...
        return self.config.load()

    def clone(self):
        """
        Return a copy of this configuration object.

        The new object will be identical in terms of data, but will be a
        distinct object with no shared mutable state.
        """
        # New config w/ formatter preserved
        c = EtcConfig(formatter=self.config.formatter)
        # New adapters, ditto + deepcopy internal data
        adapters = []
        for old in self.config.adapters:
            c.register(_clone_adapter(old))
        # Then deepcopy loaded data in case already loaded (we don't
        # necessarily know at this point if loading has occurred).
        c.update(copy.deepcopy(dict(self.config)))
        # All set
        new = Config()
        new.config = c
        return new

    def _env_keys(self):
        """
        Return list of valid configuration keys for use in an ``Env`` adapter.
        """
        keys = []
        # TODO: force pre-loading of File adapters :(
        for adapter in self.config.adapters:
            keys.extend(adapter.data.keys())
        return keys

def _clone_adapter(old):
    if isinstance(old, (Defaults, Overrides)):
        new = old.__class__(formatter=old.formatter)
    elif isinstance(old, File):
        new = File(
            filepath=old.filepath,
            python_uppercase=old.python_uppercase,
            formatter=old.formatter,
        )
    elif isinstance(old, Env):
        new = Env(*old.keys)
        new.formatter = old.formatter # not available as init kwarg
    new.data = copy.deepcopy(old.data)
    return new
<|endoftext|>