<commit_before>__author__    = 'Bojan Delic <bojan@delic.in.rs>'
__date__      = 'Aug 21, 2013'
__copyright__ = 'Copyright (c) 2013 Bojan Delic'

from functools import partial
from Queue import Queue
from threading import Lock
from collections import defaultdict
import wpf

from System import TimeSpan
from System.Windows.Input import ICommand
from System import EventArgs
from System.Windows.Threading import DispatcherTimer
from System.ComponentModel import INotifyPropertyChanged, PropertyChangedEventArgs
from System.Collections.ObjectModel import ObservableCollection

class MvvmError(Exception):
    pass

class _Messenger(object):
    '''
    Thread-safe messenger that ensures that all message handlers are executed
    in main dispatcher thread.

    This class should be used as singleton. It is not enforced, but recomanded
    way of getting instance is by using :meth:`_Messenger.instance` class
    method.
    '''

    _instance = None

    @classmethod
    def instance(cls, interval=5):
        '''
        Returns existing instance of messenger. If one does not exist it will
        be created and returned.

        :param int interval:
            Number of miliseconds that represents interval when messages will
            be processed.
            Note that this parameter will be used only the first time when
            instance is requested, every other time it will be ignored
            because existing instance of :class:`._Messenger` is returned.
        '''
        if not cls._instance:
            cls._instance = _Messenger(interval)
        return cls._instance

    def __init__(self, interval=5):
        '''
        :param int interval:
            Number of milliseconds that represents interval when messages will
            be processed.
        '''
        self._subscribers = defaultdict(list)
        self._messages = Queue()
        self._lock = Lock()
        self._timer = DispatcherTimer()
        self._timer.Interval = TimeSpan.FromMilliseconds(5)
        self._timer.Tick += self._execute
        self._timer.Start()

    def send(self, message, *args, **kwargs):
        '''
        Sends provided message to all listeners. Message is only added to
        queue and will be processed on next tick.

        :param Message message:
            Message to send.
        '''
        self._messages.put((message, args, kwargs), False)

    def subscribe(self, message, handler):
        '''
        Adds hander for specified message.

        :param str message:
            Name of message to subscribe to.

        :param callable handler:
            Handler for this message type. Handler must receive single parameter
            and that parameter will be instance of sent message.
        '''
        with self._lock:
            self._subscribers[message].append(handler)

    def unsubscribe(self, message, handler):
        '''
        Removes handler from message listeners.

        :param str message:
            Name of message to unsubscribe handler from.

        :param callable handler:
            Callable that should be removed as handler for `message`.
        '''
        with self._lock:
            self._subscribers[message].remove(handler)

    def _execute(self, sender, event_args):
        '''
        Event handler for timer that processes all queued messages.
        '''
        with self._lock:
            while not self._messages.empty():
                msg, args, kwargs = self._messages.get(False)
                for subscriber in self._subscribers[msg]:
                    subscriber(*args, **kwargs)

class Signal(object):
    '''Signal object for messaging.

    Can be used to connect directly to an object without specifying
    the message name. It works similarly to Qt Signals and Slots.
    '''
    def __init__(self, name=None):
        '''
        :param str name:
            Name of signal, for easier debuging. If not provided name of
            property to which signal is assigned to will be used if signal
            is creates in :class:`.ViewModel` class.
        '''
        self._messanger = _Messenger.instance()

    def connect(self, handler):
        '''
        Connects handler to this signal.
        '''
        self._messanger.subscribe(self, handler)

    def disconnect(self, handler):
        '''
        Disconnects handler from this singal.
        '''
        self._messanger.unsubscribe(self, handler)

    def emit(self, *args, **kwargs):
        '''
        Emits this signal. As result, all handlers will be invoked.
        '''
        self._messanger.send(self, *args, **kwargs)

    def __str__(self):
        return 'signal {name}'.format(name=self.name)

class notifiable(property):
    '''
    Decorator that replaces @property decorator by adding raising
    property changed event when setter is invoked.

    Example of usage::

        class MyViewModel(ViewModel):
            @notifiable
            def foo(self):
                return self._foo
            @foo.setter
            def foo(self, value):
                self._foo = value

    For simple properties without getter and setter function and with
    automatic event raising :class:`.Notifiable` can be used.

    Idea and initial code for this is taken from
    http://gui-at.blogspot.com/2009/11/inotifypropertychanged-in-ironpython.html
    '''
    def __init__(self, getter):
        def newgetter(slf):
            try:
                return getter(slf)
            except AttributeError:
                return None
        super(notifiable, self).__init__(newgetter)

    def setter(self, setter):
        def newsetter(slf, newvalue):
            oldvalue = self.fget(slf)
            if oldvalue != newvalue:
                setter(slf, newvalue)
                slf.RaisePropertyChanged(setter.__name__)
        return property(fget=self.fget,
            fset=newsetter,
            fdel=self.fdel,
            doc=self.__doc__)

_OCO = ObservableCollection[object]

class List(_OCO):
    '''
    ObservableCollection that can be used as ordinary python :class:`list`.
    '''
    append = _OCO.Add
    count = _OCO.Count
    index = _OCO.IndexOf
    insert = _OCO.Insert
    remove = _OCO.Remove

    def extend(self, seq):
        for item in seq:
            return self.Add(item)

    def pop(self, index=None):
        if index:
            return self.RemoveAt(index)
        else:
            return self.RemoveAt(self.Count - 1)

    def __getitem__(self, y):
        return list(self)[y]

class Notifiable(object):
    '''
    Descriptor class that raises `PropertyChanged` event when new value is
    set. For this to work, this descriptor can only be used in classes that
    implements interface `INotifyPropertyChanged`

    Class is designed to work with subclasses of :class:`ViewModel`
    because this class implements `INotofyPropertyChanged` and adds metaclass
    that discovers names of variables for raising events.

    Example of usage::

        class MyViewModel(ViewModel):
            my_property = NotifProperty()
    '''
    def __init__(self, initial=None, name=None):
        '''
        :param initial:
            Initial value of this property.

        :param name:
            Name of this property. If not provided and if this is used with
            :class:`.ViewModel`, name will be set automatically to name
            of property.
        '''
        self.name = name
        self.initial = initial

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return getattr(obj, '__notifiable_%s' % self.name, self.initial)

    def __set__(self, obj, value):
        current = getattr(obj, '__notifiable_%s' % self.name, self.initial)
        if current != value:
            setattr(obj, '__notifiable_%s' % self.name, value)
            obj.RaisePropertyChanged(self.name)

    def __delete__(self, obj):
        if hasattr(obj, '__notifiable_%s' % self.name):
            delattr(obj, '__notifiable_%s' % self.name)

class ViewModelMeta(type):
    '''
    MetaClass that examines fields of new class and populates names of
    :class:`.NotifProperty` fields to names of variables.
    '''
    def __new__(cls, name, bases, dct):
        super_new = super(ViewModelMeta, cls).__new__
        for name, val in dct.items():
            if isinstance(val, (Notifiable, Signal)):
                if not hasattr(val, 'name') or not val.name:
                    val.name = name
        return super_new(cls, name, bases, dct)

class ViewModel(object, INotifyPropertyChanged):
    '''
    Base ViewModel class that all view-model classes should inherit from.
    '''
    __metaclass__ = ViewModelMeta

    def __init__(self):
        self.property_chaged_handlers = []
        self.messenger = _Messenger.instance()

    def RaisePropertyChanged(self, property_name):
        '''
        Raises event that property value has changed for provided property name.

        :param str property_name:
            Name of property whose value has changed.
        '''
        args = PropertyChangedEventArgs(property_name)
        for handler in self.property_chaged_handlers:
            handler(self, args)

    def add_PropertyChanged(self, handler):
        self.property_chaged_handlers.append(handler)

    def remove_PropertyChanged(self, handler):
        self.property_chaged_handlers.Remove(handler)

class Command(ICommand):
    '''
    Implementation of WPF command.
    '''
    def __init__(self, execute, can_execute=None):
        self.execute = execute
        self.can_execute = can_execute
        self._can_execute_changed_handlers = []

    def Execute(self, parameter):
        '''
        Executes handler for this command.
        '''
        self.execute()

    def add_CanExecuteChanged(self, handler):
        '''
        Adds new listener to CanExecuteChanged event.
        '''
        self._can_execute_changed_handlers.append(handler)

    def remove_CanExecuteChanged(self, handler):
        '''
        Removes listener for CanExecuteChanged event.
        '''
        self._can_execute_changed_handlers.remove(handler)

    def RaiseCanExecuteChanged(self):
        '''
        Raises CanExecuteChanged event.
        '''
        for handler in self._can_execute_changed_handlers:
            handler(self, EventArgs.Empty)

    def CanExecute(self, parameter):
        '''
        Returns `True` if command can be executed, `False` otherwise.
        '''
        if self.can_execute:
            return self.can_execute()
        return True

class command(object):
    '''
    Decorator to that turns method to command handler. Example of usage::

        class MyClass(ViewModel):
            @command
            def command_handler(self):
                # do something
                pass
            @command_handler.can_execute
            def command_can_execute(self):
                # return True if command can execute, False otherwise
                return True
    '''
    def __init__(self, handler, can_execute=None):
        '''
        :param callable handler:
            Method that will be called when command executed.

        :param callable can_execute:
            Method that will be called when GUI needs information if command
            can be executed. If not provided, default implementation always
            returns `True`.
        '''
        self._handler = handler
        self._can_execute = can_execute
        self._command = None

    def __get__(self, obj, objtype):
        if not self._handler:
            raise AttributeError('Unable to get field')
        if not self._command:
            self._command = Command(partial(self._handler, obj),
                                    partial(self._can_execute, obj) if self._can_execute else None)
        return self._command

    def can_execute(self, can_execute):
        '''
        Decorator that adds function that determines if command can be executed.

        Decorated function should return `True` if command can be executed
        and false if it can not.
        '''
        self._can_execute = can_execute
<commit_msg>Begin to add weak reference support #3<commit_after>__author__    = 'Bojan Delic <bojan@delic.in.rs>'
__date__      = 'Aug 21, 2013'
__copyright__ = 'Copyright (c) 2013 Bojan Delic'

import weakref
from functools import partial
from Queue import Queue
from threading import Lock
from collections import defaultdict
import wpf

from System import TimeSpan
from System.Windows.Input import ICommand
from System import EventArgs
from System.Windows.Threading import DispatcherTimer
from System.ComponentModel import INotifyPropertyChanged, PropertyChangedEventArgs
from System.Collections.ObjectModel import ObservableCollection

class MvvmError(Exception):
    pass

class WeakCallable(object):
    '''
    Weak reference to bound methods.

    Because bound method is created at the moment when it is accessed every
    time, it will be garbage collected very soon, even if object it is bound
    to is not garbage collected. This class holds weak references to
    methods `im_self` and `im_func` that will not be garbage collected
    until `self` of the method is alive.
    '''

    def __init__(self, func, on_collect=None):
        '''
        Both `on_func_collect` and `on_self_collect` should accept single
        parameter that will be instance of :class:`.WeakCallable` that died.

        :param callable func:
            Methods to create weak reference for.
        :param callable on_collect:
            Callable that will be called when function is garbage collected.
        '''
        self._ext_on_collect = on_collect
        if hasattr(func, 'im_func'): # if this is method
            self._func = weakref.ref(func.im_func, self._on_collect)
            self._obj = weakref.ref(func.im_self)
        else:
            self._func = weakref.ref(func, self._on_collect)
            self._obj = None

    def __call__(self, *args, **kwargs):
        if self._obj is not None:
            cl = self._func()
            obj = self._obj()
            if cl is not None and obj is not None:
                return cl(obj, *args, **kwargs)
            else:
                raise weakref.ReferenceError('Method no longer available')
        else:
            cl = self._func()
            if cl is not None:
                return sl(*args, **kwargs)
            else:
                raise weakref.ReferenceError('Function no longer available')

    def _on_collect(self, ref):
        if self._ext_on_collect is not None:
            self._ext_on_collect(self)

class _Messenger(object):
    '''
    Thread-safe messenger that ensures that all message handlers are executed
    in main dispatcher thread.

    This class should be used as singleton. It is not enforced, but recomanded
    way of getting instance is by using :meth:`_Messenger.instance` class
    method.
    '''

    _instance = None

    @classmethod
    def instance(cls, interval=5):
        '''
        Returns existing instance of messenger. If one does not exist it will
        be created and returned.

        :param int interval:
            Number of miliseconds that represents interval when messages will
            be processed.
            Note that this parameter will be used only the first time when
            instance is requested, every other time it will be ignored
            because existing instance of :class:`._Messenger` is returned.
        '''
        if not cls._instance:
            cls._instance = _Messenger(interval)
        return cls._instance

    def __init__(self, interval=5):
        '''
        :param int interval:
            Number of milliseconds that represents interval when messages will
            be processed.
        '''
        self._subscribers = defaultdict(list)
        self._messages = Queue()
        self._lock = Lock()
        self._timer = DispatcherTimer()
        self._timer.Interval = TimeSpan.FromMilliseconds(5)
        self._timer.Tick += self._execute
        self._timer.Start()

    def send(self, message, *args, **kwargs):
        '''
        Sends provided message to all listeners. Message is only added to
        queue and will be processed on next tick.

        :param Message message:
            Message to send.
        '''
        self._messages.put((message, args, kwargs), False)

    def subscribe(self, message, handler):
        '''
        Adds hander for specified message.

        :param str message:
            Name of message to subscribe to.

        :param callable handler:
            Handler for this message type. Handler must receive single parameter
            and that parameter will be instance of sent message.
        '''
        with self._lock:
            ref = WeakCallable(handler, self._on_collect)
            self._subscribers[message].append(ref)

    # TODO: Unsubscribing with WeakCallable does not work
    def unsubscribe(self, message, handler):
        '''
        Removes handler from message listeners.

        :param str message:
            Name of message to unsubscribe handler from.

        :param callable handler:
            Callable that should be removed as handler for `message`.
        '''
        with self._lock:
            self._subscribers[message].remove(WeakCallable(handler))

    def _execute(self, sender, event_args):
        '''
        Event handler for timer that processes all queued messages.
        '''
        with self._lock:
            while not self._messages.empty():
                msg, args, kwargs = self._messages.get(False)
                for subscriber in self._subscribers[msg]:
                    try:
                        subscriber(*args, **kwargs)
                    except weakref.ReferenceError:
                        # Reference to handler is lost and it is OK to silence it
                        pass

    def _on_collect(self, ref):
        with self._lock:
            for msg in self._subscribers:
                if ref in self._subscribers[msg]:
                    self._subscribers[msg].remove(ref)

class Signal(object):
    '''Signal object for messaging.

    Can be used to connect directly to an object without specifying
    the message name. It works similarly to Qt Signals and Slots.
    '''
    def __init__(self, name=None):
        '''
        :param str name:
            Name of signal, for easier debuging. If not provided name of
            property to which signal is assigned to will be used if signal
            is creates in :class:`.ViewModel` class.
        '''
        self._messanger = _Messenger.instance()

    def connect(self, handler):
        '''
        Connects handler to this signal.
        '''
        self._messanger.subscribe(self, handler)

    def disconnect(self, handler):
        '''
        Disconnects handler from this singal.
        '''
        self._messanger.unsubscribe(self, handler)

    def emit(self, *args, **kwargs):
        '''
        Emits this signal. As result, all handlers will be invoked.
        '''
        self._messanger.send(self, *args, **kwargs)

    def __str__(self):
        return 'signal {name}'.format(name=self.name)

class notifiable(property):
    '''
    Decorator that replaces @property decorator by adding raising
    property changed event when setter is invoked.

    Example of usage::

        class MyViewModel(ViewModel):
            @notifiable
            def foo(self):
                return self._foo
            @foo.setter
            def foo(self, value):
                self._foo = value

    For simple properties without getter and setter function and with
    automatic event raising :class:`.Notifiable` can be used.

    Idea and initial code for this is taken from
    http://gui-at.blogspot.com/2009/11/inotifypropertychanged-in-ironpython.html
    '''
    def __init__(self, getter):
        def newgetter(slf):
            try:
                return getter(slf)
            except AttributeError:
                return None
        super(notifiable, self).__init__(newgetter)

    def setter(self, setter):
        def newsetter(slf, newvalue):
            oldvalue = self.fget(slf)
            if oldvalue != newvalue:
                setter(slf, newvalue)
                slf.RaisePropertyChanged(setter.__name__)
        return property(fget=self.fget,
            fset=newsetter,
            fdel=self.fdel,
            doc=self.__doc__)

_OCO = ObservableCollection[object]

class List(_OCO):
    '''
    ObservableCollection that can be used as ordinary python :class:`list`.
    '''
    append = _OCO.Add
    count = _OCO.Count
    index = _OCO.IndexOf
    insert = _OCO.Insert
    remove = _OCO.Remove

    def extend(self, seq):
        for item in seq:
            return self.Add(item)

    def pop(self, index=None):
        if index:
            return self.RemoveAt(index)
        else:
            return self.RemoveAt(self.Count - 1)

    def __getitem__(self, y):
        return list(self)[y]

class Notifiable(object):
    '''
    Descriptor class that raises `PropertyChanged` event when new value is
    set. For this to work, this descriptor can only be used in classes that
    implements interface `INotifyPropertyChanged`

    Class is designed to work with subclasses of :class:`ViewModel`
    because this class implements `INotofyPropertyChanged` and adds metaclass
    that discovers names of variables for raising events.

    Example of usage::

        class MyViewModel(ViewModel):
            my_property = NotifProperty()
    '''
    def __init__(self, initial=None, name=None):
        '''
        :param initial:
            Initial value of this property.

        :param name:
            Name of this property. If not provided and if this is used with
            :class:`.ViewModel`, name will be set automatically to name
            of property.
        '''
        self.name = name
        self.initial = initial

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        return getattr(obj, '__notifiable_%s' % self.name, self.initial)

    def __set__(self, obj, value):
        current = getattr(obj, '__notifiable_%s' % self.name, self.initial)
        if current != value:
            setattr(obj, '__notifiable_%s' % self.name, value)
            obj.RaisePropertyChanged(self.name)

    def __delete__(self, obj):
        if hasattr(obj, '__notifiable_%s' % self.name):
            delattr(obj, '__notifiable_%s' % self.name)

class ViewModelMeta(type):
    '''
    MetaClass that examines fields of new class and populates names of
    :class:`.NotifProperty` fields to names of variables.
    '''
    def __new__(cls, name, bases, dct):
        super_new = super(ViewModelMeta, cls).__new__
        for name, val in dct.items():
            if isinstance(val, (Notifiable, Signal)):
                if not hasattr(val, 'name') or not val.name:
                    val.name = name
        return super_new(cls, name, bases, dct)

class ViewModel(object, INotifyPropertyChanged):
    '''
    Base ViewModel class that all view-model classes should inherit from.
    '''
    __metaclass__ = ViewModelMeta

    def __init__(self):
        self.property_chaged_handlers = []
        self.messenger = _Messenger.instance()

    def RaisePropertyChanged(self, property_name):
        '''
        Raises event that property value has changed for provided property name.

        :param str property_name:
            Name of property whose value has changed.
        '''
        args = PropertyChangedEventArgs(property_name)
        for handler in self.property_chaged_handlers:
            handler(self, args)

    def add_PropertyChanged(self, handler):
        self.property_chaged_handlers.append(handler)

    def remove_PropertyChanged(self, handler):
        self.property_chaged_handlers.Remove(handler)

class Command(ICommand):
    '''
    Implementation of WPF command.
    '''
    def __init__(self, execute, can_execute=None):
        self.execute = execute
        self.can_execute = can_execute
        self._can_execute_changed_handlers = []

    def Execute(self, parameter):
        '''
        Executes handler for this command.
        '''
        self.execute()

    def add_CanExecuteChanged(self, handler):
        '''
        Adds new listener to CanExecuteChanged event.
        '''
        self._can_execute_changed_handlers.append(handler)

    def remove_CanExecuteChanged(self, handler):
        '''
        Removes listener for CanExecuteChanged event.
        '''
        self._can_execute_changed_handlers.remove(handler)

    def RaiseCanExecuteChanged(self):
        '''
        Raises CanExecuteChanged event.
        '''
        for handler in self._can_execute_changed_handlers:
            handler(self, EventArgs.Empty)

    def CanExecute(self, parameter):
        '''
        Returns `True` if command can be executed, `False` otherwise.
        '''
        if self.can_execute:
            return self.can_execute()
        return True

class command(object):
    '''
    Decorator to that turns method to command handler. Example of usage::

        class MyClass(ViewModel):
            @command
            def command_handler(self):
                # do something
                pass
            @command_handler.can_execute
            def command_can_execute(self):
                # return True if command can execute, False otherwise
                return True
    '''
    def __init__(self, handler, can_execute=None):
        '''
        :param callable handler:
            Method that will be called when command executed.

        :param callable can_execute:
            Method that will be called when GUI needs information if command
            can be executed. If not provided, default implementation always
            returns `True`.
        '''
        self._handler = handler
        self._can_execute = can_execute
        self._command = None

    def __get__(self, obj, objtype):
        if not self._handler:
            raise AttributeError('Unable to get field')
        if not self._command:
            self._command = Command(partial(self._handler, obj),
                                    partial(self._can_execute, obj) if self._can_execute else None)
        return self._command

    def can_execute(self, can_execute):
        '''
        Decorator that adds function that determines if command can be executed.

        Decorated function should return `True` if command can be executed
        and false if it can not.
        '''
        self._can_execute = can_execute
<|endoftext|>


<commit_before>#!/usr/bin/env python2

# Quick and dirty script to get the current temp from the National weather
# service.
#
# Dependencies:
#
# * requests -- for http requests
# * arrow -- date/time stuff
# * defusedxml -- for xml safety.
import requests
import arrow
import json
import sys
from defusedxml import ElementTree as ET

ENDPOINT = 'http://graphical.weather.gov/xml/sample_products/browser_interface/ndfdXMLclient.php'
ZIPCODE = '02215'

layouts = None

class LengthMismatch(Exception):

    def __init__(self, l, r):
        Exception.__init__(self, 'Length mismatch: len(%r) != len(%r)' %
                           (l, r))

def fetch_data():
    """Fetch the data from NOAA.

    Returns an element tree of the parsed xml.
    """
    resp = requests.get(ENDPOINT, params={
        'zipCodeList': ZIPCODE,
        'unit': 'e',  # Standard/English; 'm' denotes metric.
    })
    return ET.fromstring(resp.text)

def build_time_layouts(tree):
    """Extracts a dict of time layouts from the element tree.

    Keys are the values of the 'layout-key' elements, values are lists
    of (start-valid-time, end-valid-time) pairs, each time being of type
    arrow, with end times possibly None.
    """
    layouts = {}
    for tl in tree.findall('.//time-layout'):
        # Find the layout key:
        key = tl.find('layout-key')
        if key is None:
            continue
        key = key.text

        # Convert list of start/end-valid-time elements to list of arrow
        def arrow_times(elements): return [arrow.get(e.text) for e in elements]
        starts = arrow_times(tl.findall('./start-valid-time'))
        ends = arrow_times(tl.findall('./end-valid-time'))
        if ends == []:
            ends = [None for s in starts]

        layouts[key] = zip(starts, ends)
    return layouts

def time_map_for(tree, parent_query, child_query):
    """Return a list of pairs mapping times to elements.

    - `tree` should be the root of the element tree.
    - `parent_query` must be an xpath query, which when executed on `tree`
       will select an element with a 'time-layout' attribute.
    - `child_query` must be an xpath query, which when executed on the parent
      will select one element for each entry in the time layout.

    The times will be taken from the 'time-layout' parameter of the parent
    element, and the values will be the child elements.
    """
    parent = tree.find(parent_query)
    time_layout = layouts[parent.attrib['time-layout']]
    children = parent.findall(child_query)
    if len(time_layout) != len(children):
        raise LengthMismatch(time_layout, children)
    return zip(time_layout, children)

def entry_for_time(time_map, time):
    """Return the first entry in `time_map` whose time range contains `time`.

    `time_map` should be a value returend by `time_map_for`. `time` should be
    an arrow.

    Return none if no such range exists.
    """
    result = None
    for ((start, end), value) in time_map:
        if start <= time and (end is None or end > time):
            result = value
    return result

def get_hourly_temps(tree):
    """Return a mapping from an hourly time layout to temperatures."""
    time_map = time_map_for(tree, ".//temperature[@type='hourly']", './value')
    return [(k, float(v.text)) for k, v in time_map]

if __name__ == '__main__':
    root = fetch_data()
    layouts = build_time_layouts(root)

    now = arrow.now()
    temps = get_hourly_temps(root)

    current_temp = entry_for_time(temps, now)

    if current_temp is None:
        sys.exit('No reading for right now')

    print(json.dumps({"current_temp": current_temp, "units": "F"}))
<commit_msg>noaa.py: build_time_layouts: touch global<commit_after>#!/usr/bin/env python2

# Quick and dirty script to get the current temp from the National weather
# service.
#
# Dependencies:
#
# * requests -- for http requests
# * arrow -- date/time stuff
# * defusedxml -- for xml safety.
import requests
import arrow
import json
import sys
from defusedxml import ElementTree as ET

ENDPOINT = 'http://graphical.weather.gov/xml/sample_products/browser_interface/ndfdXMLclient.php'
ZIPCODE = '02215'

layouts = {}

class LengthMismatch(Exception):

    def __init__(self, l, r):
        Exception.__init__(self, 'Length mismatch: len(%r) != len(%r)' %
                           (l, r))

def fetch_data():
    """Fetch the data from NOAA.

    Returns an element tree of the parsed xml.
    """
    resp = requests.get(ENDPOINT, params={
        'zipCodeList': ZIPCODE,
        'unit': 'e',  # Standard/English; 'm' denotes metric.
    })
    return ET.fromstring(resp.text)

def build_time_layouts(tree):
    """Extracts a dict of time layouts from the element tree.

    Keys are the values of the 'layout-key' elements, values are lists
    of (start-valid-time, end-valid-time) pairs, each time being of type
    arrow, with end times possibly None.
    """
    for tl in tree.findall('.//time-layout'):
        # Find the layout key:
        key = tl.find('layout-key')
        if key is None:
            continue
        key = key.text

        # Convert list of start/end-valid-time elements to list of arrow
        def arrow_times(elements): return [arrow.get(e.text) for e in elements]
        starts = arrow_times(tl.findall('./start-valid-time'))
        ends = arrow_times(tl.findall('./end-valid-time'))
        if ends == []:
            ends = [None for s in starts]

        layouts[key] = zip(starts, ends)

def time_map_for(tree, parent_query, child_query):
    """Return a list of pairs mapping times to elements.

    - `tree` should be the root of the element tree.
    - `parent_query` must be an xpath query, which when executed on `tree`
       will select an element with a 'time-layout' attribute.
    - `child_query` must be an xpath query, which when executed on the parent
      will select one element for each entry in the time layout.

    The times will be taken from the 'time-layout' parameter of the parent
    element, and the values will be the child elements.
    """
    parent = tree.find(parent_query)
    time_layout = layouts[parent.attrib['time-layout']]
    children = parent.findall(child_query)
    if len(time_layout) != len(children):
        raise LengthMismatch(time_layout, children)
    return zip(time_layout, children)

def entry_for_time(time_map, time):
    """Return the first entry in `time_map` whose time range contains `time`.

    `time_map` should be a value returend by `time_map_for`. `time` should be
    an arrow.

    Return none if no such range exists.
    """
    result = None
    for ((start, end), value) in time_map:
        if start <= time and (end is None or end > time):
            result = value
    return result

def get_hourly_temps(tree):
    """Return a mapping from an hourly time layout to temperatures."""
    time_map = time_map_for(tree, ".//temperature[@type='hourly']", './value')
    return [(k, float(v.text)) for k, v in time_map]

if __name__ == '__main__':
    root = fetch_data()
    build_time_layouts(root)

    now = arrow.now()
    temps = get_hourly_temps(root)

    current_temp = entry_for_time(temps, now)

    if current_temp is None:
        sys.exit('No reading for right now')

    print(json.dumps({"current_temp": current_temp, "units": "F"}))
<|endoftext|>


<commit_before>import socket
from termcolor import colored
import subprocess
import json
import time
import signal
import sys
import threading
import signal
from sys import platform

def add_key(key):
    """
    Add key to node's store
    """
    keys.append(key)

def handle_message(data):
    """
    Decode message and determine task
    :param data: Message string
    """
    message = json.load(data)

    if 'action' in message:
        action = message['action']

        if action == 'add':
            add_key(message['key'])

def listening_thread(buffer_size):
    """
    Receive incoming messages
    """
    global s
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    s.bind(('', node_port))
    s.listen(1)
    while 1:
        conn, addr = s.accept()
        data = conn.recv(buffer_size)
        handle_message(data)
        conn.close()

def term_handler(signal):
    """
    Gracefully exit when encountering SIGTERM
    """
    s.close()
    exit(0)

def send_message(port, data):
    """
    Create socket and send data
    """
    s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s2.connect(('', coordinator_port))
    s2.send(data)
    s2.close()

def create_action(action):
    """
    Create an 'action' dictionary entry
    :param action: Type of action
    :return: Dictionary entry
    """
    return {'action':action}

def send_ack():
    """
    Send acknowledgement to coordinator thread
    """
    encoded_string = json.dump(create_action('ack'))
    send_message(coordinator_port, encoded_string)

def main():
    """
    Initialize node variables, start listener thread, send ack to coordinator
    """
    global coordinator_port
    global node_port
    global keys
    global pred_port
    global succ_port

    signal.signal(signal.SIGTERM, term_handler)

    coordinator_port = sys.argv[1]
    node_port = sys.argv[2]
    json_data = sys.argv[3]
    keys = []

    listener = threading.Thread(target = listening_thread, args=[1024])
    listener.daemon = True
    listener.start()

    send_ack()

if __name__ == "__main__":
    main()
<commit_msg>Fixed dumps typo<commit_after>import socket
from termcolor import colored
import subprocess
import json
import time
import signal
import sys
import threading
import signal
from sys import platform

def add_key(key):
    """
    Add key to node's store
    """
    keys.append(key)

def handle_message(data):
    """
    Decode message and determine task
    :param data: Message string
    """
    message = json.load(data)

    if 'action' in message:
        action = message['action']

        if action == 'add':
            add_key(message['key'])

def listening_thread(buffer_size):
    """
    Receive incoming messages
    """
    global s
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    s.bind(('', node_port))
    s.listen(1)
    while 1:
        conn, addr = s.accept()
        data = conn.recv(buffer_size)
        handle_message(data)
        conn.close()

def term_handler(signal):
    """
    Gracefully exit when encountering SIGTERM
    """
    s.close()
    exit(0)

def send_message(port, data):
    """
    Create socket and send data
    """
    s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s2.connect(('', coordinator_port))
    s2.send(data)
    s2.close()

def create_action(action):
    """
    Create an 'action' dictionary entry
    :param action: Type of action
    :return: Dictionary entry
    """
    return {'action':action}

def send_ack():
    """
    Send acknowledgement to coordinator thread
    """
    encoded_string = json.dumps(create_action('ack'))
    send_message(coordinator_port, encoded_string)

def main():
    """
    Initialize node variables, start listener thread, send ack to coordinator
    """
    global coordinator_port
    global node_port
    global keys
    global pred_port
    global succ_port

    signal.signal(signal.SIGTERM, term_handler)

    coordinator_port = sys.argv[1]
    node_port = sys.argv[2]
    json_data = sys.argv[3]
    keys = []

    listener = threading.Thread(target = listening_thread, args=[1024])
    listener.daemon = True
    listener.start()

    send_ack()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from peewee import *

db = SqliteDatabase(':memory:')

class BaseModel(Model):
	class Meta:
		database = db

class Node(BaseModel):

	label = CharField()
	parent = ForeignKeyField('self', default=0)
	left = IntegerField(default=0)
	right = IntegerField(default=1)
	depth = IntegerField(default=0)

	def __init__(self, **kw):
		cls = self.__class__
		super(cls, self).__init__(**kw)
		if self.is_dirty():
			left = cls.select(fn.Max(cls.right)).scalar()
			if not left:
				left = 0
			else:
				left += 1
			self.left = left
			self.right = left + 1
			self.save()

	def __repr__(self):

		#	cf. http://peewee.readthedocs.org/en/latest/peewee/api.html#SelectQuery.dicts

		nodes_list = []
		def add_node(node):
			if previous_node is not None:
				try:
					parent_id = node.parent.id
				except:
					parent_id = 0
				nodes_list.append({
					'id': node.id,
					'label': node.label,
					'parent_id': parent_id,
					'children': []
				})

		previous_node = None
		for node in self.get_with_descendants():
			add_node(previous_node)
			previous_node = node
		add_node(previous_node)

		for parent_node in nodes_list:
			for child_node in nodes_list:
				if child_node['parent_id'] == parent_node['id']:
					parent_node['children'].append(child_node)

		def show_node(node, prefix1='', prefix2=''):
			result = '%s%d %s\n' % (prefix1, node['id'], node['label'])
			if node['children']:
				last_child = node['children'][-1]
				for child in node['children']:
					(p1,p2) = ('└',' ') if (child == last_child) else ('├','│')
					result += show_node(child, prefix2 + p1 + '────', prefix2 + p2 + '    ')
			return result

		result = ''
		for node in nodes_list:
			if node['parent_id'] == 0:
				result += show_node(node)

		return result

	@classmethod
	def _shift(cls, shift, leftBound=None, rightBound=None):
		#	Update left sides
		query = cls.update(left = cls.left + shift)
		if leftBound is not None:
			query = query.where(cls.left >= leftBound)
		if rightBound is not None:
			query = query.where(cls.left <= rightBound)
		query.execute()
		#	Update right sides
		query = cls.update(right = cls.right + shift)
		if leftBound is not None:
			query = query.where(cls.right >= leftBound)
		if rightBound is not None:
			query = query.where(cls.right <= rightBound)
		query.execute()

	def _reload(self):
		row = self.select().where(self.pk_expr()).first()
		for key, value in row._data.items():
			self.__setattr__(key, value)
		self._dirty.clear()

	def _insert_at(self, position):
		cls = self.__class__

		#	Parent
		parent_left = cls.select(fn.Max(cls.left)).where(cls.left <= position, cls.right >= position).scalar()
		if parent_left is None:
			parent_depth = -1
			self.parent = 0
		else:
			parent_depth = cls.select(cls.depth).where(cls.left == parent_left).scalar()
			self.parent = cls.select(cls.id).where(cls.left == parent_left).first()
			self.save()

		#
		#	│    m   p     i       │
		#	│    └───┘     │       │
		#	└──────────────┴───────┘
		#
		i = position
		m = self.left
		p = self.right
		#	Leave a gap for the inserted node
		cls._shift(p-m+1, leftBound=i)
		#	Fill the gap with the inserted node
		if m < i:
			cls._shift(i-m, leftBound=m, rightBound=p)
		else:
			cls._shift(i-p-1, leftBound=p+1, rightBound=2*p-m+1)
		#	Left-shift the nodes after previous position of the inserted node
		cls._shift(m-p-1, leftBound=p+1)

		#	Depth
		cls.update(depth = cls.depth + parent_depth + 1 - self.depth).where(cls.left >= i, cls.right <= i+p-m).execute()

	def pop(self):
		cls = self.__class__
		self._reload()
		self_width = self.right - self.left
		#	Parent
		try:
			self.parent = None
			self.save()
		except:
			pass
		#	Put it at the end
		end = cls.select(fn.Max(cls.right)).scalar()
		self._insert_at(end + 1)
		#	Depth
		cls.update(depth = cls.depth - self.depth).where(cls.left >= end - self_width).execute()

	def prepend(self, child):
		self._reload()
		child._reload()
		child._insert_at(self.left + 1)
	def append(self, child):
		self._reload()
		child._reload()
		child._insert_at(self.right)

	def append_to(self, parent):
		parent.append(self)
	def prepend_to(self, parent):
		parent.prepend(self)

	def insert_before(self, sibling):
		self._update()
		sibling._update()
		self._insert_at(sibling.left)
	def insert_after(self, sibling):
		self._update()
		sibling._update()
		self._insert_at(sibling.right+1)

	def remove(self):
		cls = self.__class__
		self.pop()
		self._reload()
		cls.delete().where(cls.left >= self.left, cls.right <= self.right).execute()

	"""Get the node's parent to the nth level
	"""
	def get_parent(self, level=1):
		parent = self
		for i in range(level):
			parent = parent.parent
		return parent
	"""Only get direct descendants of the node
	"""
	def get_children(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left > self.left, cls.right < self.right, cls.depth == self.depth + 1).order_by(cls.left)
	"""Get all descendants from the node, whatever the level
	"""
	def get_descendants(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left > self.left, cls.right < self.right).order_by(cls.left)
	"""Get all descendants from the node, whatever the level,
	plus the containing node
	"""
	def get_with_descendants(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left >= self.left, cls.right <= self.right).order_by(cls.left)

	@classmethod
	def get_all(cls):
		return cls.select().order_by(cls.left)

if __name__ == '__main__':

	Node.create_table()

	n1 = Node(label='root')
	n2 = Node(label='a')
	n3 = Node(label='b')
	n4 = Node(label='c')

	n1.append(n2)
	n1.append(n3)
	n2.append(n4)
	# n4.append(n3)

	# n2.remove()

	# n1.pop()

	print()
	# for node in n1.get_children():
	# for node in n1.get_descendants():
	for node in Node.get_all():
		print(node.depth * '├── ' + str(node))
	print()

<commit_msg>Made the foreign key nullable<commit_after>from peewee import *

db = SqliteDatabase(':memory:')

class BaseModel(Model):
	class Meta:
		database = db

class Node(BaseModel):

	label = CharField()
	parent = ForeignKeyField('self', default=0, null=True)
	left = IntegerField(default=0)
	right = IntegerField(default=1)
	depth = IntegerField(default=0)

	def __init__(self, **kw):
		cls = self.__class__
		super(cls, self).__init__(**kw)
		if self.is_dirty():
			left = cls.select(fn.Max(cls.right)).scalar()
			if not left:
				left = 0
			else:
				left += 1
			self.left = left
			self.right = left + 1
			self.save()

	def __repr__(self):

		#	cf. http://peewee.readthedocs.org/en/latest/peewee/api.html#SelectQuery.dicts

		nodes_list = []
		def add_node(node):
			if previous_node is not None:
				try:
					parent_id = node.parent.id
				except:
					parent_id = 0
				nodes_list.append({
					'id': node.id,
					'label': node.label,
					'parent_id': parent_id,
					'children': []
				})

		previous_node = None
		for node in self.get_with_descendants():
			add_node(previous_node)
			previous_node = node
		add_node(previous_node)

		for parent_node in nodes_list:
			for child_node in nodes_list:
				if child_node['parent_id'] == parent_node['id']:
					parent_node['children'].append(child_node)

		def show_node(node, prefix1='', prefix2=''):
			result = '%s%d %s\n' % (prefix1, node['id'], node['label'])
			if node['children']:
				last_child = node['children'][-1]
				for child in node['children']:
					(p1,p2) = ('└',' ') if (child == last_child) else ('├','│')
					result += show_node(child, prefix2 + p1 + '────', prefix2 + p2 + '    ')
			return result

		result = ''
		for node in nodes_list:
			if node['parent_id'] == 0:
				result += show_node(node)

		return result

	@classmethod
	def _shift(cls, shift, leftBound=None, rightBound=None):
		#	Update left sides
		query = cls.update(left = cls.left + shift)
		if leftBound is not None:
			query = query.where(cls.left >= leftBound)
		if rightBound is not None:
			query = query.where(cls.left <= rightBound)
		query.execute()
		#	Update right sides
		query = cls.update(right = cls.right + shift)
		if leftBound is not None:
			query = query.where(cls.right >= leftBound)
		if rightBound is not None:
			query = query.where(cls.right <= rightBound)
		query.execute()

	def _reload(self):
		row = self.select().where(self.pk_expr()).first()
		for key, value in row._data.items():
			self.__setattr__(key, value)
		self._dirty.clear()

	def _insert_at(self, position):
		cls = self.__class__

		#	Parent
		parent_left = cls.select(fn.Max(cls.left)).where(cls.left <= position, cls.right >= position).scalar()
		if parent_left is None:
			parent_depth = -1
			self.parent = 0
		else:
			parent_depth = cls.select(cls.depth).where(cls.left == parent_left).scalar()
			self.parent = cls.select(cls.id).where(cls.left == parent_left).first()
			self.save()

		#
		#	│    m   p     i       │
		#	│    └───┘     │       │
		#	└──────────────┴───────┘
		#
		i = position
		m = self.left
		p = self.right
		#	Leave a gap for the inserted node
		cls._shift(p-m+1, leftBound=i)
		#	Fill the gap with the inserted node
		if m < i:
			cls._shift(i-m, leftBound=m, rightBound=p)
		else:
			cls._shift(i-p-1, leftBound=p+1, rightBound=2*p-m+1)
		#	Left-shift the nodes after previous position of the inserted node
		cls._shift(m-p-1, leftBound=p+1)

		#	Depth
		cls.update(depth = cls.depth + parent_depth + 1 - self.depth).where(cls.left >= i, cls.right <= i+p-m).execute()

	def pop(self):
		cls = self.__class__
		self._reload()
		self_width = self.right - self.left
		#	Parent
		try:
			self.parent = None
			self.save()
		except:
			pass
		#	Put it at the end
		end = cls.select(fn.Max(cls.right)).scalar()
		self._insert_at(end + 1)
		#	Depth
		cls.update(depth = cls.depth - self.depth).where(cls.left >= end - self_width).execute()

	def prepend(self, child):
		self._reload()
		child._reload()
		child._insert_at(self.left + 1)
	def append(self, child):
		self._reload()
		child._reload()
		child._insert_at(self.right)

	def append_to(self, parent):
		parent.append(self)
	def prepend_to(self, parent):
		parent.prepend(self)

	def insert_before(self, sibling):
		self._update()
		sibling._update()
		self._insert_at(sibling.left)
	def insert_after(self, sibling):
		self._update()
		sibling._update()
		self._insert_at(sibling.right+1)

	def remove(self):
		cls = self.__class__
		self.pop()
		self._reload()
		cls.delete().where(cls.left >= self.left, cls.right <= self.right).execute()

	"""Get the node's parent to the nth level
	"""
	def get_parent(self, level=1):
		parent = self
		for i in range(level):
			parent = parent.parent
		return parent
	"""Only get direct descendants of the node
	"""
	def get_children(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left > self.left, cls.right < self.right, cls.depth == self.depth + 1).order_by(cls.left)
	"""Get all descendants from the node, whatever the level
	"""
	def get_descendants(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left > self.left, cls.right < self.right).order_by(cls.left)
	"""Get all descendants from the node, whatever the level,
	plus the containing node
	"""
	def get_with_descendants(self):
		self._reload()
		cls = self.__class__
		return cls.select().where(cls.left >= self.left, cls.right <= self.right).order_by(cls.left)

	@classmethod
	def get_all(cls):
		return cls.select().order_by(cls.left)

if __name__ == '__main__':

	Node.create_table()

	n1 = Node(label='root')
	n2 = Node(label='a')
	n3 = Node(label='b')
	n4 = Node(label='c')

	n1.append(n2)
	n1.append(n3)
	n2.append(n4)
	# n4.append(n3)

	# n2.remove()

	# n1.pop()

	print()
	# for node in n1.get_children():
	# for node in n1.get_descendants():
	for node in Node.get_all():
		print(node.depth * '├── ' + str(node))
	print()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Brief Summary
Attributes:

Google Python Style Guide:
http://google-styleguide.googlecode.com/svn/trunk/pyguide.html
"""
__copyright__ = "Zhaoyu Luo"

import simpy

class Node(object):
    def __init__(self, env, cpu_cores=4, memory=8*1024*1024*1024, disk=320*1024*1024*1024, disk_speed=100*1024*1024):
        "One node is a resouce entity"
        #: need to contend for the CPU resource
        self.cpu_cores = cpu_cores
        self.memory = memory
        self.disk = disk
        self.disk_speed = disk_speed

        self.env = env
        self.action = env.process(self.run())

    def run(self):
        while True:
            yield self.env.process(self.map())
            yield self.env.process(self.reduce())

    def map(self):
        print("I am doing map")
        yield self.env.timeout(5)

    def reduce(self):
        print("I am doing reduce")
        yield self.env.timeout(2)

class Mapper(object):
    def __init__(self, env, finish_flag):
        "docstring"
        self.env = env
        self.action = env.process(self.run())
        self.finish_flag = finish_flag

    def run(self):
        while True:
            yield self.env.process(self.map())

    def map(self):
        with self.finish_flag.request() as req:
            yield req
            print("start map task")
            yield self.env.timeout(5)
            print("end map task: 5s")

class Reducer(object):
    def __init__(self, env, finish_flag):
        "docstring"
        self.env = env
        self.action = env.process(self.run())
        self.finish_flag = finish_flag

    def run(self):
        while True:
            yield self.env.process(self.reduce())

    def reduce(self):
        with self.finish_flag.request() as req:
            yield req
            print("start reduce task")
            yield self.env.timeout(2)
            print("end reduce task: 2s")

def main():
    """Main function only in command line"""
    from sys import argv
    env = simpy.Environment()
    #node = Node(env)

    finish_flag = simpy.Resource(env, capacity=1)
    mapper = Mapper(env, finish_flag)
    env.process(mapper.run())
    reducer = Reducer(env, finish_flag)
    env.process(reducer.run())

    env.run(until=15)

if __name__ == '__main__':
    main()
<commit_msg>monitor hard disk<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Brief Summary
Attributes:

Google Python Style Guide:
http://google-styleguide.googlecode.com/svn/trunk/pyguide.html
"""
__copyright__ = "Zhaoyu Luo"

import simpy
from simpy.events import AnyOf

class SimulatorException(Exception):
    pass

env = simpy.Environment()

def debugprint(msg):
    print("[%6.2f] %s" % (env.now, msg))

class Node(object):
    def __init__(self, env, cpu_cores=4, memory=8*1024*1024*1024, disk=320*1024*1024*1024, disk_speed=100*1024*1024):
        "One node is a resouce entity"
        self.env = env

        #: need to contend for the CPU resource
        self.cpu_cores = cpu_cores
        self.memory = memory
        self.disk = disk
        self.disk_speed = simpy.Container(self.env, init=disk_speed, capacity=disk_speed)
        self.disk_events = {}
        self.event_id = 0

    def random_write_tasks(self):
        current_time = 0
        task_time = [1, 1, 2, 3, 3, 3, 4, 9, 9, 9, 30]
        for i in task_time:
            if i == current_time:
                self.new_disk_write_request(1001*1024*1024)
            elif i > current_time:
                try:
                    self.env.run(until=i)
                except SimulatorException as e:
                    print(e)
                self.new_disk_write_request(1001*1024*1024)
            current_time = i

    def new_disk_write_request(self, total_bytes, start_time=0):
        """This is called by client"""
        self.event_id += 1
        event_id = self.event_id
        new_event = self.env.process(self.create_disk_write_event(total_bytes, event_id, start_time))
        self.disk_events[event_id] = new_event

    def create_disk_write_event(self, total_bytes, event_id, start_time=0):
        if start_time > 0:
            yield self.env.timeout(start_time)
        written_bytes = 0
        current_speed = 0

        while written_bytes < total_bytes:
            try:
                if current_speed > 0 and self.disk_speed.level < self.disk_speed.capacity:
                    yield self.disk_speed.put(min(current_speed, self.disk_speed.capacity - self.disk_speed.level))
                #: it should slow down, since we are adding new disk write event
                ideal_speed = int(float(self.disk_speed.capacity) / len(self.disk_events))
                debugprint("%s\tbytes_written: %s/%s\tspeed_request: %s/%s" % (event_id, written_bytes, total_bytes, ideal_speed, self.disk_speed.level))
                # Here is a compromise: if I could not obtain disk right now, I will wait for 1 second
                # then I interrupt others to gain disk
                request_ideal_disk = self.disk_speed.get(ideal_speed)
                timeout = 0.1
                request_timeout = self.env.timeout(timeout)

                try:
                    yield request_ideal_disk | request_timeout
                except simpy.Interrupt as e:
                    debugprint(e)
                    continue
                except SimulatorException as e:
                    print(e)
                    continue

                debugprint("%s\tDiskRequestSuccess?: %s\t%s" % (event_id, request_ideal_disk.processed, self.disk_speed.level))
                if request_ideal_disk.processed: # I have got disk!
                    current_speed = ideal_speed
                    estimated_finish_time = (total_bytes - written_bytes) / current_speed
                    debugprint("speed/estimated_finish_time: %6.2f/%6.2f" % (current_speed, estimated_finish_time + self.env.now))
                    try:
                        yield self.env.timeout(estimated_finish_time)
                    except simpy.Interrupt as e:
                        written_bytes += current_speed * (self.env.now - e.cause['time'])
                        debugprint("%s is interrupted %s: %s/%s" % (event_id, e, written_bytes, total_bytes))
                        continue
                    break
                else: # interrupt others!
                   # try:
                   #     request_ideal_disk.fail(SimulatorException("fail myself"))
                   # except SimulatorException as e:
                   #     print(e)
                    if self.disk_speed.capacity > self.disk_speed.level:
                        yield self.disk_speed.put(self.disk_speed.capacity - self.disk_speed.level)
                    debugprint("%s\tis interrupting others!" % event_id)
                    for k, e in self.disk_events.items():
                        if k != event_id:
                            e.interrupt({"info": "make a space for new task", "time": self.env.now - timeout})
            except simpy.Interrupt as e:
                raise e("it should not happen")

        # event finished
        self.disk_events.pop(event_id)
        debugprint("%s\tFINISHED\t%s B written" % (event_id, total_bytes))

def main():
    """Main function only in command line"""
    from sys import argv
    node = Node(env)

    node.random_write_tasks()

    env.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2015 Adiyat Mubarak

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import sys
import json
import random
import urllib2
import traceback

class ODOA(object):

    TOTAL_SURAH = 114  # Total surah within quran : https://en.wikipedia.org/wiki/List_of_surahs_in_the_Quran
    BASE_API = 'https://raw.githubusercontent.com/Keda87/quranjson/master/source'
    SUPPORTED_LANGUAGES = ['id', 'en']

    def get_random_surah(self, lang='id'):
        """
        Perform http request to get random surah.

        Parameter:
            :lang       --  String contains language code.

        Return:
            :dict       --  Paired ayat, sound, description and the translation.
        """

        # Ensure the language supported.
        if lang not in self.SUPPORTED_LANGUAGES:
            message = 'Currently your selected language not yet supported.'
            raise ValueError(message)

        # Get random surah and construct the url.
        rand_surah = random.randint(1, self.TOTAL_SURAH)
        surah_url = '{base}/surah/surah_{pages}.json'.format(base=self.BASE_API,
                                                             pages=rand_surah)
        try:
            response = urllib2.urlopen(surah_url)  # Fetch data from given url.
            data = json.loads(response.read())     # Get response and convert to dict.
        except IOError:
            traceback.print_exc(file=sys.stdout)
            return {}
        else:
            # Get random ayat.
            random_ayah = random.randint(1, int(data.get('count')))
            ayah_key = 'verse_{index}'.format(index=random_ayah)
            ayah = data['verse'][ayah_key].encode('utf-8')
            surah_index = data.get('index')
            surah_name = data.get('name')

            # Get translation and sound url.
            translation = self.__get_translation(surah=surah_index, ayah=ayah_key, lang=lang)
            sound = self.__get_sound(surah=surah_index, ayah=random_ayah)
            return {
                'ayat': ayah,
                'translate': translation,
                'sound': sound,
                'description': '{name}:{ayah}'.format(name=surah_name, ayah=random_ayah)
            }

    def __get_translation(self, surah, ayah, lang):
        """
        Perform http request to get translation from given surah, ayah and
        language.

        Parameter:
            :surah      --  Surah index from API pages.
            :ayat       --  Ayat key.
            :lang       --  Language code.

        Return:
            :string     --  Translation from given surah and ayat.
        """

        # Construct url to fetch translation data.
        translation_url = '{base}/translations/{lang}/{lang}_translation_{surah}.json' \
            .format(base=self.BASE_API, lang=lang, surah=int(surah))

        try:
            response = urllib2.urlopen(translation_url)  # Fetch data from give url.
            data = json.loads(response.read())           # Get response and convert to dict.
            translation = data['verse'][ayah]
        except Exception:
            return None
        else:
            return translation

    def __get_sound(self, surah, ayah):
        """
        Perform http request to get sound from given surah and ayah.

        Parameter:
            :surah      --  Surah index from API pages.
            :ayat       --  Ayat key.

        Return:
            :string     --  URL for mp3 sound.
        """

        # Formatting ayah with 0 leading.
        # http://stackoverflow.com/questions/17118071/python-add-leading-zeroes-using-str-format
        format_ayah = '{0:0>3}'.format(ayah)

        sound_url = '{base}/sounds/{surah}/{ayah}.mp3'.format(base=self.BASE_API,
                                                              surah=surah,
                                                              ayah=format_ayah)
        return sound_url
<commit_msg>feat: Add python 3 support<commit_after>"""
The MIT License (MIT)

Copyright (c) 2015 Adiyat Mubarak

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import sys
import json
import random
import traceback
try:
    # For Python 3.0 and later
    from urllib.request import urlopen
except ImportError:
    # Fall back to Python 2's urllib2
    from urllib2 import urlopen

class ODOA(object):

    TOTAL_SURAH = 114  # Total surah within quran : https://en.wikipedia.org/wiki/List_of_surahs_in_the_Quran
    BASE_API = 'https://raw.githubusercontent.com/Keda87/quranjson/master/source'
    SUPPORTED_LANGUAGES = ['id', 'en']

    def get_random_surah(self, lang='id'):
        """
        Perform http request to get random surah.

        Parameter:
            :lang       --  String contains language code.

        Return:
            :dict       --  Paired ayat, sound, description and the translation.
        """

        # Ensure the language supported.
        if lang not in self.SUPPORTED_LANGUAGES:
            message = 'Currently your selected language not yet supported.'
            raise ValueError(message)

        # Get random surah and construct the url.
        rand_surah = random.randint(1, self.TOTAL_SURAH)
        surah_url = '{base}/surah/surah_{pages}.json'.format(base=self.BASE_API,
                                                             pages=rand_surah)
        try:
            response = urlopen(surah_url)          # Fetch data from given url.
            data = json.loads(response.read())     # Get response and convert to dict.
        except IOError:
            traceback.print_exc(file=sys.stdout)
            return {}
        else:
            # Get random ayat.
            random_ayah = random.randint(1, int(data.get('count')))
            ayah_key = 'verse_{index}'.format(index=random_ayah)
            ayah = data['verse'][ayah_key].encode('utf-8')
            surah_index = data.get('index')
            surah_name = data.get('name')

            # Get translation and sound url.
            translation = self.__get_translation(surah=surah_index, ayah=ayah_key, lang=lang)
            sound = self.__get_sound(surah=surah_index, ayah=random_ayah)
            return {
                'ayat': ayah,
                'translate': translation,
                'sound': sound,
                'description': '{name}:{ayah}'.format(name=surah_name, ayah=random_ayah)
            }

    def __get_translation(self, surah, ayah, lang):
        """
        Perform http request to get translation from given surah, ayah and
        language.

        Parameter:
            :surah      --  Surah index from API pages.
            :ayat       --  Ayat key.
            :lang       --  Language code.

        Return:
            :string     --  Translation from given surah and ayat.
        """

        # Construct url to fetch translation data.
        translation_url = '{base}/translations/{lang}/{lang}_translation_{surah}.json' \
            .format(base=self.BASE_API, lang=lang, surah=int(surah))

        try:
            response = urlopen(translation_url)    # Fetch data from give url.
            data = json.loads(response.read())     # Get response and convert to dict.
            translation = data['verse'][ayah]
        except Exception:
            return None
        else:
            return translation

    def __get_sound(self, surah, ayah):
        """
        Perform http request to get sound from given surah and ayah.

        Parameter:
            :surah      --  Surah index from API pages.
            :ayat       --  Ayat key.

        Return:
            :string     --  URL for mp3 sound.
        """

        # Formatting ayah with 0 leading.
        # http://stackoverflow.com/questions/17118071/python-add-leading-zeroes-using-str-format
        format_ayah = '{0:0>3}'.format(ayah)

        sound_url = '{base}/sounds/{surah}/{ayah}.mp3'.format(base=self.BASE_API,
                                                              surah=surah,
                                                              ayah=format_ayah)
        return sound_url
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# vim:fileencoding=utf-8:ft=python
# file: open.py
#
# Author: R.F. Smith <rsmith@xs4all.nl>
# Created: 2014-12-26 11:45:59 +0100
# Last modified: 2017-07-15 09:29:10 +0200
#
# To the extent possible under law, R.F. Smith has waived all copyright and
# related or neighboring rights to open.py. This work is published from the
# Netherlands. See http://creativecommons.org/publicdomain/zero/1.0/
"""
Open file(s) given on the command line in the appropriate program.

Some of the programs are X11 programs.
"""

from os.path import isdir, isfile, exists
from re import search, IGNORECASE
from subprocess import Popen, run, PIPE
from sys import argv
import argparse
import logging
from magic import from_file

__version__ = '1.5.0'

# You should adjust the programs called to suit your preferences.
filetypes = {
    '\.(pdf|epub)$': ['mupdf'],
    '\.html$': ['firefox'],
    '\.xcf$': ['gimp'],
    '\.e?ps$': ['gv'],
    '\.(jpe?g|png|gif|tiff?|p[abgp]m|svg)$': ['gpicview'],
    '\.(pax|cpio|zip|jar|ar|xar|rpm|7z)$': ['tar', 'tf'],
    '\.(tar\.|t)(z|gz|bz2?|xz)$': ['tar', 'tf'],
    '\.(mp4|mkv|avi|flv|mpg|movi?|m4v|webm)$': ['mpv']
}
othertypes = {'dir': ['rox'], 'txt': ['gvim', '--nofork']}

def main(argv):  # noqa
    """Entry point for open.

    Arguments:
        argv: command line arguments; list of strings.
    """
    if argv[0].endswith(('open', 'open.py')):
        del argv[0]
    opts = argparse.ArgumentParser(prog='open', description=__doc__)
    opts.add_argument('-v', '--version', action='version', version=__version__)
    opts.add_argument('-a', '--application', help='application to use')
    opts.add_argument(
        '--log',
        default='warning',
        choices=['debug', 'info', 'warning', 'error'],
        help="logging level (defaults to 'warning')")
    opts.add_argument(
        "files",
        metavar='file',
        nargs='*',
        help="one or more files to process")
    args = opts.parse_args(argv)
    logging.basicConfig(
        level=getattr(logging, args.log.upper(), None),
        format='%(levelname)s: %(message)s')
    logging.info('command line arguments = {}'.format(argv))
    logging.info('parsed arguments = {}'.format(args))
    fail = "opening '{}' failed: {}"
    # Check for non-local files with `locate`.
    try:
        files = []
        for nm in args.files:
            if exists(nm):
                files.append(nm)
            else:
                cp = run(['locate', nm], stdout=PIPE)
                paths = cp.stdout.decode('utf-8').splitlines()
                if len(paths) == 1:
                    files.append(paths[0])
                elif len(paths) == 0:
                    logging.warning("path '{}' not found".format(nm))
                else:
                    logging.warning("ambiguous path '{}' skipped".format(nm))
                    for p in paths:
                        logging.warning("found '{}'".format(p))
    except FileNotFoundError:  # `locate` not available.
        files = args.files
    # Open the file(s).
    for nm in files:
        logging.info("Trying '{}'".format(nm))
        if not args.application:
            if isdir(nm):
                cmds = othertypes['dir'] + [nm]
            elif isfile(nm):
                cmds = matchfile(filetypes, othertypes, nm)
            else:
                cmds = None
        else:
            cmds = [args.application, nm]
        if not cmds:
            logging.warning("do not know how to open '{}'".format(nm))
            continue
        try:
            Popen(cmds)
        except OSError as e:
            logging.error(fail.format(nm, e))
    else:  # No files named
        if args.application:
            try:
                Popen([args.application])
            except OSError as e:
                logging.error(fail.format(args.application, e))

def matchfile(fdict, odict, fname):
    """
    Return the matching program for a given filename.

    Arguments:
        fdict: Handlers for files. A dictionary of regex:(commands)
            representing the file type and the action that is to be taken for
            opening one.
        odict: Handlers for other types. A dictionary of str:(arguments).
        fname: A string containing the name of the file to be opened.

    Returns: A list of commands for subprocess.Popen.
    """
    for k, v in fdict.items():
        if search(k, fname, IGNORECASE) is not None:
            return v + [fname]
    if b'text' in from_file(fname):
        return odict['txt'] + [fname]
    return None

if __name__ == '__main__':
    main(argv)
<commit_msg>Add BMP type to ‘open’.<commit_after>#!/usr/bin/env python3
# vim:fileencoding=utf-8:ft=python
# file: open.py
#
# Author: R.F. Smith <rsmith@xs4all.nl>
# Created: 2014-12-26 11:45:59 +0100
# Last modified: 2017-10-07 22:39:54 +0200
#
# To the extent possible under law, R.F. Smith has waived all copyright and
# related or neighboring rights to open.py. This work is published from the
# Netherlands. See http://creativecommons.org/publicdomain/zero/1.0/
"""
Open file(s) given on the command line in the appropriate program.

Some of the programs are X11 programs.
"""

from os.path import isdir, isfile, exists
from re import search, IGNORECASE
from subprocess import Popen, run, PIPE
from sys import argv
import argparse
import logging
from magic import from_file

__version__ = '1.5.1'

# You should adjust the programs called to suit your preferences.
filetypes = {
    '\.(pdf|epub)$': ['mupdf'],
    '\.html$': ['firefox'],
    '\.xcf$': ['gimp'],
    '\.e?ps$': ['gv'],
    '\.(jpe?g|png|gif|tiff?|p[abgp]m|bmp|svg)$': ['gpicview'],
    '\.(pax|cpio|zip|jar|ar|xar|rpm|7z)$': ['tar', 'tf'],
    '\.(tar\.|t)(z|gz|bz2?|xz)$': ['tar', 'tf'],
    '\.(mp4|mkv|avi|flv|mpg|movi?|m4v|webm)$': ['mpv']
}
othertypes = {'dir': ['rox'], 'txt': ['gvim', '--nofork']}

def main(argv):  # noqa
    """Entry point for open.

    Arguments:
        argv: command line arguments; list of strings.
    """
    if argv[0].endswith(('open', 'open.py')):
        del argv[0]
    opts = argparse.ArgumentParser(prog='open', description=__doc__)
    opts.add_argument('-v', '--version', action='version', version=__version__)
    opts.add_argument('-a', '--application', help='application to use')
    opts.add_argument(
        '--log',
        default='warning',
        choices=['debug', 'info', 'warning', 'error'],
        help="logging level (defaults to 'warning')")
    opts.add_argument(
        "files",
        metavar='file',
        nargs='*',
        help="one or more files to process")
    args = opts.parse_args(argv)
    logging.basicConfig(
        level=getattr(logging, args.log.upper(), None),
        format='%(levelname)s: %(message)s')
    logging.info('command line arguments = {}'.format(argv))
    logging.info('parsed arguments = {}'.format(args))
    fail = "opening '{}' failed: {}"
    # Check for non-local files with `locate`.
    try:
        files = []
        for nm in args.files:
            if exists(nm):
                files.append(nm)
            else:
                cp = run(['locate', nm], stdout=PIPE)
                paths = cp.stdout.decode('utf-8').splitlines()
                if len(paths) == 1:
                    files.append(paths[0])
                elif len(paths) == 0:
                    logging.warning("path '{}' not found".format(nm))
                else:
                    logging.warning("ambiguous path '{}' skipped".format(nm))
                    for p in paths:
                        logging.warning("found '{}'".format(p))
    except FileNotFoundError:  # `locate` not available.
        files = args.files
    # Open the file(s).
    for nm in files:
        logging.info("Trying '{}'".format(nm))
        if not args.application:
            if isdir(nm):
                cmds = othertypes['dir'] + [nm]
            elif isfile(nm):
                cmds = matchfile(filetypes, othertypes, nm)
            else:
                cmds = None
        else:
            cmds = [args.application, nm]
        if not cmds:
            logging.warning("do not know how to open '{}'".format(nm))
            continue
        try:
            Popen(cmds)
        except OSError as e:
            logging.error(fail.format(nm, e))
    else:  # No files named
        if args.application:
            try:
                Popen([args.application])
            except OSError as e:
                logging.error(fail.format(args.application, e))

def matchfile(fdict, odict, fname):
    """
    Return the matching program for a given filename.

    Arguments:
        fdict: Handlers for files. A dictionary of regex:(commands)
            representing the file type and the action that is to be taken for
            opening one.
        odict: Handlers for other types. A dictionary of str:(arguments).
        fname: A string containing the name of the file to be opened.

    Returns: A list of commands for subprocess.Popen.
    """
    for k, v in fdict.items():
        if search(k, fname, IGNORECASE) is not None:
            return v + [fname]
    if b'text' in from_file(fname):
        return odict['txt'] + [fname]
    return None

if __name__ == '__main__':
    main(argv)
<|endoftext|>


<commit_before># opts.py - module to parse command line options and output what parameters will be used for this run

import os
import os.path
import json
import sys
import common
from common import OK, NOTOK, FsDriftException
import argparse
import parser_data_types
from parser_data_types import boolean, positive_integer, non_negative_integer
from parser_data_types import positive_float, non_negative_float, positive_percentage
from parser_data_types import host_set, file_access_distrib, directory_list

# command line parameter variables here

class FsDriftOpts:
    def __init__(self):
        self.top_directory = '/tmp/foo'
        self.output_json_path = None  # filled in later
        self.host_set = [] # default is local test
        self.threads = 2 #  number of subprocesses per host
        self.is_slave = False
        self.duration = 1
        self.max_files = 200
        self.max_file_size_kb = 10
        self.max_record_size_kb = 1
        self.max_random_reads = 2
        self.max_random_writes = 2
        self.fdatasync_probability_pct = 10
        self.fsync_probability_pct = 20
        self.levels = 2
        self.subdirs_per_dir = 3
        self.rsptimes = False
        self.workload_table_csv_path = None
        self.stats_report_interval = 0
        self.pause_between_ops = 100
        self.pause_secs = self.pause_between_ops / float(common.USEC_PER_SEC)
        self.incompressible = False
        # new parameters related to gaussian filename distribution
        self.rand_distr_type = common.FileAccessDistr.uniform
        self.mean_index_velocity = 0.0  # default is a fixed mean for the distribution
        self.gaussian_stddev = 1000.0  # just a guess, means most of accesses within 1000 files?
        # just a guess, most files will be created before they are read
        self.create_stddevs_ahead = 3.0
        self.drift_time = -1
        self.mount_command = None
        # not settable
        self.is_slave = False
        self.as_host = None  # filled in by worker host

    def kvtuplelist(self):
        return [
            ('top directory', self.top_directory),
            ('JSON output file', self.output_json_path),
            ('save response times?', self.rsptimes),
            ('stats report interval', self.stats_report_interval),
            ('workload table csv path', self.workload_table_csv_path),
            ('host set', ','.join(self.host_set)),
            ('threads', self.threads),
            ('test duration', self.duration),
            ('maximum file count', self.max_files),
            ('maximum file size (KB)', self.max_file_size_kb),
            ('maximum record size (KB)', self.max_record_size_kb),
            ('maximum random reads per op', self.max_random_reads),
            ('maximum random writes per op', self.max_random_writes),
            ('fsync probability pct', self.fsync_probability_pct),
            ('fdatasync probability pct', self.fdatasync_probability_pct),
            ('directory levels', self.levels),
            ('subdirectories per directory', self.subdirs_per_dir),
            ('incompressible data', self.incompressible),
            ('pause between opts (usec)', self.pause_between_ops),
            ('distribution', common.FileAccessDistr2str(self.random_distribution)),
            ('mean index velocity', self.mean_index_velocity),
            ('gaussian std. dev.', self.gaussian_stddev),
            ('create stddevs ahead', self.create_stddevs_ahead),
            ('mount command', self.mount_command),
            ('pause path', self.pause_path),
            ]

    def __str__(self, use_newline=True, indentation='  '):
        kvlist = [ '%s = %s' % (k, str(v)) for (k, v) in self.kvtuplelist() ]
        if use_newline:
            return ('\n%s' % indentation).join(kvlist)
        else:
            return ' , '.join(kvlist)

    def to_json_obj(self):
        d = {}
        for (k, v) in self.kvtuplelist():
            d[k] = v
        return d

    def validate(self):

        if len(self.top_directory) < 6:
            raise FsDriftException(
                'top directory %s too short, may be system directory' %
                self.top_directory)

        if not os.path.isdir(self.top_directory):
            raise FsDriftException(
                'top directory %s does not exist, so please create it' %
                self.top_directory)

        if self.workload_table_csv_path == None:
            self.workload_table_csv_path = os.path.join(self.top_directory,
                                                        'example_workload_table.csv')
            workload_table = [
                    'read, 2',
                    'random_read, 1',
                    'random_write, 1',
                    'append, 4',
                    'delete, 0.1',
                    'hardlink, 0.01',
                    'softlink, 0.02',
                    'truncate, 0.05',
                    'rename, 1',
                    'create, 4']
            with open(self.workload_table_csv_path, 'w') as w_f:
                w_f.write( '\n'.join(workload_table))

def parseopts():
    o = FsDriftOpts()

    parser = argparse.ArgumentParser(description='parse fs-drift parameters')
    add = parser.add_argument
    add('--top', help='directory containing all file accesses',
            default=o.top_directory)
    add('--output-json', help='output file containing results in JSON format',
            default=None)
    add('--workload-table', help='.csv file containing workload mix',
            default=None)
    add('--duration', help='seconds to run test',
            type=positive_integer,
            default=o.duration)
    add('--host-set', help='comma-delimited list of host names/ips',
            type=host_set,
            default=o.host_set)
    add('--threads', help='number of subprocesses per host',
            type=positive_integer,
            default=o.threads)
    add('--max-files', help='maximum number of files to access',
            type=positive_integer,
            default=o.max_files)
    add('--max-file-size-kb', help='maximum file size in KB',
            type=positive_integer,
            default=o.max_file_size_kb)
    add('--pause-between-ops', help='delay between ops in microsec',
            type=non_negative_integer,
            default=o.pause_between_ops)
    add('--max-record-size-kb', help='maximum read/write size in KB',
            type=positive_integer,
            default=o.max_record_size_kb)
    add('--max-random-reads', help='maximum consecutive random reads',
            type=positive_integer,
            default=o.max_random_reads)
    add('--max-random-writes', help='maximum consecutive random writes',
            type=positive_integer,
            default=o.max_random_writes)
    add('--fdatasync-pct', help='probability of fdatasync after write',
            type=positive_percentage,
            default=o.fdatasync_probability_pct)
    add('--fsync-pct', help='probability of fsync after write',
            type=positive_percentage,
            default=o.fsync_probability_pct)
    add('--levels', help='number of directory levels in tree',
            type=non_negative_integer,
            default=o.levels)
    add('--dirs-per-level', help='number of subdirectories per directory',
            type=non_negative_integer,
            default=o.subdirs_per_dir)
    add('--report-interval', help='seconds between counter output',
            type=positive_integer,
            default=o.stats_report_interval)
    add('--response-times', help='if True then save response times to CSV file',
            type=boolean,
            default=o.rsptimes)
    add('--incompressible', help='if True then write incompressible data',
            type=boolean,
            default=o.incompressible)
    add('--random-distribution', help='either "uniform" or "gaussian"',
            type=file_access_distrib,
            default=common.FileAccessDistr.uniform)
    add('--mean-velocity', help='rate at which mean advances through files',
            type=float,
            default=o.mean_index_velocity)
    add('--gaussian-stddev', help='std. dev. of file number',
            type=float,
            default=o.gaussian_stddev)
    add('--create-stddevs-ahead', help='file creation ahead of other opts by this many stddevs',
            type=float,
            default=o.create_stddevs_ahead)
    add('--mount-command', help='command to mount the filesystem containing top directory',
            default=o.mount_command)
    # parse the command line and update opts
    args = parser.parse_args()
    o.top_directory = args.top
    o.output_json_path = args.output_json
    o.rsptimes = args.response_times
    o.host_set = args.host_set
    o.threads = args.threads
    o.report_interval = args.report_interval
    o.workload_table_csv_path = args.workload_table
    o.duration = args.duration
    o.max_files = args.max_files
    o.max_file_size_kb = args.max_file_size_kb
    o.max_record_size_kb = args.max_record_size_kb
    o.max_random_reads = args.max_random_reads
    o.max_random_writes = args.max_random_writes
    o.fdatasync_probability_pct = args.fdatasync_pct
    o.fsync_probability_pct = args.fsync_pct
    o.levels = args.levels
    o.subdirs_per_dir = args.dirs_per_level
    o.incompressible = args.incompressible
    o.pause_between_ops = args.pause_between_ops
    o.pause_secs = o.pause_between_ops / float(common.USEC_PER_SEC)
    o.response_times = args.response_times
    o.random_distribution = args.random_distribution
    o.mean_index_velocity = args.mean_velocity
    o.gaussian_stddev = args.gaussian_stddev
    o.create_stddevs_ahead = args.create_stddevs_ahead
    o.mount_command = args.mount_command

    # some fields derived from user inputs

    o.network_shared_path = os.path.join(o.top_directory, 'network-shared')

    nsjoin = lambda fn : os.path.join(o.network_shared_path, fn)
    o.starting_gun_path     = nsjoin('starting-gun.tmp')
    o.stop_file_path        = nsjoin('stop-file.tmp')
    o.param_pickle_path     = nsjoin('params.pickle')
    o.rsptime_path          = nsjoin('host-%s_thrd-%s_rsptimes.csv')
    o.abort_path            = nsjoin('abort.tmp')
    o.pause_path            = nsjoin('pause.tmp')
    o.checkerflag_path      = nsjoin('checkered_flag.tmp')

    o.remote_pgm_dir = os.path.dirname(sys.argv[0])
    if o.remote_pgm_dir == '.':
        o.remote_pgm_dir = os.getcwd()

    o.is_slave = sys.argv[0].endswith('fs-drift-remote.py')

    return o

# unit test

if __name__ == "__main__":
    options = parseopts()
    options.validate()
    print(options)
    print(json.dumps(options.to_json_obj(), indent=2, sort_keys=True))
<commit_msg>add verbosity parameter<commit_after># opts.py - module to parse command line options and output what parameters will be used for this run

import os
import os.path
import json
import sys
import common
from common import OK, NOTOK, FsDriftException
import argparse
import parser_data_types
from parser_data_types import boolean, positive_integer, non_negative_integer, bitmask
from parser_data_types import positive_float, non_negative_float, positive_percentage
from parser_data_types import host_set, file_access_distrib, directory_list

# command line parameter variables here

class FsDriftOpts:
    def __init__(self):
        self.top_directory = '/tmp/foo'
        self.output_json_path = None  # filled in later
        self.host_set = [] # default is local test
        self.threads = 2 #  number of subprocesses per host
        self.is_slave = False
        self.duration = 1
        self.max_files = 200
        self.max_file_size_kb = 10
        self.max_record_size_kb = 1
        self.max_random_reads = 2
        self.max_random_writes = 2
        self.fdatasync_probability_pct = 10
        self.fsync_probability_pct = 20
        self.levels = 2
        self.subdirs_per_dir = 3
        self.rsptimes = False
        self.workload_table_csv_path = None
        self.stats_report_interval = 0
        self.pause_between_ops = 100
        self.pause_secs = self.pause_between_ops / float(common.USEC_PER_SEC)
        self.incompressible = False
        # new parameters related to gaussian filename distribution
        self.rand_distr_type = common.FileAccessDistr.uniform
        self.mean_index_velocity = 0.0  # default is a fixed mean for the distribution
        self.gaussian_stddev = 1000.0  # just a guess, means most of accesses within 1000 files?
        # just a guess, most files will be created before they are read
        self.create_stddevs_ahead = 3.0
        self.drift_time = -1
        self.mount_command = None
        # not settable
        self.is_slave = False
        self.as_host = None  # filled in by worker host
        self.verbosity = 0

    def kvtuplelist(self):
        return [
            ('top directory', self.top_directory),
            ('JSON output file', self.output_json_path),
            ('save response times?', self.rsptimes),
            ('stats report interval', self.stats_report_interval),
            ('workload table csv path', self.workload_table_csv_path),
            ('host set', ','.join(self.host_set)),
            ('threads', self.threads),
            ('test duration', self.duration),
            ('maximum file count', self.max_files),
            ('maximum file size (KB)', self.max_file_size_kb),
            ('maximum record size (KB)', self.max_record_size_kb),
            ('maximum random reads per op', self.max_random_reads),
            ('maximum random writes per op', self.max_random_writes),
            ('fsync probability pct', self.fsync_probability_pct),
            ('fdatasync probability pct', self.fdatasync_probability_pct),
            ('directory levels', self.levels),
            ('subdirectories per directory', self.subdirs_per_dir),
            ('incompressible data', self.incompressible),
            ('pause between opts (usec)', self.pause_between_ops),
            ('distribution', common.FileAccessDistr2str(self.random_distribution)),
            ('mean index velocity', self.mean_index_velocity),
            ('gaussian std. dev.', self.gaussian_stddev),
            ('create stddevs ahead', self.create_stddevs_ahead),
            ('mount command', self.mount_command),
            ('verbosity', self.verbosity),
            ('pause path', self.pause_path),
            ]

    def __str__(self, use_newline=True, indentation='  '):
        kvlist = [ '%s = %s' % (k, str(v)) for (k, v) in self.kvtuplelist() ]
        if use_newline:
            return ('\n%s' % indentation).join(kvlist)
        else:
            return ' , '.join(kvlist)

    def to_json_obj(self):
        d = {}
        for (k, v) in self.kvtuplelist():
            d[k] = v
        return d

    def validate(self):

        if len(self.top_directory) < 6:
            raise FsDriftException(
                'top directory %s too short, may be system directory' %
                self.top_directory)

        if not os.path.isdir(self.top_directory):
            raise FsDriftException(
                'top directory %s does not exist, so please create it' %
                self.top_directory)

        if self.workload_table_csv_path == None:
            self.workload_table_csv_path = os.path.join(self.top_directory,
                                                        'example_workload_table.csv')
            workload_table = [
                    'read, 2',
                    'random_read, 1',
                    'random_write, 1',
                    'append, 4',
                    'delete, 0.1',
                    'hardlink, 0.01',
                    'softlink, 0.02',
                    'truncate, 0.05',
                    'rename, 1',
                    'create, 4']
            with open(self.workload_table_csv_path, 'w') as w_f:
                w_f.write( '\n'.join(workload_table))

def parseopts():
    o = FsDriftOpts()

    parser = argparse.ArgumentParser(description='parse fs-drift parameters')
    add = parser.add_argument
    add('--top', help='directory containing all file accesses',
            default=o.top_directory)
    add('--output-json', help='output file containing results in JSON format',
            default=None)
    add('--workload-table', help='.csv file containing workload mix',
            default=None)
    add('--duration', help='seconds to run test',
            type=positive_integer,
            default=o.duration)
    add('--host-set', help='comma-delimited list of host names/ips',
            type=host_set,
            default=o.host_set)
    add('--threads', help='number of subprocesses per host',
            type=positive_integer,
            default=o.threads)
    add('--max-files', help='maximum number of files to access',
            type=positive_integer,
            default=o.max_files)
    add('--max-file-size-kb', help='maximum file size in KB',
            type=positive_integer,
            default=o.max_file_size_kb)
    add('--pause-between-ops', help='delay between ops in microsec',
            type=non_negative_integer,
            default=o.pause_between_ops)
    add('--max-record-size-kb', help='maximum read/write size in KB',
            type=positive_integer,
            default=o.max_record_size_kb)
    add('--max-random-reads', help='maximum consecutive random reads',
            type=positive_integer,
            default=o.max_random_reads)
    add('--max-random-writes', help='maximum consecutive random writes',
            type=positive_integer,
            default=o.max_random_writes)
    add('--fdatasync-pct', help='probability of fdatasync after write',
            type=positive_percentage,
            default=o.fdatasync_probability_pct)
    add('--fsync-pct', help='probability of fsync after write',
            type=positive_percentage,
            default=o.fsync_probability_pct)
    add('--levels', help='number of directory levels in tree',
            type=non_negative_integer,
            default=o.levels)
    add('--dirs-per-level', help='number of subdirectories per directory',
            type=non_negative_integer,
            default=o.subdirs_per_dir)
    add('--report-interval', help='seconds between counter output',
            type=positive_integer,
            default=o.stats_report_interval)
    add('--response-times', help='if True then save response times to CSV file',
            type=boolean,
            default=o.rsptimes)
    add('--incompressible', help='if True then write incompressible data',
            type=boolean,
            default=o.incompressible)
    add('--random-distribution', help='either "uniform" or "gaussian"',
            type=file_access_distrib,
            default=common.FileAccessDistr.uniform)
    add('--mean-velocity', help='rate at which mean advances through files',
            type=float,
            default=o.mean_index_velocity)
    add('--gaussian-stddev', help='std. dev. of file number',
            type=float,
            default=o.gaussian_stddev)
    add('--create-stddevs-ahead', help='file creation ahead of other opts by this many stddevs',
            type=float,
            default=o.create_stddevs_ahead)
    add('--mount-command', help='command to mount the filesystem containing top directory',
            default=o.mount_command)
    add('--verbosity', help='decimal or hexadecimal integer bitmask controlling debug logging',
            type=bitmask,
            default=o.verbosity)

    # parse the command line and update opts
    args = parser.parse_args()
    o.top_directory = args.top
    o.output_json_path = args.output_json
    o.rsptimes = args.response_times
    o.host_set = args.host_set
    o.threads = args.threads
    o.report_interval = args.report_interval
    o.workload_table_csv_path = args.workload_table
    o.duration = args.duration
    o.max_files = args.max_files
    o.max_file_size_kb = args.max_file_size_kb
    o.max_record_size_kb = args.max_record_size_kb
    o.max_random_reads = args.max_random_reads
    o.max_random_writes = args.max_random_writes
    o.fdatasync_probability_pct = args.fdatasync_pct
    o.fsync_probability_pct = args.fsync_pct
    o.levels = args.levels
    o.subdirs_per_dir = args.dirs_per_level
    o.incompressible = args.incompressible
    o.pause_between_ops = args.pause_between_ops
    o.pause_secs = o.pause_between_ops / float(common.USEC_PER_SEC)
    o.response_times = args.response_times
    o.random_distribution = args.random_distribution
    o.mean_index_velocity = args.mean_velocity
    o.gaussian_stddev = args.gaussian_stddev
    o.create_stddevs_ahead = args.create_stddevs_ahead
    o.mount_command = args.mount_command
    o.verbosity = args.verbosity

    # some fields derived from user inputs

    o.network_shared_path = os.path.join(o.top_directory, 'network-shared')

    nsjoin = lambda fn : os.path.join(o.network_shared_path, fn)
    o.starting_gun_path     = nsjoin('starting-gun.tmp')
    o.stop_file_path        = nsjoin('stop-file.tmp')
    o.param_pickle_path     = nsjoin('params.pickle')
    o.rsptime_path          = nsjoin('host-%s_thrd-%s_rsptimes.csv')
    o.abort_path            = nsjoin('abort.tmp')
    o.pause_path            = nsjoin('pause.tmp')
    o.checkerflag_path      = nsjoin('checkered_flag.tmp')

    o.remote_pgm_dir = os.path.dirname(sys.argv[0])
    if o.remote_pgm_dir == '.':
        o.remote_pgm_dir = os.getcwd()

    o.is_slave = sys.argv[0].endswith('fs-drift-remote.py')

    return o

# unit test

if __name__ == "__main__":
    options = parseopts()
    options.validate()
    print(options)
    print(json.dumps(options.to_json_obj(), indent=2, sort_keys=True))
<|endoftext|>