<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import json
try:
    # 3.8 and up
    from collections.abc import Iterable
except ImportError:
    from collections import Iterable

from util import check_if_numbers_are_consecutive
import six

def _construct_key(previous_key, separator, new_key):
    """
    Returns the new_key if no previous key exists, otherwise concatenates
    previous key, separator, and new_key
    :param previous_key:
    :param separator:
    :param new_key:
    :return: a string if previous_key exists and simply passes through the
    new_key otherwise
    """
    if previous_key:
        return u"{}{}{}".format(previous_key, separator, new_key)
    else:
        return new_key

def flatten(nested_dict, separator="_", root_keys_to_ignore=set()):
    """
    Flattens a dictionary with nested structure to a dictionary with no
    hierarchy
    Consider ignoring keys that you are not interested in to prevent
    unnecessary processing
    This is specially true for very deep objects

    :param nested_dict: dictionary we want to flatten
    :param separator: string to separate dictionary keys by
    :param root_keys_to_ignore: set of root keys to ignore from flattening
    :return: flattened dictionary
    """
    assert isinstance(nested_dict, dict), "flatten requires a dictionary input"
    assert isinstance(separator, six.string_types), "separator must be string"

    # This global dictionary stores the flattened keys and values and is
    # ultimately returned
    flattened_dict = dict()

    def _flatten(object_, key):
        """
        For dict, list and set objects_ calls itself on the elements and for
        other types assigns the object_ to
        the corresponding key in the global flattened_dict
        :param object_: object to flatten
        :param key: carries the concatenated key for the object_
        :return: None
        """
        # Empty object can't be iterated, take as is
        if not object_:
            flattened_dict[key] = object_
        # These object types support iteration
        elif isinstance(object_, dict):
            for object_key in object_:
                if not (not key and object_key in root_keys_to_ignore):
                    _flatten(object_[object_key], _construct_key(key,
                                                                 separator,
                                                                 object_key))
        elif isinstance(object_, list) or isinstance(object_, set):
            for index, item in enumerate(object_):
                _flatten(item, _construct_key(key, separator, index))
        # Anything left take as is
        else:
            flattened_dict[key] = object_

    _flatten(nested_dict, None)
    return flattened_dict

flatten_json = flatten

def _unflatten_asserts(flat_dict, separator):
    assert isinstance(flat_dict, dict), "un_flatten requires dictionary input"
    assert isinstance(separator, six.string_types), "separator must be string"
    assert all((not value or not isinstance(value, Iterable) or
                isinstance(value, six.string_types)
                for value in flat_dict.values())), "provided dict is not flat"

def unflatten(flat_dict, separator='_'):
    """
    Creates a hierarchical dictionary from a flattened dictionary
    Assumes no lists are present
    :param flat_dict: a dictionary with no hierarchy
    :param separator: a string that separates keys
    :return: a dictionary with hierarchy
    """
    _unflatten_asserts(flat_dict, separator)

    # This global dictionary is mutated and returned
    unflattened_dict = dict()

    def _unflatten(dic, keys, value):
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})

        dic[keys[-1]] = value

    list_keys = sorted(flat_dict.keys())
    for i, item in enumerate(list_keys):
        if i != len(list_keys)-1:
            if not list_keys[i+1].startswith(list_keys[i]):
                _unflatten(unflattened_dict, item.split(separator),
                           flat_dict[item])
            else:
                pass  # if key contained in next key, json will be invalid.
        else:
            #  last element
            _unflatten(unflattened_dict, item.split(separator),
                       flat_dict[item])
    return unflattened_dict

def unflatten_list(flat_dict, separator='_'):
    """
    Unflattens a dictionary, first assuming no lists exist and then tries to
    identify lists and replaces them
    This is probably not very efficient and has not been tested extensively
    Feel free to add test cases or rewrite the logic
    Issues that stand out to me:
    - Sorting all the keys in the dictionary, which specially for the root
    dictionary can be a lot of keys
    - Checking that numbers are consecutive is O(N) in number of keys

    :param flat_dict: dictionary with no hierarchy
    :param separator: a string that separates keys
    :return: a dictionary with hierarchy
    """
    _unflatten_asserts(flat_dict, separator)

    # First unflatten the dictionary assuming no lists exist
    unflattened_dict = unflatten(flat_dict, separator)

    def _convert_dict_to_list(object_, parent_object, parent_object_key):
        if isinstance(object_, dict):
            for key in object_:
                if isinstance(object_[key], dict):
                    _convert_dict_to_list(object_[key], object_, key)
            try:
                keys = [int(key) for key in object_]
                keys.sort()
            except (ValueError, TypeError):
                keys = []
            keys_len = len(keys)

            if (keys_len > 0 and sum(keys) ==
                int(((keys_len - 1) * keys_len) / 2) and keys[0] == 0 and
                    keys[-1] == keys_len - 1 and
                    check_if_numbers_are_consecutive(keys)):

                # The dictionary looks like a list so we're going to replace it
                parent_object[parent_object_key] = []
                for key_index, key in enumerate(keys):
                    parent_object[parent_object_key].append(object_[str(key)])
                    # The list item we just added might be a list itself
                    # https://github.com/amirziai/flatten/issues/15
                    _convert_dict_to_list(parent_object[parent_object_key][-1],
                                          parent_object[parent_object_key],
                                          key_index)

    _convert_dict_to_list(unflattened_dict, None, None)
    return unflattened_dict

def cli(input_stream=sys.stdin, output_stream=sys.stdout):
    raw = input_stream.read()
    input_json = json.loads(raw)
    output = json.dumps(flatten(input_json))
    output_stream.write('{}\n'.format(output))
    output_stream.flush()

if __name__ == '__main__':
    cli()
<commit_msg>added flatten_preserver_lists function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import json
try:
    # 3.8 and up
    from collections.abc import Iterable
except ImportError:
    from collections import Iterable

from util import check_if_numbers_are_consecutive
import six
import copy, re
from math import isnan

def _construct_key(previous_key, separator, new_key):
    """
    Returns the new_key if no previous key exists, otherwise concatenates
    previous key, separator, and new_key
    :param previous_key:
    :param separator:
    :param new_key:
    :return: a string if previous_key exists and simply passes through the
    new_key otherwise
    """
    if previous_key:
        return u"{}{}{}".format(previous_key, separator, new_key)
    else:
        return new_key

def flatten(nested_dict, separator="_", root_keys_to_ignore=set()):
    """
    Flattens a dictionary with nested structure to a dictionary with no
    hierarchy
    Consider ignoring keys that you are not interested in to prevent
    unnecessary processing
    This is specially true for very deep objects

    :param nested_dict: dictionary we want to flatten
    :param separator: string to separate dictionary keys by
    :param root_keys_to_ignore: set of root keys to ignore from flattening
    :return: flattened dictionary
    """
    assert isinstance(nested_dict, dict), "flatten requires a dictionary input"
    assert isinstance(separator, six.string_types), "separator must be string"

    # This global dictionary stores the flattened keys and values and is
    # ultimately returned
    flattened_dict = dict()

    def _flatten(object_, key):
        """
        For dict, list and set objects_ calls itself on the elements and for
        other types assigns the object_ to
        the corresponding key in the global flattened_dict
        :param object_: object to flatten
        :param key: carries the concatenated key for the object_
        :return: None
        """
        # Empty object can't be iterated, take as is
        if not object_:
            flattened_dict[key] = object_
        # These object types support iteration
        elif isinstance(object_, dict):
            for object_key in object_:
                if not (not key and object_key in root_keys_to_ignore):
                    _flatten(object_[object_key], _construct_key(key,
                                                                 separator,
                                                                 object_key))
        elif isinstance(object_, list) or isinstance(object_, set):
            for index, item in enumerate(object_):
                _flatten(item, _construct_key(key, separator, index))
        # Anything left take as is
        else:
            flattened_dict[key] = object_

    _flatten(nested_dict, None)
    return flattened_dict

flatten_json = flatten

def flatten_preserve_lists(nested_dict, separator="_", root_keys_to_ignore=set(), max_list_index=3, max_depth=3):
    """
    Flattens a dictionary with nested structure to a dictionary with no
    hierarchy
    Consider ignoring keys that you are not interested in to prevent
    unnecessary processing
    This is specially true for very deep objects
    This preserves list structure, and you can specify max_list_index and max_depth to limit processing

    Child elements with only one value inside will be unwrapped and become parent's value.

    :param nested_dict: dictionary we want to flatten
    :param separator: string to separate dictionary keys by
    :param root_keys_to_ignore: set of root keys to ignore from flattening
    :param max_list_index: maximum list index to process. Any index beyond this will be ignored.
    :param max_depth: maximum nesting depth to process. Anything further down is written as a string into value.
    :return: flattened dictionary
    """

    assert isinstance(nested_dict, dict), "flatten requires a dictionary input"
    assert isinstance(separator, str), "separator must be a string"

    # This global dictionary stores the flattened keys and values and is
    # ultimately returned
    flattened_dict = dict()

    def _flatten(object_, key, cur_depth, max_depth):
        """
        For dict, list and set objects_ calls itself on the elements and for
        other types assigns the object_ to
        the corresponding key in the global flattened_dict
        :param object_: object to flatten
        :param key: carries the concatenated key for the object_
        :return: None
        """

        # Empty object can't be iterated, take as is
        if not object_:
            flattened_dict[key] = object_

        # These object types support iteration
        # dict always go into columns
        elif isinstance(object_, dict):
            first_key = list(object_.keys())[0]
            # if only 1 child value, and it's first_key's value not a dict or list, flatten immediately
            if len(object_) == 1 and not (isinstance(object_[first_key], dict) or isinstance(object_[first_key], list)):
                flattened_dict[key] = object_[first_key]
            else:
                for object_key in object_:
                    if not (not key and object_key in root_keys_to_ignore):
                        _flatten(object_[object_key], _construct_key(key,
                                                                     separator,
                                                                     object_key), cur_depth, max_depth) # build the next child key

        elif isinstance(object_, list) or isinstance(object_, set):
            for index, item in enumerate(object_):
                _flatten(item, _construct_key(key, separator, index), cur_depth, max_depth)

        else:
            flattened_dict[key] = object_

    def _flatten_low_entropy(object_, key, cur_depth, max_depth):
        """
        For dict, list and set objects_ calls itself on the elements and for
        other types assigns the object_ to
        the corresponding key in the global flattened_dict

        :param object_: object to flatten
        :param key: carries the concatenated key for the object_
        :return: None
        """
        cur_depth = cur_depth + 1  # increase current_depth
        debug = 0

        # write latest child as value if max_depth exceeded
        if cur_depth > max_depth:
            global_max_record = int(max(list(list_prebuilt_flattened_dict.keys())))
            for d in list_prebuilt_flattened_dict[str(global_max_record)]:
                d[key] = object_

        else:
            # Empty object can't be iterated, take as is
            if not object_:
                global_max_record = int(max(list(list_prebuilt_flattened_dict.keys())))
                for d in list_prebuilt_flattened_dict[str(global_max_record)]:
                    d[key] = object_

            # These object types support iteration
            # dict always go into columns
            elif isinstance(object_, dict):
                first_key = list(object_.keys())[0]
                # if only 1 child value, and not a dict or list, flatten immediately
                if len(object_) == 1 and not (isinstance(object_[first_key], dict) or isinstance(object_[first_key], list)):
                    global_max_record = int(max(list(list_prebuilt_flattened_dict.keys())))

                    # decide if to update only last (potentially list) of record or all
                    for d in list_prebuilt_flattened_dict[str(global_max_record)]:
                        d[key] = object_[first_key]

                else:
                    for object_key in object_:
                        if not (not key and object_key in root_keys_to_ignore):
                            _flatten_low_entropy(object_[object_key],
                                                 _construct_key(key,
                                                 separator,
                                                object_key), cur_depth, max_depth)  # build the next child key

            # lists could go into rows, like in a relational database
            elif isinstance(object_, list) or isinstance(object_, set):
                if debug: print("\nparent key of list:", key, "| length: ", str(len(object_)))

                # need to remember global list state when we entered this recursion
                global_max_record_start = int(max(list(list_prebuilt_flattened_dict.keys())))
                entry = copy.deepcopy(list_prebuilt_flattened_dict[str(global_max_record_start)])

                for index, item in enumerate(object_):
                    # create new copy of all current records in global list for each element in any list
                    if debug: print("  list key:", key, " index: " + str(index), "vals: ", item)

                    sub = -1
                    if isinstance(item, dict):
                        first_value = list(item.values())[0]
                        if isinstance(first_value, float):
                            sub = first_value

                    if not isnan(sub) and index < max_list_index:
                        if index > 0:  # start from second element, 1st element is like column
                            global_max_record = int(max(list(list_prebuilt_flattened_dict.keys())))
                            if debug: print("copy entry: ", entry)

                            list_prebuilt_flattened_dict[str(global_max_record+1)] = copy.deepcopy(entry)

                            if debug: print("    copied max record in global dict")
                            if debug: print("    len global list last rec: ", len(list_prebuilt_flattened_dict[str(global_max_record)]),
                                  "len list: ", len(list_prebuilt_flattened_dict))

                            # copy only last element(s), by index. Need to index the global list?
                            # in global list, each record can have several rows. Keep an index.

                        # dig in
                        _flatten_low_entropy(item, key, cur_depth, max_depth)
                    else:
                        pass

                list_prebuilt_flattened_dict['0'] = [subel for k, v in list_prebuilt_flattened_dict.items() for idx, subel in enumerate(v)]

                for key in list(list_prebuilt_flattened_dict.keys()):
                    if key != '0':
                        del list_prebuilt_flattened_dict[key]
                if debug: print("collapsed global list")

            # Anything left take as is, assuming you hit the end of the line.
            else:
                # in this case, there may be a list of prebuilt_flattened_dict by now, so need to update them all.
                global_max_record = int(max(list(list_prebuilt_flattened_dict.keys())))

                for d in list_prebuilt_flattened_dict[str(global_max_record)]:
                    d[key] = object_

        # decrease depth counter
        cur_depth -= 1

    _flatten(nested_dict, None, cur_depth=0, max_depth=max_depth)

    # get unique column names, without the integers
    # TODO: potential issue: what if column names have digits naturally?
    reskeys = list(flattened_dict.keys())
    unique_integers = list(set([separator+char for key in reskeys for char in key if char.isdigit()]))
    regex = '|'.join(unique_integers)
    regex += "|" + regex.replace(".", "")
    unique_columns = list(set([re.sub("("+regex+")", "", key) for key in reskeys]))

    # create global dict, now with unique column names
    prebuilt_flattened_dict = {column: None for column in unique_columns}
    # if debug: print("unique columns: ",unique_columns)

    # initialize global record list
    list_prebuilt_flattened_dict = {'0': [prebuilt_flattened_dict]}

    _flatten_low_entropy(nested_dict, None, cur_depth=0, max_depth=max_depth)

    return list_prebuilt_flattened_dict['0']

def _unflatten_asserts(flat_dict, separator):
    assert isinstance(flat_dict, dict), "un_flatten requires dictionary input"
    assert isinstance(separator, six.string_types), "separator must be string"
    assert all((not value or not isinstance(value, Iterable) or
                isinstance(value, six.string_types)
                for value in flat_dict.values())), "provided dict is not flat"

def unflatten(flat_dict, separator='_'):
    """
    Creates a hierarchical dictionary from a flattened dictionary
    Assumes no lists are present
    :param flat_dict: a dictionary with no hierarchy
    :param separator: a string that separates keys
    :return: a dictionary with hierarchy
    """
    _unflatten_asserts(flat_dict, separator)

    # This global dictionary is mutated and returned
    unflattened_dict = dict()

    def _unflatten(dic, keys, value):
        for key in keys[:-1]:
            dic = dic.setdefault(key, {})

        dic[keys[-1]] = value

    list_keys = sorted(flat_dict.keys())
    for i, item in enumerate(list_keys):
        if i != len(list_keys)-1:
            if not list_keys[i+1].startswith(list_keys[i]):
                _unflatten(unflattened_dict, item.split(separator),
                           flat_dict[item])
            else:
                pass  # if key contained in next key, json will be invalid.
        else:
            #  last element
            _unflatten(unflattened_dict, item.split(separator),
                       flat_dict[item])
    return unflattened_dict

def unflatten_list(flat_dict, separator='_'):
    """
    Unflattens a dictionary, first assuming no lists exist and then tries to
    identify lists and replaces them
    This is probably not very efficient and has not been tested extensively
    Feel free to add test cases or rewrite the logic
    Issues that stand out to me:
    - Sorting all the keys in the dictionary, which specially for the root
    dictionary can be a lot of keys
    - Checking that numbers are consecutive is O(N) in number of keys

    :param flat_dict: dictionary with no hierarchy
    :param separator: a string that separates keys
    :return: a dictionary with hierarchy
    """
    _unflatten_asserts(flat_dict, separator)

    # First unflatten the dictionary assuming no lists exist
    unflattened_dict = unflatten(flat_dict, separator)

    def _convert_dict_to_list(object_, parent_object, parent_object_key):
        if isinstance(object_, dict):
            for key in object_:
                if isinstance(object_[key], dict):
                    _convert_dict_to_list(object_[key], object_, key)
            try:
                keys = [int(key) for key in object_]
                keys.sort()
            except (ValueError, TypeError):
                keys = []
            keys_len = len(keys)

            if (keys_len > 0 and sum(keys) ==
                int(((keys_len - 1) * keys_len) / 2) and keys[0] == 0 and
                    keys[-1] == keys_len - 1 and
                    check_if_numbers_are_consecutive(keys)):

                # The dictionary looks like a list so we're going to replace it
                parent_object[parent_object_key] = []
                for key_index, key in enumerate(keys):
                    parent_object[parent_object_key].append(object_[str(key)])
                    # The list item we just added might be a list itself
                    # https://github.com/amirziai/flatten/issues/15
                    _convert_dict_to_list(parent_object[parent_object_key][-1],
                                          parent_object[parent_object_key],
                                          key_index)

    _convert_dict_to_list(unflattened_dict, None, None)
    return unflattened_dict

def cli(input_stream=sys.stdin, output_stream=sys.stdout):
    raw = input_stream.read()
    input_json = json.loads(raw)
    output = json.dumps(flatten(input_json))
    output_stream.write('{}\n'.format(output))
    output_stream.flush()

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Process that subscribes to ALL events and persists them efficiently in bulk into the events datastore"""

import pprint
from gevent.queue import Queue
from gevent.event import Event

from pyon.event.event import EventSubscriber
from pyon.ion.process import StandaloneProcess
from pyon.util.async import spawn
from pyon.public import log

class EventPersister(StandaloneProcess):

    def on_init(self):
        # Time in between event persists
        self.persist_interval = float(self.CFG.get_safe("process.event_persister.persist_interval", 1.0))

        self.persist_blacklist = self.CFG.get_safe("process.event_persister.persist_blacklist", 1.0)

        self._event_type_blacklist = [entry['event_type'] for entry in self.persist_blacklist if entry.get('event_type', None) and len(entry) == 1]
        self._complex_blacklist = [entry for entry in self.persist_blacklist if not (entry.get('event_type', None) and len(entry) == 1)]
        if self._complex_blacklist:
            log.warn("EventPersister does not yet support complex blacklist expressions: %s", self._complex_blacklist)

        # Holds received events FIFO in syncronized queue
        self.event_queue = Queue()

        # Temporarily holds list of events to persist while datastore operation are not yet completed
        # This is where events to persist will remain if datastore operation fails occasionally.
        self.events_to_persist = None

        # Number of unsuccessful attempts to persist in a row
        self.failure_count = 0

        # bookkeeping for timeout greenlet
        self._persist_greenlet = None
        self._terminate_persist = Event() # when set, exits the timeout greenlet

        # The event subscriber
        self.event_sub = None

    def on_start(self):
        # Persister thread
        self._persist_greenlet = spawn(self._trigger_func, self.persist_interval)
        log.debug('EventPersister timer greenlet started in "%s" (interval %s)', self.__class__.__name__, self.persist_interval)

        # Event subscription
        self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS,
                                         callback=self._on_event,
                                         queue_name="event_persister")

        self.event_sub.start()

    def on_quit(self):
        # Stop event subscriber
        self.event_sub.stop()

        # tell the trigger greenlet we're done
        self._terminate_persist.set()

        # wait on the greenlet to finish cleanly
        self._persist_greenlet.join(timeout=10)

    def _on_event(self, event, *args, **kwargs):
        if not self._in_blacklist(event):
            self.event_queue.put(event)

    def _in_blacklist(self, event):
        if event.type_ in self._event_type_blacklist:
            return True
        if event.base_types:
            for base_type in event.base_types:
                if base_type in self._event_type_blacklist:
                    return True
            # TODO: Complex event blacklist
        return False

    def _trigger_func(self, persist_interval):
        log.debug('Starting event persister thread with persist_interval=%s', persist_interval)

        # Event.wait returns False on timeout (and True when set in on_quit), so we use this to both exit cleanly and do our timeout in a loop
        while not self._terminate_persist.wait(timeout=persist_interval):
            try:
                if self.events_to_persist and self.failure_count > 2:
                    bad_events = []
                    log.warn("Attempting to persist %s events individually" % (len(self.events_to_persist)))
                    for event in self.events_to_persist:
                        try:
                            self.container.event_repository.put_event(event)
                        except Exception:
                            bad_events.append(event)

                    if len(self.events_to_persist) != len(bad_events):
                        log.warn("Succeeded to persist some of the events - rest must be bad")
                        self._log_events(bad_events)
                    elif bad_events:
                        log.error("Discarding %s events after %s attempts!!" % (len(self.bad_events), self.failure_count))
                        self._log_events(bad_events)

                    self.events_to_persist = None
                    self.failure_count = 0

                elif self.events_to_persist:
                    # There was an error last time and we need to retry
                    log.info("Retry persisting %s events" % len(self.events_to_persist))
                    self._persist_events(self.events_to_persist)
                    self.events_to_persist = None

                self.events_to_persist = [self.event_queue.get() for x in xrange(self.event_queue.qsize())]

                self._persist_events(self.events_to_persist)
                self.events_to_persist = None
                self.failure_count = 0
            except Exception as ex:
                # Note: Persisting events may fail occasionally during test runs (when the "events" datastore is force
                # deleted and recreated). We'll log and keep retrying forever.
                log.exception("Failed to persist %s received events. Will retry next cycle" % len(self.events_to_persist))
                self.failure_count += 1
                self._log_events(self.events_to_persist)

    def _persist_events(self, event_list):
        if event_list:
            self.container.event_repository.put_events(event_list)

    def _log_events(self, events):
        events_str = pprint.pformat([event.__dict__ for event in events]) if events else ""
        log.warn("EVENTS:\n%s", events_str)
<commit_msg>Fix issue with event persister<commit_after>#!/usr/bin/env python

"""Process that subscribes to ALL events and persists them efficiently in bulk into the events datastore"""

import pprint
from gevent.queue import Queue
from gevent.event import Event

from pyon.event.event import EventSubscriber
from pyon.ion.process import StandaloneProcess
from pyon.util.async import spawn
from pyon.public import log

class EventPersister(StandaloneProcess):

    def on_init(self):
        # Time in between event persists
        self.persist_interval = float(self.CFG.get_safe("process.event_persister.persist_interval", 1.0))

        self.persist_blacklist = self.CFG.get_safe("process.event_persister.persist_blacklist", 1.0)

        self._event_type_blacklist = [entry['event_type'] for entry in self.persist_blacklist if entry.get('event_type', None) and len(entry) == 1]
        self._complex_blacklist = [entry for entry in self.persist_blacklist if not (entry.get('event_type', None) and len(entry) == 1)]
        if self._complex_blacklist:
            log.warn("EventPersister does not yet support complex blacklist expressions: %s", self._complex_blacklist)

        # Holds received events FIFO in syncronized queue
        self.event_queue = Queue()

        # Temporarily holds list of events to persist while datastore operation are not yet completed
        # This is where events to persist will remain if datastore operation fails occasionally.
        self.events_to_persist = None

        # Number of unsuccessful attempts to persist in a row
        self.failure_count = 0

        # bookkeeping for timeout greenlet
        self._persist_greenlet = None
        self._terminate_persist = Event() # when set, exits the timeout greenlet

        # The event subscriber
        self.event_sub = None

    def on_start(self):
        # Persister thread
        self._persist_greenlet = spawn(self._trigger_func, self.persist_interval)
        log.debug('EventPersister timer greenlet started in "%s" (interval %s)', self.__class__.__name__, self.persist_interval)

        # Event subscription
        self.event_sub = EventSubscriber(pattern=EventSubscriber.ALL_EVENTS,
                                         callback=self._on_event,
                                         queue_name="event_persister")

        self.event_sub.start()

    def on_quit(self):
        # Stop event subscriber
        self.event_sub.stop()

        # tell the trigger greenlet we're done
        self._terminate_persist.set()

        # wait on the greenlet to finish cleanly
        self._persist_greenlet.join(timeout=10)

    def _on_event(self, event, *args, **kwargs):
        if not self._in_blacklist(event):
            self.event_queue.put(event)

    def _in_blacklist(self, event):
        if event.type_ in self._event_type_blacklist:
            return True
        if event.base_types:
            for base_type in event.base_types:
                if base_type in self._event_type_blacklist:
                    return True
            # TODO: Complex event blacklist
        return False

    def _trigger_func(self, persist_interval):
        log.debug('Starting event persister thread with persist_interval=%s', persist_interval)

        # Event.wait returns False on timeout (and True when set in on_quit), so we use this to both exit cleanly and do our timeout in a loop
        while not self._terminate_persist.wait(timeout=persist_interval):
            try:
                if self.events_to_persist and self.failure_count > 2:
                    bad_events = []
                    log.warn("Attempting to persist %s events individually" % (len(self.events_to_persist)))
                    for event in self.events_to_persist:
                        try:
                            self.container.event_repository.put_event(event)
                        except Exception:
                            bad_events.append(event)

                    if len(self.events_to_persist) != len(bad_events):
                        log.warn("Succeeded to persist some of the events - rest must be bad")
                        self._log_events(bad_events)
                    elif bad_events:
                        log.error("Discarding %s events after %s attempts!!" % (len(bad_events), self.failure_count))
                        self._log_events(bad_events)

                    self.events_to_persist = None
                    self.failure_count = 0

                elif self.events_to_persist:
                    # There was an error last time and we need to retry
                    log.info("Retry persisting %s events" % len(self.events_to_persist))
                    self._persist_events(self.events_to_persist)
                    self.events_to_persist = None

                self.events_to_persist = [self.event_queue.get() for x in xrange(self.event_queue.qsize())]

                self._persist_events(self.events_to_persist)
                self.events_to_persist = None
                self.failure_count = 0
            except Exception as ex:
                # Note: Persisting events may fail occasionally during test runs (when the "events" datastore is force
                # deleted and recreated). We'll log and keep retrying forever.
                log.exception("Failed to persist %s received events. Will retry next cycle" % len(self.events_to_persist))
                self.failure_count += 1
                self._log_events(self.events_to_persist)

    def _persist_events(self, event_list):
        if event_list:
            self.container.event_repository.put_events(event_list)

    def _log_events(self, events):
        events_str = pprint.pformat([event.__dict__ for event in events]) if events else ""
        log.warn("EVENTS:\n%s", events_str)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

#
# The MIT License (MIT)
#
# Copyright (c) 2013 Camille GUINAUDEAU
# Copyright (c) 2013 Hervé BREDIN (http://herve.niderb.fr/)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

from tvd import Plugin
import re
import urllib3
from bs4 import BeautifulSoup
from tvd import TFloating, TStart, TEnd, AnnotationGraph
from pkg_resources import resource_filename

class GameOfThrones(Plugin):

    def outline(self, url=None, episode=None, **kwargs):
        """
        Parameters
        ----------
        url : str, optional
            URL where resource is available
        episode : Episode, optional
            Episode for which resource should be downloaded
            Useful in case a same URL contains resources for multiple episodes.

        Returns
        -------
        G : AnnotationGraph
        """

        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)
        h2 = soup.find_all('h2')
        sp = ""
        i = 0
        outline = {}

        for element in h2[0].next_elements:
            if element.name == 'p':
                if outline.get(i) == "----":
                    sp = element.text
                else :
                    sp = outline.get(i) + " " + element.text
                outline.update({i:sp})
            if element.name == 'h2':
                i = i + 1
                sp = "----"
                outline.update({i:sp})

        G = AnnotationGraph(episode=episode)
        t2 = TStart()

        i = 1
        while outline.get(i):
            # add /empty/ edge between previous and next annotations
            t1 = t2
            t2 = TFloating()
            G.add_annotation(t1, t2, {})

            # add next annotation
            t1 = t2
            t2 = TFloating()
            G.add_annotation(t1, t2, {'scene': outline.get(i)})

            i = i + 1

        # add /empty/ edge between previous annotation and episode end
        t1 = t2
        t2 = TEnd()
        G.add_annotation(t1, t2, {})

        return G

    def summary(self, url=None, episode=None, **kwargs):
        """
        Parameters
        ----------
        url : str, optional
            URL where resource is available
        episode : Episode, optional
            Episode for which resource should be downloaded
            Useful in case a same URL contains resources for multiple episodes.

        Returns
        -------
        G : AnnotationGraph
        """

        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)

        G = AnnotationGraph(episode=episode)
        t1 = TStart()
        t2 = TFloating()
        G.add_annotation(t1, t2, {})

        sp = ""
        scene_location = ""

        h3 = soup.find_all('h3')
        summary_tag = h3[1]
        ok = 1
        end = 0

        for element in summary_tag.next_elements:
            if element.name == "h4" and ok == 1:
                if end == 1:
                    t3 = TFloating()
                    G.add_annotation(t2, t3, {'location': scene_location, 'summary': sp})
                    sp = ""
                end = 1
                scene_location = element.contents[0].text
            if element.name == "p" and ok == 1:
                sp = sp + " " + element.text
            if element.name == "h3":
                if ok == 1:
                    t3 = TFloating()
                    G.add_annotation(t2, t3, {'location': scene_location, 'summary': sp})
                ok = 0

        # add /empty/ edge between previous annotation and episode end
        t4 = TEnd()
        G.add_annotation(t3, t4, {})

        return G

    def manual_transcript(self, url=None, episode=None, **kwargs):
        pathToMappingFile = resource_filename(self.__class__.name__, 'data/mapping.txt')
        mapping = {}
        f = open(pathToMappingFile, 'r')

        for line in f:
            name = line.rstrip('\n\r').split('\t')
            mapping.update({name[0]: name[1]})

        #print(mapping)
        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)

        G = AnnotationGraph()
        t2 = TStart()

        div = soup.find_all('div')
        transcript = ""

        for i in range(0,len(div)):
            if re.match("{'class': \['postbody'\]}", unicode(div[i].attrs)):
                transcript = div[i]

        transcription = 0
        for i in range(0,len(transcript.contents)):
            string = unicode(transcript.contents[i])
            if not re.match("\[(.*)\]", string):
                if re.match("(.*) : (.*)", string) and not re.match("(.*) by : (.*)", string):
                    ligne = re.split(' : ', transcript.contents[i])

                    # add /empty/ edge between previous and next annotations
                    t1 = t2
                    t2 = TFloating()
                    G.add_annotation(t1, t2, {})

                    # add next annotation
                    t1 = t2
                    t2 = TFloating()

                    spk = ligne[0].lower().replace(' ', '_')

                    if re.match("(.*)_\(|\[(.*)\)|\]", spk):
                        match = re.match("(.*)_\(|\[(.*)\)|\]", spk)
                        spk = match.group(1)
                    if spk in mapping:
                        spk = mapping.get(spk)

                    if re.match("(.*)/(.*)", spk):
                        spks = spk.split('/')
                        if spks[0] in mapping:
                            spk = mapping.get(spks[0])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                        if spks[1] in mapping:
                            spk = mapping.get(spks[1])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                    else:
                        G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})

                elif re.match("(.*): (.*)", string) and not re.match("Credit: (.*)", string) and not re.match("(.*) by: (.*)", string):
                    ligne = re.split(': ', transcript.contents[i])

                    # add /empty/ edge between previous and next annotations
                    t1 = t2
                    t2 = TFloating()
                    G.add_annotation(t1, t2, {})

                    # add next annotation
                    t1 = t2
                    t2 = TFloating()
                    spk = ligne[0].lower().replace(' ', '_')

                    if re.match("(.*)_\(|\[(.*)\)|\]", spk):
                        match = re.match("(.*)_\(|\[(.*)\)|\]", spk)
                        spk = match.group(1)
                    if spk in mapping:
                        spk = mapping.get(spk)

                    if re.match("(.*)/(.*)", spk):
                        spks = spk.split('/')
                        if spks[0] in mapping:
                            spk = mapping.get(spks[0])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                        if spks[1] in mapping:
                            spk = mapping.get(spks[1])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                    else:
                        G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})

        # add /empty/ edge between previous annotation and episode end
        t1 = t2
        t2 = TEnd()
        G.add_annotation(t1, t2, {})

        return G
<commit_msg>Character names normalization<commit_after>#!/usr/bin/env python
# encoding: utf-8

#
# The MIT License (MIT)
#
# Copyright (c) 2013 Camille GUINAUDEAU
# Copyright (c) 2013 Hervé BREDIN (http://herve.niderb.fr/)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

from tvd import Plugin
import re
import urllib3
from bs4 import BeautifulSoup
from tvd import TFloating, TStart, TEnd, AnnotationGraph
from pkg_resources import resource_filename

class GameOfThrones(Plugin):

    def outline(self, url=None, episode=None, **kwargs):
        """
        Parameters
        ----------
        url : str, optional
            URL where resource is available
        episode : Episode, optional
            Episode for which resource should be downloaded
            Useful in case a same URL contains resources for multiple episodes.

        Returns
        -------
        G : AnnotationGraph
        """

        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)
        h2 = soup.find_all('h2')
        sp = ""
        i = 0
        outline = {}

        for element in h2[0].next_elements:
            if element.name == 'p':
                if outline.get(i) == "----":
                    sp = element.text
                else :
                    sp = outline.get(i) + " " + element.text
                outline.update({i:sp})
            if element.name == 'h2':
                i = i + 1
                sp = "----"
                outline.update({i:sp})

        G = AnnotationGraph(episode=episode)
        t2 = TStart()

        i = 1
        while outline.get(i):
            # add /empty/ edge between previous and next annotations
            t1 = t2
            t2 = TFloating()
            G.add_annotation(t1, t2, {})

            # add next annotation
            t1 = t2
            t2 = TFloating()
            G.add_annotation(t1, t2, {'scene': outline.get(i)})

            i = i + 1

        # add /empty/ edge between previous annotation and episode end
        t1 = t2
        t2 = TEnd()
        G.add_annotation(t1, t2, {})

        return G

    def summary(self, url=None, episode=None, **kwargs):
        """
        Parameters
        ----------
        url : str, optional
            URL where resource is available
        episode : Episode, optional
            Episode for which resource should be downloaded
            Useful in case a same URL contains resources for multiple episodes.

        Returns
        -------
        G : AnnotationGraph
        """

        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)

        G = AnnotationGraph(episode=episode)
        t1 = TStart()
        t2 = TFloating()
        G.add_annotation(t1, t2, {})

        sp = ""
        scene_location = ""

        h3 = soup.find_all('h3')
        summary_tag = h3[1]
        ok = 1
        end = 0

        for element in summary_tag.next_elements:
            if element.name == "h4" and ok == 1:
                if end == 1:
                    t3 = TFloating()
                    G.add_annotation(t2, t3, {'location': scene_location, 'summary': sp})
                    sp = ""
                end = 1
                scene_location = element.contents[0].text
            if element.name == "p" and ok == 1:
                sp = sp + " " + element.text
            if element.name == "h3":
                if ok == 1:
                    t3 = TFloating()
                    G.add_annotation(t2, t3, {'location': scene_location, 'summary': sp})
                ok = 0

        # add /empty/ edge between previous annotation and episode end
        t4 = TEnd()
        G.add_annotation(t3, t4, {})

        return G

    def manual_transcript(self, url=None, episode=None, **kwargs):
        pathToMappingFile = resource_filename(self.__class__.name__, 'data/mapping.txt')
        mapping = {}
        f = open(pathToMappingFile, 'r')

        for line in f:
            name = line.rstrip('\n\r').split('\t')
            mapping.update({name[0]: name[1]})

        f.close()

        http = urllib3.PoolManager()
        r = http.request('GET', url)
        soup = BeautifulSoup(r.data)

        G = AnnotationGraph()
        t2 = TStart()

        div = soup.find_all('div')
        transcript = ""

        for i in range(0,len(div)):
            if re.match("{'class': \['postbody'\]}", unicode(div[i].attrs)):
                transcript = div[i]

        transcription = 0
        for i in range(0,len(transcript.contents)):
            string = unicode(transcript.contents[i])
            if not re.match("\[(.*)\]", string):
                if re.match("(.*) : (.*)", string) and not re.match("(.*) by : (.*)", string):
                    ligne = re.split(' : ', transcript.contents[i])

                    # add /empty/ edge between previous and next annotations
                    t1 = t2
                    t2 = TFloating()
                    G.add_annotation(t1, t2, {})

                    # add next annotation
                    t1 = t2
                    t2 = TFloating()

                    spk = ligne[0].lower().replace(' ', '_')

                    if re.match("(.*)_\(|\[(.*)\)|\]", spk):
                        match = re.match("(.*)_\(|\[(.*)\)|\]", spk)
                        spk = match.group(1)
                    if spk in mapping:
                        spk = mapping.get(spk)

                    if re.match("(.*)/(.*)", spk):
                        spks = spk.split('/')
                        if spks[0] in mapping:
                            spk = mapping.get(spks[0])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                        if spks[1] in mapping:
                            spk = mapping.get(spks[1])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                    else:
                        G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})

                elif re.match("(.*): (.*)", string) and not re.match("Credit: (.*)", string) and not re.match("(.*) by: (.*)", string):
                    ligne = re.split(': ', transcript.contents[i])

                    # add /empty/ edge between previous and next annotations
                    t1 = t2
                    t2 = TFloating()
                    G.add_annotation(t1, t2, {})

                    # add next annotation
                    t1 = t2
                    t2 = TFloating()
                    spk = ligne[0].lower().replace(' ', '_')

                    if re.match("(.*)_\(|\[(.*)\)|\]", spk):
                        match = re.match("(.*)_\(|\[(.*)\)|\]", spk)
                        spk = match.group(1)
                    if spk in mapping:
                        spk = mapping.get(spk)

                    if re.match("(.*)/(.*)", spk):
                        spks = spk.split('/')
                        if spks[0] in mapping:
                            spk = mapping.get(spks[0])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                        if spks[1] in mapping:
                            spk = mapping.get(spks[1])
                            G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})
                    else:
                        G.add_annotation(t1, t2, {'speaker': spk, 'speech': ligne[1]})

        # add /empty/ edge between previous annotation and episode end
        t1 = t2
        t2 = TEnd()
        G.add_annotation(t1, t2, {})

        return G
<|endoftext|>


<commit_before>from __future__ import division,print_function
import os,os.path
import numpy as np
import pkg_resources

from scipy.interpolate import LinearNDInterpolator as interpnd
import pandas as pd
import pickle

from .isochrones import Isochrone

#DATADIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data'))
DATADIR = os.getenv('ISOCHRONES',
                    os.path.expanduser(os.path.join('~','.isochrones')))
if not os.path.exists(DATADIR):
    os.mkdir(DATADIR)

MASTERFILE = '{}/dartmouth.h5'.format(DATADIR)
TRI_FILE = '{}/dartmouth.tri'.format(DATADIR)

def _download_h5():
    url = 'http://zenodo.org/record/12800/files/dartmouth.h5'
    import urllib
    print('Downloading Dartmouth stellar model data (should happen only once)...')
    if os.path.exists(MASTERFILE):
        os.remove(MASTERFILE)
    urllib.urlretrieve(url,MASTERFILE)

def _download_tri():
    url = 'http://zenodo.org/record/12800/files/dartmouth.tri'
    import urllib
    print('Downloading Dartmouth isochrone pre-computed triangulation (should happen only once...)')
    if os.path.exists(TRI_FILE):
        os.remove(TRI_FILE)
    urllib.urlretrieve(url,TRI_FILE)

if not os.path.exists(MASTERFILE):
    _download_h5()

if not os.path.exists(TRI_FILE):
    _download_tri()

MASTERDF = pd.read_hdf(MASTERFILE,'df')

class Dartmouth_Isochrone(Isochrone):
    """Dotter (2008) Stellar Models
    """
    def __init__(self,bands=['U','B','V','R','I','J','H',
                             'K','g','r','i','z','Kepler','D51',
                             'W1','W2','W3','W4']):

        df = MASTERDF

        mags = {}
        for band in bands:
            try:
                if band in ['g','r','i','z']:
                    mags[band] = df['sdss_{}'.format(band)]
                else:
                    mags[band] = df[band]
            except:
                if band == 'kep' or band == 'Kepler':
                    mags[band] = df['Kp']
                elif band == 'K':
                    mags['K'] = df['Ks']
                else:
                    raise

        f = open(TRI_FILE,'rb')
        tri = pickle.load(f)
        f.close()

        Isochrone.__init__(self,df['M/Mo'],np.log10(df['age']*1e9),
                           df['feh'],df['M/Mo'],df['LogL/Lo'],
                           10**df['LogTeff'],df['LogG'],mags,tri=tri)

############ utility functions used to set up data sets from original isochrone data files ########

DARTMOUTH_DATAFOLDER = ''  #this would be location of raw data files.

def write_tri(df=MASTERDF, outfile=TRI_FILE):
    """Writes the Delanuay triangulation of the models to file.  Takes a few minutes, so beware.

    Typically no need to do this unless you can't download the .tri file for some reason...
    """
    N = len(df)
    pts = np.zeros((N,3))
    pts[:,0] = np.array(df['M'])
    pts[:,1] = np.log10(np.array(df['age'])*1e9)
    pts[:,2] = np.array(df['feh'])
    Jmags = np.array(df['J'])

    Jfn = interpnd(pts,Jmags)

    f = open(outfile,'wb')
    pickle.dump(Jfn.tri,f)
    f.close()

def fehstr(feh,minfeh=-1.0,maxfeh=0.5):
        if feh < minfeh:
            return '%.1f' % minfeh
        elif feh > maxfeh:
            return '%.1f' % maxfeh
        elif (feh > -0.05 and feh < 0):
            return '0.0'
        else:
            return '%.1f' % feh

def writeall_h5(fehs=np.arange(-1,0.51,0.1)):
    store = pd.HDFStore('%s/dartmouth.h5' % DARTMOUTH_DATAFOLDER)
    for feh in fehs:
        name = fehstr(feh)
        print(name)
        store[name] = dartmouth_to_df(feh)
    store.close()

def dartmouth_all_df(fehs=np.arange(-1,0.51,0.1),savefile=None):
    store = pd.HDFStore('%s/dartmouth.h5' % DARTMOUTH_DATAFOLDER)
    df = pd.DataFrame()
    for feh in fehs:
        newdf = store[fehstr(feh)].copy()
        newdf['feh'] = np.ones(len(newdf))*feh
        df = df.append(newdf)
    store.close()
    if savefile is not None:
        df.to_hdf(savefile)
    return df

def dartmouth_to_df(feh):
    filename_2mass = '%s/dartmouth_%s_2massKp.iso' % (DARTMOUTH_DATAFOLDER,fehstr(feh,-1.0,0.5))
    filename_ugriz = '%s/dartmouth_%s_ugriz.iso' % (DARTMOUTH_DATAFOLDER,fehstr(feh,-1.0,0.5))

    bands_ugriz = ['u','g','r','i','z']
    darnames_ugriz = ['sdss_u','sdss_g','sdss_r','sdss_i','sdss_z']

    bands_2mass = ['U','B','V','R','I','J','H','Ks','Kp','D51']
    darnames_2mass = bands_2mass

    rec = np.recfromtxt(filename_2mass,skiprows=8,names=True)
    rec2 = np.recfromtxt(filename_ugriz,skiprows=8,names=True,usecols=(5,6,7,8,9))
    df = pd.DataFrame(rec)
    df2 = pd.DataFrame(rec2)
    for b,d in zip(bands_ugriz,darnames_ugriz):
        df[b] = df2[d]

    n = len(df)
    ages = np.zeros(n)
    curage = 0
    i=0
    for line in open(filename_2mass):
        m = re.match('#',line)
        if m:
            m = re.match('#AGE=\s*(\d+\.\d+)\s+',line)
            if m:
                curage=float(m.group(1))
        else:
            if re.search('\d',line):
                ages[i]=curage
                i+=1

    df['age'] = ages
    columns = {darnames_2mass[i]:bands_2mass[i] for i in range(len(bands_2mass))}
    columns.update({'MMo':'M','LogTeff':'logTeff',
                    'LogG':'logg','LogLLo':'logL'})
    df.rename(columns=columns,inplace=True)
    return df
<commit_msg>updated zenodo urls for new dartmouth files<commit_after>from __future__ import division,print_function
import os,os.path
import numpy as np
import pkg_resources

from scipy.interpolate import LinearNDInterpolator as interpnd
import pandas as pd
import pickle

from .isochrones import Isochrone

#DATADIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data'))
DATADIR = os.getenv('ISOCHRONES',
                    os.path.expanduser(os.path.join('~','.isochrones')))
if not os.path.exists(DATADIR):
    os.mkdir(DATADIR)

MASTERFILE = '{}/dartmouth.h5'.format(DATADIR)
TRI_FILE = '{}/dartmouth.tri'.format(DATADIR)

def _download_h5():
    #url = 'http://zenodo.org/record/12800/files/dartmouth.h5'
    url = 'http://zenodo.org/record/15843/files/dartmouth.h5'
    import urllib
    print('Downloading Dartmouth stellar model data (should happen only once)...')
    if os.path.exists(MASTERFILE):
        os.remove(MASTERFILE)
    urllib.urlretrieve(url,MASTERFILE)

def _download_tri():
    #url = 'http://zenodo.org/record/12800/files/dartmouth.tri'
    url = 'http://zenodo.org/record/15843/files/dartmouth.tri'
    import urllib
    print('Downloading Dartmouth isochrone pre-computed triangulation (should happen only once...)')
    if os.path.exists(TRI_FILE):
        os.remove(TRI_FILE)
    urllib.urlretrieve(url,TRI_FILE)

if not os.path.exists(MASTERFILE):
    _download_h5()

if not os.path.exists(TRI_FILE):
    _download_tri()

MASTERDF = pd.read_hdf(MASTERFILE,'df')

class Dartmouth_Isochrone(Isochrone):
    """Dotter (2008) Stellar Models
    """
    def __init__(self,bands=['U','B','V','R','I','J','H',
                             'K','g','r','i','z','Kepler','D51',
                             'W1','W2','W3','W4']):

        df = MASTERDF

        mags = {}
        for band in bands:
            try:
                if band in ['g','r','i','z']:
                    mags[band] = df['sdss_{}'.format(band)]
                else:
                    mags[band] = df[band]
            except:
                if band == 'kep' or band == 'Kepler':
                    mags[band] = df['Kp']
                elif band == 'K':
                    mags['K'] = df['Ks']
                else:
                    raise

        f = open(TRI_FILE,'rb')
        tri = pickle.load(f)
        f.close()

        Isochrone.__init__(self,df['M/Mo'],np.log10(df['age']*1e9),
                           df['feh'],df['M/Mo'],df['LogL/Lo'],
                           10**df['LogTeff'],df['LogG'],mags,tri=tri)

############ utility functions used to set up data sets from original isochrone data files ########

DARTMOUTH_DATAFOLDER = ''  #this would be location of raw data files.

def write_tri(df=MASTERDF, outfile=TRI_FILE):
    """Writes the Delanuay triangulation of the models to file.  Takes a few minutes, so beware.

    Typically no need to do this unless you can't download the .tri file for some reason...
    """
    N = len(df)
    pts = np.zeros((N,3))
    pts[:,0] = np.array(df['M'])
    pts[:,1] = np.log10(np.array(df['age'])*1e9)
    pts[:,2] = np.array(df['feh'])
    Jmags = np.array(df['J'])

    Jfn = interpnd(pts,Jmags)

    f = open(outfile,'wb')
    pickle.dump(Jfn.tri,f)
    f.close()

def fehstr(feh,minfeh=-1.0,maxfeh=0.5):
        if feh < minfeh:
            return '%.1f' % minfeh
        elif feh > maxfeh:
            return '%.1f' % maxfeh
        elif (feh > -0.05 and feh < 0):
            return '0.0'
        else:
            return '%.1f' % feh

def writeall_h5(fehs=np.arange(-1,0.51,0.1)):
    store = pd.HDFStore('%s/dartmouth.h5' % DARTMOUTH_DATAFOLDER)
    for feh in fehs:
        name = fehstr(feh)
        print(name)
        store[name] = dartmouth_to_df(feh)
    store.close()

def dartmouth_all_df(fehs=np.arange(-1,0.51,0.1),savefile=None):
    store = pd.HDFStore('%s/dartmouth.h5' % DARTMOUTH_DATAFOLDER)
    df = pd.DataFrame()
    for feh in fehs:
        newdf = store[fehstr(feh)].copy()
        newdf['feh'] = np.ones(len(newdf))*feh
        df = df.append(newdf)
    store.close()
    if savefile is not None:
        df.to_hdf(savefile)
    return df

def dartmouth_to_df(feh):
    filename_2mass = '%s/dartmouth_%s_2massKp.iso' % (DARTMOUTH_DATAFOLDER,fehstr(feh,-1.0,0.5))
    filename_ugriz = '%s/dartmouth_%s_ugriz.iso' % (DARTMOUTH_DATAFOLDER,fehstr(feh,-1.0,0.5))

    bands_ugriz = ['u','g','r','i','z']
    darnames_ugriz = ['sdss_u','sdss_g','sdss_r','sdss_i','sdss_z']

    bands_2mass = ['U','B','V','R','I','J','H','Ks','Kp','D51']
    darnames_2mass = bands_2mass

    rec = np.recfromtxt(filename_2mass,skiprows=8,names=True)
    rec2 = np.recfromtxt(filename_ugriz,skiprows=8,names=True,usecols=(5,6,7,8,9))
    df = pd.DataFrame(rec)
    df2 = pd.DataFrame(rec2)
    for b,d in zip(bands_ugriz,darnames_ugriz):
        df[b] = df2[d]

    n = len(df)
    ages = np.zeros(n)
    curage = 0
    i=0
    for line in open(filename_2mass):
        m = re.match('#',line)
        if m:
            m = re.match('#AGE=\s*(\d+\.\d+)\s+',line)
            if m:
                curage=float(m.group(1))
        else:
            if re.search('\d',line):
                ages[i]=curage
                i+=1

    df['age'] = ages
    columns = {darnames_2mass[i]:bands_2mass[i] for i in range(len(bands_2mass))}
    columns.update({'MMo':'M','LogTeff':'logTeff',
                    'LogG':'logg','LogLLo':'logL'})
    df.rename(columns=columns,inplace=True)
    return df
<|endoftext|>


<commit_before>import os,re, glob
import numpy as np
import pandas as pd
import logging
import tarfile

from ..config import ISOCHRONES
from ..grid import ModelGrid

class MISTModelGrid(ModelGrid):
    name = 'mist'
    common_columns = ('EEP', 'log10_isochrone_age_yr', 'initial_mass',
                        'log_Teff', 'log_g', 'log_L', 'Z_surf', 'feh', 'phase')

    phot_systems = ('CFHT', 'DECam', 'GALEX', 'JWST', 'LSST', 'PanSTARRS',
                    'SDSS', 'SPITZER', 'SkyMapper', 'UBVRIplus', 'UKIDSS', 'WISE')

    phot_bands = dict(UBVRIplus=['Bessell_U', 'Bessell_B', 'Bessell_V',
                        'Bessell_R', 'Bessell_I', '2MASS_J', '2MASS_H', '2MASS_Ks',
                        'Kepler_Kp', 'Kepler_D51', 'Hipparcos_Hp',
                        'Tycho_B', 'Tycho_V', 'Gaia_G'],
                      WISE=['WISE_W1', 'WISE_W2', 'WISE_W3', 'WISE_W4'],
                      CFHT=['CFHT_u', 'CFHT_g', 'CFHT_r',
                            'CFHT_i_new', 'CFHT_i_old', 'CFHT_z'],
                      DECam=['DECam_u', 'DECam_g', 'DECam_r',
                             'DECam_i', 'DECam_z', 'DECam_Y'],
                      GALEX=['GALEX_FUV', 'GALEX_NUV'],
                      JWST=['F070W', 'F090W', 'F115W', 'F140M',
                           'F150W2', 'F150W', 'F162M', 'F164N', 'F182M', 'F187N', 'F200W',
                           'F210M', 'F212N', 'F250M', 'F277W', 'F300M', 'F322W2', 'F323N',
                           'F335M', 'F356W', 'F360M', 'F405N', 'F410M', 'F430M', 'F444W',
                           'F460M', 'F466N', 'F470N', 'F480M'],
                      LSST=['LSST_u', 'LSST_g', 'LSST_r',
                            'LSST_i', 'LSST_z', 'LSST_y'],
                      PanSTARRS=['PS_g', 'PS_r', 'PS_i', 'PS_z',
                                 'PS_y', 'PS_w', 'PS_open'],
                      SkyMapper=['SkyMapper_u', 'SkyMapper_v', 'SkyMapper_g',
                                 'SkyMapper_r', 'SkyMapper_i', 'SkyMapper_z'],
                      SPITZER=['IRAC_3.6', 'IRAC_4.5', 'IRAC_5.8', 'IRAC_8.0'],
                      UKIDSS=['UKIDSS_Z', 'UKIDSS_Y', 'UKIDSS_J',
                                'UKIDSS_H', 'UKIDSS_K'],
                      SDSS=['SDSS_u', 'SDSS_g', 'SDSS_r', 'SDSS_i', 'SDSS_z'])

    default_kwargs = {'version':'1.0'}
    datadir = os.path.join(ISOCHRONES, 'mist')
    zenodo_record = 161241
    zenodo_files = ('mist.tgz')
    master_tarball_file = 'mist.tgz'

    @classmethod
    def get_band(cls, b):
        """Defines what a "shortcut" band name refers to.  Returns phot_system, band

        """
        phot = None

        # Default to SDSS for these
        if b in ['u','g','r','i','z']:
            phot = 'SDSS'
            band = 'SDSS_{}'.format(b)
        elif b in ['B','V']:
            phot = 'UBVRIplus'
            band = 'Tycho_{}'.format(b)
        elif b in ['U','R','I']:
            phot = 'UBVRIplus'
            band = 'Bessel_{}'.format(b)
        elif b in  ['J','H','Ks']:
            phot = 'UBVRIplus'
            band = '2MASS_{}'.format(b)
        elif b=='K':
            phot = 'UBVRIplus'
            band = '2MASS_Ks'
        elif b in ['kep','Kepler','Kp']:
            phot = 'UBVRIplus'
            band = 'Kepler_Kp'
        elif b in ['W1','W2','W3','W4']:
            phot = 'WISE'
            band = 'WISE_{}'.format(b)
        elif b=='G':
            phot = 'UBVRIplus'
            band = 'Gaia_G'
        else:
            m = re.match('([a-zA-Z]+)_([a-zA-Z_]+)',b)
            if m:
                if m.group(1) in self.phot_systems:
                    phot = m.group(1)
                    if phot=='PanSTARRS':
                        band = 'PS_{}'.format(m.group(2))
                    else:
                        band = m.group(0)
                elif m.group(1) in ['UK','UKIRT']:
                    phot = 'UKIDSS'
                    band = 'UKIDSS_{}'.format(m.group(2))
        if phot is None:
            raise ValueError('MIST grids cannot resolve band {}!'.format(b))
        return phot, band

    def phot_tarball_file(self, phot, version='1.0'):
        return os.path.join(self.datadir, 'MIST_v{}_{}.tar.gz'.format(version, phot))

    def extract_phot_tarball(self, phot, version='1.0'):
        phot_tarball = self.phot_tarball_file(phot)
        with tarfile.open(phot_tarball) as tar:
            logging.info('Extracting {}...'.format(phot_tarball))
            tar.extractall(self.datadir)

    def get_filenames(self, phot, version='1.0'):
        d = os.path.join(self.datadir, 'MIST_v{}_{}'.format(version, phot))
        if not os.path.exists(d):
            if not os.path.exists(self.phot_tarball_file(phot, version=version)):
                self.extract_master_tarball()
            self.extract_phot_tarball(phot, version=version)

        return [os.path.join(d,f) for f in os.listdir(d) if re.search('\.cmd$', f)]

    @classmethod
    def get_feh(cls, filename):
        m = re.search('feh_([mp])([0-9]\.[0-9]{2})_afe', filename)
        if m:
            sign = 1 if m.group(1)=='p' else -1
            return float(m.group(2)) * sign
        else:
            raise ValueError('{} not a valid MIST file? Cannnot parse [Fe/H]'.format(filename))

    @classmethod
    def to_df(cls, filename):
        with open(filename, 'r') as fin:
            while True:
                line = fin.readline()
                if re.match('# EEP', line):
                    column_names = line[1:].split()
                    break
        feh = cls.get_feh(filename)
        df = pd.read_table(filename, comment='#', delim_whitespace=True,
                             skip_blank_lines=True, names=column_names)
        df['feh'] = feh
        return df

    def df_all(self, phot):
        df = super(MISTModelGrid, self).df_all(phot)
        df = df.sort_values(by=['feh','log10_isochrone_age_yr','initial_mass'])
        df.index = [df.feh, df.log10_isochrone_age_yr]
        return df

    def hdf_filename(self, phot, version='1.0'):
        return os.path.join(self.datadir, 'MIST_v{}_{}.h5'.format(version, phot))

<commit_msg>made sure files is iterable<commit_after>import os,re, glob
import numpy as np
import pandas as pd
import logging
import tarfile

from ..config import ISOCHRONES
from ..grid import ModelGrid

class MISTModelGrid(ModelGrid):
    name = 'mist'
    common_columns = ('EEP', 'log10_isochrone_age_yr', 'initial_mass',
                        'log_Teff', 'log_g', 'log_L', 'Z_surf', 'feh', 'phase')

    phot_systems = ('CFHT', 'DECam', 'GALEX', 'JWST', 'LSST', 'PanSTARRS',
                    'SDSS', 'SPITZER', 'SkyMapper', 'UBVRIplus', 'UKIDSS', 'WISE')

    phot_bands = dict(UBVRIplus=['Bessell_U', 'Bessell_B', 'Bessell_V',
                        'Bessell_R', 'Bessell_I', '2MASS_J', '2MASS_H', '2MASS_Ks',
                        'Kepler_Kp', 'Kepler_D51', 'Hipparcos_Hp',
                        'Tycho_B', 'Tycho_V', 'Gaia_G'],
                      WISE=['WISE_W1', 'WISE_W2', 'WISE_W3', 'WISE_W4'],
                      CFHT=['CFHT_u', 'CFHT_g', 'CFHT_r',
                            'CFHT_i_new', 'CFHT_i_old', 'CFHT_z'],
                      DECam=['DECam_u', 'DECam_g', 'DECam_r',
                             'DECam_i', 'DECam_z', 'DECam_Y'],
                      GALEX=['GALEX_FUV', 'GALEX_NUV'],
                      JWST=['F070W', 'F090W', 'F115W', 'F140M',
                           'F150W2', 'F150W', 'F162M', 'F164N', 'F182M', 'F187N', 'F200W',
                           'F210M', 'F212N', 'F250M', 'F277W', 'F300M', 'F322W2', 'F323N',
                           'F335M', 'F356W', 'F360M', 'F405N', 'F410M', 'F430M', 'F444W',
                           'F460M', 'F466N', 'F470N', 'F480M'],
                      LSST=['LSST_u', 'LSST_g', 'LSST_r',
                            'LSST_i', 'LSST_z', 'LSST_y'],
                      PanSTARRS=['PS_g', 'PS_r', 'PS_i', 'PS_z',
                                 'PS_y', 'PS_w', 'PS_open'],
                      SkyMapper=['SkyMapper_u', 'SkyMapper_v', 'SkyMapper_g',
                                 'SkyMapper_r', 'SkyMapper_i', 'SkyMapper_z'],
                      SPITZER=['IRAC_3.6', 'IRAC_4.5', 'IRAC_5.8', 'IRAC_8.0'],
                      UKIDSS=['UKIDSS_Z', 'UKIDSS_Y', 'UKIDSS_J',
                                'UKIDSS_H', 'UKIDSS_K'],
                      SDSS=['SDSS_u', 'SDSS_g', 'SDSS_r', 'SDSS_i', 'SDSS_z'])

    default_kwargs = {'version':'1.0'}
    datadir = os.path.join(ISOCHRONES, 'mist')
    zenodo_record = 161241
    zenodo_files = ('mist.tgz',)
    master_tarball_file = 'mist.tgz'

    @classmethod
    def get_band(cls, b):
        """Defines what a "shortcut" band name refers to.  Returns phot_system, band

        """
        phot = None

        # Default to SDSS for these
        if b in ['u','g','r','i','z']:
            phot = 'SDSS'
            band = 'SDSS_{}'.format(b)
        elif b in ['B','V']:
            phot = 'UBVRIplus'
            band = 'Tycho_{}'.format(b)
        elif b in ['U','R','I']:
            phot = 'UBVRIplus'
            band = 'Bessel_{}'.format(b)
        elif b in  ['J','H','Ks']:
            phot = 'UBVRIplus'
            band = '2MASS_{}'.format(b)
        elif b=='K':
            phot = 'UBVRIplus'
            band = '2MASS_Ks'
        elif b in ['kep','Kepler','Kp']:
            phot = 'UBVRIplus'
            band = 'Kepler_Kp'
        elif b in ['W1','W2','W3','W4']:
            phot = 'WISE'
            band = 'WISE_{}'.format(b)
        elif b=='G':
            phot = 'UBVRIplus'
            band = 'Gaia_G'
        else:
            m = re.match('([a-zA-Z]+)_([a-zA-Z_]+)',b)
            if m:
                if m.group(1) in self.phot_systems:
                    phot = m.group(1)
                    if phot=='PanSTARRS':
                        band = 'PS_{}'.format(m.group(2))
                    else:
                        band = m.group(0)
                elif m.group(1) in ['UK','UKIRT']:
                    phot = 'UKIDSS'
                    band = 'UKIDSS_{}'.format(m.group(2))
        if phot is None:
            raise ValueError('MIST grids cannot resolve band {}!'.format(b))
        return phot, band

    def phot_tarball_file(self, phot, version='1.0'):
        return os.path.join(self.datadir, 'MIST_v{}_{}.tar.gz'.format(version, phot))

    def extract_phot_tarball(self, phot, version='1.0'):
        phot_tarball = self.phot_tarball_file(phot)
        with tarfile.open(phot_tarball) as tar:
            logging.info('Extracting {}...'.format(phot_tarball))
            tar.extractall(self.datadir)

    def get_filenames(self, phot, version='1.0'):
        d = os.path.join(self.datadir, 'MIST_v{}_{}'.format(version, phot))
        if not os.path.exists(d):
            if not os.path.exists(self.phot_tarball_file(phot, version=version)):
                self.extract_master_tarball()
            self.extract_phot_tarball(phot, version=version)

        return [os.path.join(d,f) for f in os.listdir(d) if re.search('\.cmd$', f)]

    @classmethod
    def get_feh(cls, filename):
        m = re.search('feh_([mp])([0-9]\.[0-9]{2})_afe', filename)
        if m:
            sign = 1 if m.group(1)=='p' else -1
            return float(m.group(2)) * sign
        else:
            raise ValueError('{} not a valid MIST file? Cannnot parse [Fe/H]'.format(filename))

    @classmethod
    def to_df(cls, filename):
        with open(filename, 'r') as fin:
            while True:
                line = fin.readline()
                if re.match('# EEP', line):
                    column_names = line[1:].split()
                    break
        feh = cls.get_feh(filename)
        df = pd.read_table(filename, comment='#', delim_whitespace=True,
                             skip_blank_lines=True, names=column_names)
        df['feh'] = feh
        return df

    def df_all(self, phot):
        df = super(MISTModelGrid, self).df_all(phot)
        df = df.sort_values(by=['feh','log10_isochrone_age_yr','initial_mass'])
        df.index = [df.feh, df.log10_isochrone_age_yr]
        return df

    def hdf_filename(self, phot, version='1.0'):
        return os.path.join(self.datadir, 'MIST_v{}_{}.h5'.format(version, phot))

<|endoftext|>


<commit_before>PLUGIN_VERSION = '2.2.8'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.03'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<commit_msg>Tag new release: 2.2.9<commit_after>PLUGIN_VERSION = '2.2.9'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.03'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<|endoftext|>


<commit_before>PLUGIN_VERSION = '3.1.17'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<commit_msg>Tag new release: 3.1.18<commit_after>PLUGIN_VERSION = '3.1.18'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<|endoftext|>


<commit_before>PLUGIN_VERSION = '2.6.4'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<commit_msg>Tag new release: 2.7.0<commit_after>PLUGIN_VERSION = '2.7.0'
# The line above is auto-generated by tag_release.py. Do not change it manually.

try:
    from .common import shared as G
    assert G
except ImportError:
    from common import shared as G

G.__VERSION__ = '0.11'
G.__PLUGIN_VERSION__ = PLUGIN_VERSION
<|endoftext|>


<commit_before>"""A simple web server that accepts POSTS containing a list of feed urls,
and returns the titles of those feeds.
"""
import datetime
import eventlet
import json
import os
import logging
from genshi.template import TextTemplate

logging.basicConfig(level=logging.DEBUG)

# the pool provides a safety limit on our concurrency
pool = eventlet.GreenPool()

class LookupError(Exception):
    pass

class RenderError(Exception):
    pass

class AttributeResolutionError(Exception):
    pass

def dthandler(obj):
    if isinstance(obj, datetime.datetime):
        return obj.isoformat()
    return obj

class fm(object):
    def __init__(self, root):
        self.root = root
        self.static = root +'/static'
        self.dynamic = root +'/dynamic'
        self.data = dict()
        logging.info("Starting")
        self.data['127.0.0.1'] = dict([('image', 'ubuntu-maverick-amd64'), ('allocated', datetime.datetime.now()), ('activity', datetime.datetime.now()), ('counts', dict()), ('errors', 0)])

    def build_vars(self, address, path):
        if address not in self.data:
            raise AttributeResolutionError
        data = dict([('address', address), ('path', path), ('count', self.data[address]['counts'].get(path, 0))])
        data.update(self.data[address])
        return data

    def render_get_static(self, address, path):
            try:
                fname = self.static + '/' + path
                os.stat(fname)
                if path not in self.data[address]['counts']:
                    self.data[address]['counts'][path] = 0
                self.data[address]['counts'][path] += 1
                return open(fname).read()
            except:
                raise RenderError

    def render_get_dynamic(self, address, path):
        fname = self.dynamic + '/' + path
        try:
            os.stat(fname)
        except:
            raise LookupError
        try:
            bvars = self.build_vars(address, path)
        except AttributeResolutionError:
            raise RenderError

        # grab the requested template
        with open(fname) as infile:
            tmpl = TextTemplate(infile.read())

        # increment access count
        if path not in self.data[address]['counts']:
            self.data[address]['counts'][path] = 0
        self.data[address]['counts'][path] += 1
        # sick genshi on that template and return it
        try:
            return tmpl.generate(**bvars).render('text')
        except:
            logging.exception("Genshi template error")
            raise RenderError

    def __call__(self, environ, start_response):
        address = environ['REMOTE_ADDR']
        path = environ['PATH_INFO'][1:]
        if environ['REQUEST_METHOD'] == 'GET':
            if path == 'status':
                start_response('200 OK', [('Content-type', 'application/json')])
                return json.dumps(self.data, default=dthandler)
            try:
                if path.startswith('static/'):
                    data = self.render_get_static(address, path[path.find('/'):])
                    start_response('200 OK', [('Content-type', 'application/binary')])
                    return data
                elif path.startswith('dynamic'):
                    data = self.render_get_dynamic(address, path[path.find('/'):])
                    start_response('200 OK', [('Content-type', 'application/binary')])
                    return data
                else:
                    raise LookupError
            except LookupError:
                start_response('404 Not Found', [('Content-Type', 'text/plain')])
                return ['Not Found\r\n']
            except RenderError:
                start_response('500 Server Error', [('Content-Type', 'text/plain')])
                return ''
            except:
                logging.exception("Get failure")
        elif environ['REQUEST_METHOD'] == 'POST':
            data = environ['wsgi.input'].read()
            if path == 'info':
                logging.info(address + " INFO: " +  data)
                self.data[address]['activity'] = datetime.datetime.now()
            elif path == 'error':
                logging.error(address + " ERROR: " + data)
                self.data[address]['activity'] = datetime.datetime.now()
                self.data[address]['errors'] += 1
            else:
                start_response('404 Not Found', [('Content-Type', 'text/plain')])
                return ''
            start_response('200 OK', [('Content-type', 'application/octet-stream')])
            return ""
        start_response('404 Not Found', [('Content-Type', 'text/plain')])
        return ['Not Found\r\n']

if __name__ == '__main__':
    from eventlet import wsgi
    wsgi.server(eventlet.listen(('localhost', 8080)), fm(root='/Users/desai/tmp/flunky'))

<commit_msg>flunkymaster: add ctl infrastructure<commit_after>"""A simple web server that accepts POSTS containing a list of feed urls,
and returns the titles of those feeds.
"""
import datetime
import eventlet
import json
import os
import logging
from genshi.template import TextTemplate

logging.basicConfig(level=logging.DEBUG)

# the pool provides a safety limit on our concurrency
pool = eventlet.GreenPool()

class PageLookupError(Exception):
    pass

class RenderError(Exception):
    pass

class AttributeResolutionError(Exception):
    pass

def dthandler(obj):
    if isinstance(obj, datetime.datetime):
        return obj.isoformat()
    return obj

class fm(object):
    def __init__(self, root):
        self.root = root
        self.static = root +'/static'
        self.dynamic = root +'/dynamic'
        self.data = dict()
        logging.info("Starting")
        self.assert_setup('127.0.0.1', {'image':'ubuntu-maverick-amd64'})

    def assert_setup(self, address, info):
        newsetup = dict([('allocated', datetime.datetime.now()), ('counts', dict()), ('errors', 0),
                         ('activity', datetime.datetime.now())])
        newsetup['image'] = info['image']
        if 'extradata' in info:
            newsetup['extradata'] = info['extradata']
        self.data[address] = newsetup

    def build_vars(self, address, path):
        if address not in self.data:
            raise AttributeResolutionError
        data = dict([('address', address), ('path', path),
                     ('count', self.data[address]['counts'].get(path, 0))])
        data.update(self.data[address])
        return data

    def render_get_static(self, address, path):
        try:
            fname = self.static + '/' + path
            os.stat(fname)
            if path not in self.data[address]['counts']:
                self.data[address]['counts'][path] = 0
            self.data[address]['counts'][path] += 1
            return open(fname).read()
        except:
            raise RenderError

    def render_get_dynamic(self, address, path):
        fname = self.dynamic + '/' + path
        try:
            os.stat(fname)
        except:
            raise PageLookupError
        try:
            bvars = self.build_vars(address, path)
        except AttributeResolutionError:
            raise RenderError

        # grab the requested template
        with open(fname) as infile:
            tmpl = TextTemplate(infile.read())

        # increment access count
        if path not in self.data[address]['counts']:
            self.data[address]['counts'][path] = 0
        self.data[address]['counts'][path] += 1
        # sick genshi on that template and return it
        try:
            return tmpl.generate(**bvars).render('text')
        except:
            logging.exception("Genshi template error")
            raise RenderError

    def __call__(self, environ, start_response):
        address = environ['REMOTE_ADDR']
        path = environ['PATH_INFO'][1:]
        if environ['REQUEST_METHOD'] == 'GET':
            if path == 'status':
                start_response('200 OK', [('Content-type', 'application/json')])
                return json.dumps(self.data, default=dthandler)
            try:
                if path.startswith('static/'):
                    data = self.render_get_static(address, path[path.find('/'):])
                    start_response('200 OK', [('Content-type', 'application/binary')])
                    return data
                elif path.startswith('dynamic'):
                    data = self.render_get_dynamic(address, path[path.find('/'):])
                    start_response('200 OK', [('Content-type', 'application/binary')])
                    return data
                else:
                    raise PageLookupError
            except PageLookupError:
                start_response('404 Not Found', [('Content-Type', 'text/plain')])
                return ['Not Found\r\n']
            except RenderError:
                start_response('500 Server Error', [('Content-Type', 'text/plain')])
                return ''
            except:
                logging.exception("Get failure")
        elif environ['REQUEST_METHOD'] == 'POST':
            data = environ['wsgi.input'].read()
            if path == 'info':
                logging.info(address + " INFO: " +  data)
                self.data[address]['activity'] = datetime.datetime.now()
            elif path == 'error':
                logging.error(address + " ERROR: " + data)
                self.data[address]['activity'] = datetime.datetime.now()
                self.data[address]['errors'] += 1
            else:
                start_response('404 Not Found', [('Content-Type', 'text/plain')])
                return ''
            start_response('200 OK', [('Content-type', 'application/octet-stream')])
            return ""
        start_response('404 Not Found', [('Content-Type', 'text/plain')])
        return ['Not Found\r\n']

if __name__ == '__main__':
    from eventlet import wsgi
    wsgi.server(eventlet.listen(('localhost', 8080)), fm(root='/Users/desai/tmp/flunky'))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This module contains classes to work with custom OpenFOAM classes, e.g., lists
and dictionaries.
"""

from __future__ import division, print_function
import os
import foampy

class FoamDict(dict):
    """Object that represents an OpenFOAM dictionary."""

    def __init__(self, name="", casedir="./", subdir="system"):
        self.name = name
        self.subdir = subdir
        self.casedir = casedir
        self.fpath = os.path.join(casedir, subdir, name)
        if os.path.isfile(self.fpath):
            self.read()
        else:
            self.header = foampy.dictionaries.build_header(name,
                    foampy.foam_version, fileclass="dictionary")

    def read(self):
        """Parse dictionary."""
        pass

    def __str__(self):
        """Create text from dictionary in OpenFOAM format."""
        return "test"

class BlockMeshDict(FoamDict):
    """Object to represent a `blockMeshDict`."""

    def __init__(self, name="blockMeshDict", casedir="./",
                 subdir="constant/polyMesh"):
        FoamDict.__init__(self)

class FoamList(list):
    """Class that represents an OpenFOAM list."""

    def __init__(self, py_list=[], dtype=float):
        py_list = [dtype(i) for i in py_list]
        list.__init__(self, py_list)

    def __str__(self):
        txt = "("
        for i in self:
            txt += str(i) + " "
        txt = txt[:-1]
        txt += ")"
        return txt

def test_foamdict():
    """Test `FoamDict` class."""
    d = FoamDict(name="controlDict", casedir="test")
    print(d)

def test_foamlist():
    """Test `FoamList` class."""
    flist = FoamList([1, 2, 3, 4])
    print(flist)

    for i in flist:
        print(i)

    flist2 = FoamList([flist, flist], dtype=FoamList)
    print(flist2)<commit_msg>Add header parsing to FoamDict<commit_after># -*- coding: utf-8 -*-
"""
This module contains classes to work with custom OpenFOAM classes, e.g., lists
and dictionaries.
"""

from __future__ import division, print_function
import os
import foampy

class FoamDict(dict):
    """Object that represents an OpenFOAM dictionary."""

    def __init__(self, name="", casedir="./", subdir="system"):
        self.name = name
        self.subdir = subdir
        self.casedir = casedir
        self.fpath = os.path.join(casedir, subdir, name)
        self.foam_version = foampy.foam_version
        self.header = ""
        if os.path.isfile(self.fpath):
            self.read()
        else:
            self.header = foampy.dictionaries.build_header(name,
                    self.foam_version, fileclass="dictionary")

    def read(self):
        """Parse dictionary."""
        # Read header first
        with open(self.fpath) as f:
            in_header = False
            for line in f:
                if line[:2] == "/*":
                    in_header = True
                elif line[:2] == r"\*":
                    self.header += line
                    in_header = False
                if in_header:
                    self.header += line
        # Parse header for OpenFOAM version
        splitheader = self.header.split()
        if "Version:" in splitheader:
            index = splitheader.index("Version:") + 1
            self.foam_version = splitheader[index]

    def __str__(self):
        """Create text from dictionary in OpenFOAM format."""
        return self.header

class BlockMeshDict(FoamDict):
    """Object to represent a `blockMeshDict`."""

    def __init__(self, name="blockMeshDict", casedir="./",
                 subdir="constant/polyMesh"):
        FoamDict.__init__(self)

class FoamList(list):
    """Class that represents an OpenFOAM list."""

    def __init__(self, py_list=[], dtype=float):
        py_list = [dtype(i) for i in py_list]
        list.__init__(self, py_list)

    def __str__(self):
        txt = "("
        for i in self:
            txt += str(i) + " "
        txt = txt[:-1]
        txt += ")"
        return txt

def test_foamdict():
    """Test `FoamDict` class."""
    d = FoamDict(name="controlDict", casedir="test")
    print(d)
    print(d.foam_version)

def test_foamlist():
    """Test `FoamList` class."""
    flist = FoamList([1, 2, 3, 4])
    print(flist)

    for i in flist:
        print(i)

    flist2 = FoamList([flist, flist], dtype=FoamList)
    print(flist2)<|endoftext|>