<commit_before>from hashlib import md5
from django.conf import settings
from django.db import models
from django.core import paginator
from djangae.contrib.pagination.decorators import _field_name_for_ordering
from django.core.cache import cache

# TODO: it would be nice to be able to define a function which is given the queryset and returns
# the cache time.  That would allow different cache times for different queries.
CACHE_TIME = getattr(settings, "DJANGAE_PAGINATION_CACHE_TIME", 30*60)

class PaginationOrderingRequired(RuntimeError):
    pass

def _marker_cache_key(model, query_id, page_number):
    cache_key = "_PAGE_MARKER_{}:{}:{}".format(model._meta.db_table, query_id, page_number)
    return cache_key

def _count_cache_key(query_id):
    cache_key = "_PAGE_COUNTER_{}".format(query_id)
    return cache_key

def _update_known_count(query_id, count):
    cache_key = _count_cache_key(query_id)

    ret = cache.get(cache_key)
    if ret and ret > count:
        return

    cache.set(cache_key, count, CACHE_TIME)

def _get_known_count(query_id):
    cache_key = _count_cache_key(query_id)
    ret = cache.get(cache_key)
    if ret:
        return ret
    return 0

def _store_marker(model, query_id, page_number, marker_value):
    """
        For a model and query id, stores the marker value for previously
        queried page number.

        This stores the last item on the page identified by page number,
        not the marker that starts the page. i.e. there is a marker for page 1
    """

    cache_key = _marker_cache_key(model, query_id, page_number)
    cache.set(cache_key, marker_value, CACHE_TIME)

def _get_marker(model, query_id, page_number):
    """
        For a model and query_id, returns the marker at the end of the
        previous page. Returns a tuple of (marker, pages) where pages is
        the number of pages we had to go back to find the marker (this is the
        number of pages we need to skip in the result set)
    """

    counter = page_number - 1
    pages_skipped = 0

    while counter > 0   :
        cache_key = _marker_cache_key(model, query_id, counter)
        ret = cache.get(cache_key)

        if ret:
            return ret, pages_skipped

        counter -= 1
        pages_skipped += 1

    # If we get here then we couldn't find a stored marker anywhere
    return None, pages_skipped

def queryset_identifier(queryset):
    """ Returns a string that uniquely identifies this query excluding its low and high mark"""

    hasher = md5()
    hasher.update(str(queryset.query.where))
    hasher.update(str(queryset.query.order_by))
    return hasher.hexdigest()

class Paginator(paginator.Paginator):
    """
        A paginator that works with the @paginated_model class decorator to efficiently
        return paginated sets on the appengine datastore
    """

    def __init__(self, object_list, per_page, readahead=10, **kwargs):
        if not object_list.ordered:
            object_list.order_by("pk") # Just order by PK by default

        self.original_orderings = object_list.query.order_by
        self.field_required = _field_name_for_ordering(self.original_orderings[:])
        self.readahead = readahead

        try:
            object_list.model._meta.get_field(self.field_required)
        except models.FieldDoesNotExist:
            raise PaginationOrderingRequired("No pagination ordering specified for {}".format(self.original_orderings))

        # Wipe out the existing ordering
        object_list = object_list.order_by()

        # Add our replacement ordering
        if len(self.original_orderings) == 1 and self.original_orderings[0].startswith("-"):
            object_list = object_list.order_by("-" + self.field_required)
        else:
            object_list = object_list.order_by(self.field_required)

        super(Paginator, self).__init__(object_list, per_page, **kwargs)

    @property
    def count(self):
        return _get_known_count(queryset_identifier(self.object_list))

    def validate_number(self, number):
        """
        Validates the given 1-based page number.
        """
        try:
            number = int(number)
        except (TypeError, ValueError):
            raise paginator.PageNotAnInteger('That page number is not an integer')
        if number < 1:
            raise paginator.EmptyPage('That page number is less than 1')

        return number

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        number = self.validate_number(number)
        bottom = (number - 1) * self.per_page
        top = bottom + self.per_page
        queryset_id = queryset_identifier(self.object_list)

        marker_value, pages = _get_marker(
            self.object_list.model,
            queryset_id,
            number
        )

        if marker_value:
            if len(self.original_orderings) == 1 and self.original_orderings[0].startswith("-"):
                qs = self.object_list.all().filter(**{"{}__lt".format(self.field_required): marker_value})
            else:
                qs = self.object_list.all().filter(**{"{}__gt".format(self.field_required): marker_value})
            bottom = pages * self.per_page # We have to skip the pages here
            top = bottom + self.per_page
        else:
            qs = self.object_list

        results = list(qs[bottom:top + (self.per_page * self.readahead)])

        next_page = results[top:]
        next_page_counter = number + 1
        while next_page:
            _store_marker(
                self.object_list.model,
                queryset_id,
                next_page_counter,
                getattr(next_page[self.per_page-1], self.field_required)
            )
            next_page_counter += 1
            next_page = next_page[self.per_page:]

        if not results:
            raise paginator.EmptyPage("That page contains no results")

        known_count = ((number - 1) * self.per_page) + len(results)
        _update_known_count(queryset_id, known_count)

        page = self._get_page(results[:top], number, self)

        _store_marker(
            self.object_list.model,
            queryset_id,
            number,
            getattr(page.object_list[self.per_page-1], self.field_required)
        )

        return page
<commit_msg>Adding a comment about denormalisation of single negated orderings.<commit_after>from hashlib import md5
from django.conf import settings
from django.db import models
from django.core import paginator
from djangae.contrib.pagination.decorators import _field_name_for_ordering
from django.core.cache import cache

# TODO: it would be nice to be able to define a function which is given the queryset and returns
# the cache time.  That would allow different cache times for different queries.
CACHE_TIME = getattr(settings, "DJANGAE_PAGINATION_CACHE_TIME", 30*60)

class PaginationOrderingRequired(RuntimeError):
    pass

def _marker_cache_key(model, query_id, page_number):
    cache_key = "_PAGE_MARKER_{}:{}:{}".format(model._meta.db_table, query_id, page_number)
    return cache_key

def _count_cache_key(query_id):
    cache_key = "_PAGE_COUNTER_{}".format(query_id)
    return cache_key

def _update_known_count(query_id, count):
    cache_key = _count_cache_key(query_id)

    ret = cache.get(cache_key)
    if ret and ret > count:
        return

    cache.set(cache_key, count, CACHE_TIME)

def _get_known_count(query_id):
    cache_key = _count_cache_key(query_id)
    ret = cache.get(cache_key)
    if ret:
        return ret
    return 0

def _store_marker(model, query_id, page_number, marker_value):
    """
        For a model and query id, stores the marker value for previously
        queried page number.

        This stores the last item on the page identified by page number,
        not the marker that starts the page. i.e. there is a marker for page 1
    """

    cache_key = _marker_cache_key(model, query_id, page_number)
    cache.set(cache_key, marker_value, CACHE_TIME)

def _get_marker(model, query_id, page_number):
    """
        For a model and query_id, returns the marker at the end of the
        previous page. Returns a tuple of (marker, pages) where pages is
        the number of pages we had to go back to find the marker (this is the
        number of pages we need to skip in the result set)
    """

    counter = page_number - 1
    pages_skipped = 0

    while counter > 0   :
        cache_key = _marker_cache_key(model, query_id, counter)
        ret = cache.get(cache_key)

        if ret:
            return ret, pages_skipped

        counter -= 1
        pages_skipped += 1

    # If we get here then we couldn't find a stored marker anywhere
    return None, pages_skipped

def queryset_identifier(queryset):
    """ Returns a string that uniquely identifies this query excluding its low and high mark"""

    hasher = md5()
    hasher.update(str(queryset.query.where))
    hasher.update(str(queryset.query.order_by))
    return hasher.hexdigest()

class Paginator(paginator.Paginator):
    """
        A paginator that works with the @paginated_model class decorator to efficiently
        return paginated sets on the appengine datastore
    """

    def __init__(self, object_list, per_page, readahead=10, **kwargs):
        if not object_list.ordered:
            object_list.order_by("pk") # Just order by PK by default

        self.original_orderings = object_list.query.order_by
        self.field_required = _field_name_for_ordering(self.original_orderings[:])
        self.readahead = readahead

        try:
            object_list.model._meta.get_field(self.field_required)
        except models.FieldDoesNotExist:
            raise PaginationOrderingRequired("No pagination ordering specified for {}".format(self.original_orderings))

        # Wipe out the existing ordering
        object_list = object_list.order_by()

        # Add our replacement ordering
        # A single negated ordering can use the same field (we just flip the query), this
        # normalisation happens in _field_name_for_ordering, so we do the same here.
        if len(self.original_orderings) == 1 and self.original_orderings[0].startswith("-"):
            object_list = object_list.order_by("-" + self.field_required)
        else:
            object_list = object_list.order_by(self.field_required)

        super(Paginator, self).__init__(object_list, per_page, **kwargs)

    @property
    def count(self):
        return _get_known_count(queryset_identifier(self.object_list))

    def validate_number(self, number):
        """
        Validates the given 1-based page number.
        """
        try:
            number = int(number)
        except (TypeError, ValueError):
            raise paginator.PageNotAnInteger('That page number is not an integer')
        if number < 1:
            raise paginator.EmptyPage('That page number is less than 1')

        return number

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        number = self.validate_number(number)
        bottom = (number - 1) * self.per_page
        top = bottom + self.per_page
        queryset_id = queryset_identifier(self.object_list)

        marker_value, pages = _get_marker(
            self.object_list.model,
            queryset_id,
            number
        )

        if marker_value:
            if len(self.original_orderings) == 1 and self.original_orderings[0].startswith("-"):
                qs = self.object_list.all().filter(**{"{}__lt".format(self.field_required): marker_value})
            else:
                qs = self.object_list.all().filter(**{"{}__gt".format(self.field_required): marker_value})
            bottom = pages * self.per_page # We have to skip the pages here
            top = bottom + self.per_page
        else:
            qs = self.object_list

        results = list(qs[bottom:top + (self.per_page * self.readahead)])

        next_page = results[top:]
        next_page_counter = number + 1
        while next_page:
            _store_marker(
                self.object_list.model,
                queryset_id,
                next_page_counter,
                getattr(next_page[self.per_page-1], self.field_required)
            )
            next_page_counter += 1
            next_page = next_page[self.per_page:]

        if not results:
            raise paginator.EmptyPage("That page contains no results")

        known_count = ((number - 1) * self.per_page) + len(results)
        _update_known_count(queryset_id, known_count)

        page = self._get_page(results[:top], number, self)

        _store_marker(
            self.object_list.model,
            queryset_id,
            number,
            getattr(page.object_list[self.per_page-1], self.field_required)
        )

        return page
<|endoftext|>


<commit_before>"""Useful defualt settings that you might want to use in all your projects."""
import re

IGNORABLE_404_URLS = [
    re.compile(r'\..*$'),
    re.compile(r'^/media/'),
    re.compile(r'^/static/'),
]

IGNORABLE_404_USER_AGENTS = [
    re.compile(r'AhrefsBot', re.I),
    re.compile(r'Baiduspider', re.I),
    re.compile(r'bingbot', re.I),
    re.compile(r'BLEXBot', re.I),
    re.compile(r'coccoc', re.I),
    re.compile(r'DotBot', re.I),
    re.compile(r'EasouSpider', re.I),
    re.compile(r'Exabot', re.I),
    re.compile(r'FacebookBot', re.I),
    re.compile(r'Feedfetcher-Google', re.I),
    re.compile(r'Googlebot', re.I),
    re.compile(r'Jorgee', re.I),
    re.compile(r'Mail.RU_Bot', re.I),
    re.compile(r'mindUpBot', re.I),
    re.compile(r'MJ12bot', re.I),
    re.compile(r'msnbot', re.I),
    re.compile(r'publiclibraryarchive.org', re.I),
    re.compile(r'savetheworldheritage.org', re.I),
    re.compile(r'seoscanners', re.I),
    re.compile(r'Test Certificate Info', re.I),
    re.compile(r'Twitterbot', re.I),
    re.compile(r'WinHttp.WinHttpRequest.5', re.I),
    re.compile(r'XoviBot', re.I),
    re.compile(r'Yahoo! Slurp', re.I),
    re.compile(r'YandexBot', re.I),
]
<commit_msg>Update django_settings.py<commit_after>"""Useful defualt settings that you might want to use in all your projects."""
import re

IGNORABLE_404_URLS = [
    re.compile(r'\..*$'),
    re.compile(r'^/media/'),
    re.compile(r'^/static/'),
]

IGNORABLE_404_USER_AGENTS = [
    re.compile(r'AhrefsBot', re.I),
    re.compile(r'BacklinkCrawler', re.I),
    re.compile(r'Baiduspider', re.I),
    re.compile(r'bingbot', re.I),
    re.compile(r'BLEXBot', re.I),
    re.compile(r'coccoc', re.I),
    re.compile(r'DotBot', re.I),
    re.compile(r'EasouSpider', re.I),
    re.compile(r'Exabot', re.I),
    re.compile(r'FacebookBot', re.I),
    re.compile(r'Feedfetcher-Google', re.I),
    re.compile(r'Googlebot', re.I),
    re.compile(r'Jorgee', re.I),
    re.compile(r'Mail.RU_Bot', re.I),
    re.compile(r'mindUpBot', re.I),
    re.compile(r'MJ12bot', re.I),
    re.compile(r'msnbot', re.I),
    re.compile(r'publiclibraryarchive.org', re.I),
    re.compile(r'savetheworldheritage.org', re.I),
    re.compile(r'seoscanners', re.I),
    re.compile(r'Test Certificate Info', re.I),
    re.compile(r'Twitterbot', re.I),
    re.compile(r'WinHttp.WinHttpRequest.5', re.I),
    re.compile(r'XoviBot', re.I),
    re.compile(r'Yahoo! Slurp', re.I),
    re.compile(r'YandexBot', re.I),
]
<|endoftext|>


<commit_before>

###############################################################
###   A decorator that signals that a function is callable
###   by DMP.  This prevents "just any function" from being
###   called by the router above.  The function must be
###   decorated to be callable:
###
###       @view_function
###       function process_request(request):
###           ...

class view_function(object):
  '''A decorator to signify which view functions are "callable" by web browsers.
     I'm using a class (rather than the more common inner function) so the above
     router_request can call isinstance().
  '''
  def __init__(self, func):
    '''Constructor.  This is called by adding the @view_function decorator to any function'''
    self.func = func

  def __call__(self, *args, **kwargs):
    '''Called when the user calls this "function".'''
    return self.func(*args, **kwargs)

###############################################################
###   Exceptions used to direct the controller

class InternalRedirectException(Exception):
  '''View functions can throw this exception to indicate that a new view
     should be called by the HtmlPageServer.  The current view function
     will end immediately, and processing will be passed to the new view function.
  '''
  def __init__(self, redirect_module, redirect_function):
    '''Indicates the new view to be called.  The view should be given relative to the project root.'''
    super().__init__()
    self.redirect_module = redirect_module
    self.redirect_function = redirect_function

class RedirectException(Exception):
  '''Immediately stops processing of a view function or template and redirects to the given page.
     Perhaps it takes a little too much liberty with exceptions, but it makes returning from a
     huge call stack really nice.

     If as_javascript==True, the browser is sent <script>window.location.href="...";</script>.
     This is useful when using Ajax.  A redirect in Ajax is handled internally by libraries like
     JQuery, so a regular HTTP redirect can't direct the top-level page.  Javascript is a hack
     around this so an Ajax call can redirect the whole browser window.
  '''
  def __init__(self, redirect_to, permanent=False, as_javascript=False):
    self.redirect_to = redirect_to
    self.permanent = permanent
    self.as_javascript = as_javascript

  def get_response(self):
    if self.as_javascript:
      return HttpResponse('<script>window.location.href="%s";</script>' % self.redirect_to)
    if self.permanent:
      return HttpResponsePermanentRedirect(self.redirect_to)
    return HttpResponseRedirect(self.redirect_to)

<commit_msg>small fix 2<commit_after>from django.http import HttpResponse, Http404, HttpResponseRedirect, HttpResponsePermanentRedirect

###############################################################
###   A decorator that signals that a function is callable
###   by DMP.  This prevents "just any function" from being
###   called by the router above.  The function must be
###   decorated to be callable:
###
###       @view_function
###       function process_request(request):
###           ...

class view_function(object):
  '''A decorator to signify which view functions are "callable" by web browsers.
     I'm using a class (rather than the more common inner function) so the above
     router_request can call isinstance().
  '''
  def __init__(self, func):
    '''Constructor.  This is called by adding the @view_function decorator to any function'''
    self.func = func

  def __call__(self, *args, **kwargs):
    '''Called when the user calls this "function".'''
    return self.func(*args, **kwargs)

###############################################################
###   Exceptions used to direct the controller

class InternalRedirectException(Exception):
  '''View functions can throw this exception to indicate that a new view
     should be called by the HtmlPageServer.  The current view function
     will end immediately, and processing will be passed to the new view function.
  '''
  def __init__(self, redirect_module, redirect_function):
    '''Indicates the new view to be called.  The view should be given relative to the project root.'''
    super().__init__()
    self.redirect_module = redirect_module
    self.redirect_function = redirect_function

class RedirectException(Exception):
  '''Immediately stops processing of a view function or template and redirects to the given page.
     Perhaps it takes a little too much liberty with exceptions, but it makes returning from a
     huge call stack really nice.

     If as_javascript==True, the browser is sent <script>window.location.href="...";</script>.
     This is useful when using Ajax.  A redirect in Ajax is handled internally by libraries like
     JQuery, so a regular HTTP redirect can't direct the top-level page.  Javascript is a hack
     around this so an Ajax call can redirect the whole browser window.
  '''
  def __init__(self, redirect_to, permanent=False, as_javascript=False):
    self.redirect_to = redirect_to
    self.permanent = permanent
    self.as_javascript = as_javascript

  def get_response(self):
    if self.as_javascript:
      return HttpResponse('<script>window.location.href="%s";</script>' % self.redirect_to)
    if self.permanent:
      return HttpResponsePermanentRedirect(self.redirect_to)
    return HttpResponseRedirect(self.redirect_to)

<|endoftext|>


<commit_before># A file that defines the custom data structs used in our assignment
from helper import *
from collections import deque

variables = []
facts_raw = []
rules = []

operators = ['^', 'v']

class Variable(object):

    """ Class that represents the basic state of a variable """

    def __init__(self, name="", str_val="", bool_val=False):
        """ object state includes string value and truth value """
        self.name = name
        self.string_value = str_val
        self.truth_value = bool_val

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
            and self.name == other.name)

    def __str__(self):
        """ so str() operator can be used """
        return '{} = "{}"'.format(self.name, self.string_value)

class Rule(object):

    def __init__(self, exp, var):
        self.expression = exp  # Expression Object
        self.variable = var    # Variable object

    def __str__(self):
        return self.expression + " -> " + variable.name

    def evaluate(self):
        pass

    def validate(self):
        """ validates that all variables """

class Expression(object):

    def __init__(self, string):
        self.expr_str = string
        self.token_list = self.tokenize()

    def tokenize(self):
        temp_string = "".join(self.expr_str.split())
        l = []
        for char in temp_string:
            l.append(char)
        return l

class TreeNode(object):
    """ Class used to make abstract syntax tree for Expressions"""

    def __init__(self, value="", left=None, right=None, neg=False):
        self.value = value
        self.left = left
        self.right = right
        self.negate = neg

    def __str__(self):
        if not self.negate:
            return self.value
        else:
            return "Not " + self.value

def get_RPN(token_list):
    """
    Builds an AST using shunting-yard algo
    Returns None if malformed input
    Returns a queue in reverse polish notation if successful
    """
    # initialize data structures : SY is stack based
    stack = list()
    queue = deque([])
    for token in token_list:  # assumes token is a char
        if is_var(variables, token):
            #var = find_var(variables, token)
            #queue.append(var)
            queue.append(token)
        elif is_valid_op(token):
            if token =='!':
                queue.append(token)
            elif token in operators:
                while len(stack) > 0 and stack[len(stack) -1] in operators:
                    op = stack.pop()
                    queue.append(op)
                stack.append(token)
            elif token == '(':
                stack.append(token)
            elif token == ')':
                paren_matched = False
                while len(stack) > 0:
                    item = stack.pop()
                    if item == '(':
                        paren_matched = True
                        break
                    else:
                        queue.append(item)
                if not paren_matched:
                    #print 'parenthesis mismatch'
                    return None
    while (len(stack) > 0):
        print stack
        top = stack.pop()
        if top == '(' or top == ')':
            #print 'mismatched parenthesis'
            return None
        else:
            queue.append(top)
    return queue

def build_tree(queue):
    """
    Uses queue built in get_RPN method to build a tree ...
    Head node is returned in success
    None is returned in failure
    """

    stack = list()
    while len(queue) > 0:
        item = queue.popleft();
        if is_var(variables, item):
            stack.append(TreeNode(value=item))
        elif item == '!':
            if len(queue) > 0:
                item2 = queue.popleft()
            else:
                return None
            if is_var(variables, item2):
                stack.append(TreeNode(value=item2, neg=True))
            else:
                return None
        elif item in operators:
            right=stack.pop()
            left= stack.pop()
            stack.append(TreeNode(value=item, right=right, left=left, neg=False))
    return stack.pop()

## WARNING untested
def calc_tree(node):
    # if leaf, it is a Variable
    if node.right is None and node.left is None:
        if node.negate:
            return not node.value.truth_value
        else:
            return node.value.truth_value
    elif node.value = '^':
        return calc_tree(node.right) and calc_tree(node.left)
    elif node.value = 'v':
        return calc_tree(node.right) or calc_tree(node.left)

# def get_token(token_list, expected):
#     if token_list[0] == expected:
#         del token_list[0]
#         return True
#     else:
#         return False

# def get_variable(token_list):
#     x = token_list[0]
#     x_var = find_var(variables, x)
#     if type(x_var) != Variable:
#         return None
#     del token_list[0]
#     return TreeNode(x_var, None, None)

def print_inorder(node):
    if node is not None:
        print_inorder(node.left)
        print node
        print_inorder(node.right)

<commit_msg>bug fixes in build tree & moving from chars to Variable objects<commit_after># A file that defines the custom data structs used in our assignment
from helper import *
from collections import deque

variables = []
facts_raw = []
rules = []

operators = ['^', 'v']

class Variable(object):

    """ Class that represents the basic state of a variable """

    def __init__(self, name="", str_val="", bool_val=False):
        """ object state includes string value and truth value """
        self.name = name
        self.string_value = str_val
        self.truth_value = bool_val

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
            and self.name == other.name)

    def __str__(self):
        """ so str() operator can be used """
        return '{} = "{}"'.format(self.name, self.string_value)

class Rule(object):

    def __init__(self, exp, var):
        self.expression = exp  # Expression Object
        self.variable = var    # Variable object

    def __str__(self):
        return self.expression + " -> " + variable.name

    def evaluate(self):
        pass

    def validate(self):
        """ validates that all variables """

class Expression(object):

    def __init__(self, string):
        self.expr_str = string
        self.token_list = self.tokenize()

    def tokenize(self):
        temp_string = "".join(self.expr_str.split())
        l = []
        for char in temp_string:
            l.append(char)
        return l

class TreeNode(object):
    """ Class used to make abstract syntax tree for Expressions"""

    def __init__(self, value="", left=None, right=None, neg=False):
        self.value = value
        self.left = left
        self.right = right
        self.negate = neg

    def __str__(self):
        if not self.negate:
            return self.value
        else:
            return "Not " + self.value

def get_RPN(token_list):
    """
    uses shunting-yard algo to get infix into RPN form
    Returns None if malformed input
    Returns a queue of Variables in reverse polish notation if successful
    """
    # initialize data structures : SY is stack based
    stack = list()
    queue = deque([])
    for token in token_list:  # assumes token is a char
        if is_var(variables, token):
            var = find_var(variables, token)
            queue.append(var)
        elif is_valid_op(token):
            if token =='!':
                queue.append(token)
            elif token in operators:
                while len(stack) > 0 and stack[len(stack) -1] in operators:
                    op = stack.pop()
                    queue.append(op)
                stack.append(token)
            elif token == '(':
                stack.append(token)
            elif token == ')':
                paren_matched = False
                while len(stack) > 0:
                    item = stack.pop()
                    if item == '(':
                        paren_matched = True
                        break
                    else:
                        queue.append(item)
                if not paren_matched:
                    #print 'parenthesis mismatch'
                    return None
    while (len(stack) > 0):
        print stack
        top = stack.pop()
        if top == '(' or top == ')':
            #print 'mismatched parenthesis'
            return None
        else:
            queue.append(top)
    return queue

def build_tree(queue):
    """
    Uses queue built in get_RPN method to build a tree ...
    Head node is returned in success
    None is returned in failure
    """

    stack = list()
    while len(queue) > 0:
        item = queue.popleft();
        if type(item) is Variable:
            stack.append(TreeNode(value=item))
        elif item == '!':
            if len(queue) > 0:
                item2 = queue.popleft()
            else:
                return None
            stack.append(TreeNode(value=item2, neg=True))
        elif item in operators:
            right=stack.pop()
            left= stack.pop()
            stack.append(TreeNode(value=item, right=right, left=left, neg=False))
    return stack.pop()

## WARNING untested
def calc_tree(node):
    # if leaf, it is a Variable
    if node.right is None and node.left is None:
        if node.negate:
            return not node.value.truth_value
        else:
            return node.value.truth_value
    elif node.value == '^':
        return calc_tree(node.right) and calc_tree(node.left)
    elif node.value =='v':
        return calc_tree(node.right) or calc_tree(node.left)

# def get_token(token_list, expected):
#     if token_list[0] == expected:
#         del token_list[0]
#         return True
#     else:
#         return False

# def get_variable(token_list):
#     x = token_list[0]
#     x_var = find_var(variables, x)
#     if type(x_var) != Variable:
#         return None
#     del token_list[0]
#     return TreeNode(x_var, None, None)

def print_inorder(node):
    if node is not None:
        print_inorder(node.left)
        print node
        print_inorder(node.right)

<|endoftext|>


<commit_before># coding=utf-8

import os
import sys
import time
import logging
import traceback
import configobj
import inspect

# Path Fix
sys.path.append(
    os.path.abspath(
        os.path.join(
            os.path.dirname(__file__), "../")))

import diamond

from diamond.collector import Collector
from diamond.handler.Handler import Handler
from diamond.scheduler import ThreadedScheduler
from diamond.util import load_class_from_name

class Server(object):
    """
    Server class loads and starts Handlers and Collectors
    """

    def __init__(self, config):
        # Initialize Logging
        self.log = logging.getLogger('diamond')
        # Initialize Members
        self.config = config
        self.running = False
        self.handlers = []
        self.modules = {}
        self.tasks = {}
        # Initialize Scheduler
        self.scheduler = ThreadedScheduler()

    def load_config(self):
        """
        Load the full config
        """

        configfile = os.path.abspath(self.config['configfile'])
        config = configobj.ConfigObj(configfile)
        config['configfile'] = self.config['configfile']

        # Merge in handler config files into the main config
        if 'handlers_config_path' in config['server']:
            files = os.listdir(config['server']['handlers_config_path'])
            for filename in files:
                configname = os.path.basename(filename)
                handlername = configname.split('.')[0]
                if handlername not in self.config['handlers']:
                    config['handlers'][handlername] = configobj.ConfigObj()

                configfile = os.path.join(
                    config['server']['handlers_config_path'],
                    configname)
                hconfig = configobj.ConfigObj(configfile)
                if handlername in config['handlers']:
                    config['handlers'][handlername].merge(hconfig)
                else:
                    config['handlers'][handlername] = hconfig

        self.config = config

    def load_handler(self, fqcn):
        """
        Load Handler class named fqcn
        """
        # Load class
        cls = load_class_from_name(fqcn)
        # Check if cls is subclass of Handler
        if cls == Handler or not issubclass(cls, Handler):
            raise TypeError("%s is not a valid Handler" % fqcn)
        # Log
        self.log.debug("Loaded Handler: %s", fqcn)
        return cls

    def load_handlers(self):
        """
        Load handlers
        """
        if type(self.config['server']['handlers']) == str:
            handlers = [self.config['server']['handlers']]
            self.config['server']['handlers'] = handlers

        for h in self.config['server']['handlers']:
            try:
                # Load Handler Class
                cls = self.load_handler(h)

                # Initialize Handler config
                handler_config = configobj.ConfigObj()
                # Merge default Handler default config
                handler_config.merge(self.config['handlers']['default'])
                # Check if Handler config exists
                if cls.__name__ in self.config['handlers']:
                    # Merge Handler config section
                    handler_config.merge(self.config['handlers'][cls.__name__])

                # Initialize Handler class
                self.handlers.append(cls(handler_config))

            except ImportError:
                # Log Error
                self.log.debug("Failed to load handler %s. %s", h,
                               traceback.format_exc())
                continue

    def load_collector(self, fqcn):
        """
        Load Collector class named fqcn
        """
        # Load class
        cls = load_class_from_name(fqcn)
        # Check if cls is subclass of Collector
        if cls == Collector or not issubclass(cls, Collector):
            raise TypeError("%s is not a valid Collector" % fqcn)
        # Log
        self.log.debug("Loaded Collector: %s", fqcn)
        return cls

    def load_include_path(self, path):
        """
        Scan for and add paths to the include path
        """
        # Verify the path is valid
        if not os.path.isdir(path):
            return
        # Add path to the system path, to avoid name clashes
        # with mysql-connector for example ...
        sys.path.insert(1, path)
        # Load all the files in path
        for f in os.listdir(path):
            # Are we a directory? If so process down the tree
            fpath = os.path.join(path, f)
            if os.path.isdir(fpath):
                self.load_include_path(fpath)

    def load_collectors(self, path, filter=None):
        """
        Scan for collectors to load from path
        """
        # Initialize return value
        collectors = {}

        # Get a list of files in the directory, if the directory exists
        if not os.path.exists(path):
            raise OSError("Directory does not exist: %s" % path)

        if path.endswith('tests') or path.endswith('fixtures'):
            return collectors

        # Log
        self.log.debug("Loading Collectors from: %s", path)

        # Load all the files in path
        for f in os.listdir(path):

            # Are we a directory? If so process down the tree
            fpath = os.path.join(path, f)
            if os.path.isdir(fpath):
                subcollectors = self.load_collectors(fpath)
                for key in subcollectors:
                    collectors[key] = subcollectors[key]

            # Ignore anything that isn't a .py file
            elif (os.path.isfile(fpath)
                  and len(f) > 3
                  and f[-3:] == '.py'
                  and f[0:4] != 'test'
                  and f[0] != '.'):

                # Check filter
                if filter and os.path.join(path, f) != filter:
                    continue

                modname = f[:-3]

                # Stat module file to get mtime
                st = os.stat(os.path.join(path, f))
                mtime = st.st_mtime
                # Check if module has been loaded before
                if modname in self.modules:
                    # Check if file mtime is newer then the last loaded verison
                    if mtime <= self.modules[modname]:
                        # Module hasn't changed
                        # Log
                        self.log.debug("Found %s, but it hasn't changed.",
                                       modname)
                        continue

                try:
                    # Import the module
                    mod = __import__(modname, globals(), locals(), ['*'])
                except ImportError:
                    # Log error
                    self.log.error("Failed to import module: %s. %s", modname,
                                   traceback.format_exc())
                    continue

                # Update module mtime
                self.modules[modname] = mtime
                # Log
                self.log.debug("Loaded Module: %s", modname)

                # Find all classes defined in the module
                for attrname in dir(mod):
                    attr = getattr(mod, attrname)
                    # Only attempt to load classes that are infact classes
                    # are Collectors but are not the base Collector class
                    if (inspect.isclass(attr)
                            and issubclass(attr, Collector)
                            and attr != Collector):
                        if attrname.startswith('parent_'):
                            continue
                        # Get class name
                        fqcn = '.'.join([modname, attrname])
                        try:
                            # Load Collector class
                            cls = self.load_collector(fqcn)
                            # Add Collector class
                            collectors[cls.__name__] = cls
                        except Exception:
                            # Log error
                            self.log.error("Failed to load Collector: %s. %s",
                                           fqcn, traceback.format_exc())
                            continue

        # Return Collector classes
        return collectors

    def init_collector(self, cls):
        """
        Initialize collector
        """
        collector = None
        try:
            # Initialize Collector
            collector = cls(self.config, self.handlers)
            # Log
            self.log.debug("Initialized Collector: %s", cls.__name__)
        except Exception:
            # Log error
            self.log.error("Failed to initialize Collector: %s. %s",
                           cls.__name__, traceback.format_exc())

        # Return collector
        return collector

    def schedule_collector(self, c, interval_task=True):
        """
        Schedule collector
        """
        # Check collector is for realz
        if c is None:
            self.log.warn("Skipped loading invalid Collector: %s",
                          c.__class__.__name__)
            return

        if c.config['enabled'] != True:
            self.log.warn("Skipped loading disabled Collector: %s",
                          c.__class__.__name__)
            return

        # Get collector schedule
        for name, schedule in c.get_schedule().items():
            # Get scheduler args
            func, args, splay, interval = schedule

            # Check if Collecter with same name has already been scheduled
            if name in self.tasks:
                self.scheduler.cancel(self.tasks[name])
                # Log
                self.log.debug("Canceled task: %s", name)

            method = diamond.scheduler.method.sequential

            if 'method' in c.config:
                if c.config['method'] == 'Threaded':
                    method = diamond.scheduler.method.threaded
                elif c.config['method'] == 'Forked':
                    method = diamond.scheduler.method.forked

            # Schedule Collector
            if interval_task:
                task = self.scheduler.add_interval_task(func,
                                                        name,
                                                        splay,
                                                        interval,
                                                        method,
                                                        args,
                                                        None,
                                                        True)
            else:
                task = self.scheduler.add_single_task(func,
                                                      name,
                                                      splay,
                                                      method,
                                                      args,
                                                      None)

            # Log
            self.log.debug("Scheduled task: %s", name)
            # Add task to list
            self.tasks[name] = task

    def run(self):
        """
        Load handler and collector classes and then start collectors
        """

        # Set Running Flag
        self.running = True

        # Load handlers
        if 'handlers_path' in self.config['server']:
            handlers_path = self.config['server']['handlers_path']
            self.load_include_path(handlers_path)
        self.load_handlers()

        # Load config
        self.load_config()

        # Load collectors
        collectors_path = self.config['server']['collectors_path']
        self.load_include_path(collectors_path)
        collectors = self.load_collectors(collectors_path)

        # Setup Collectors
        for cls in collectors.values():
            # Initialize Collector
            c = self.init_collector(cls)
            # Schedule Collector
            self.schedule_collector(c)

        # Start main loop
        self.mainloop()

    def run_one(self, file):
        """
        Run given collector once and then exit
        """
        # Set Running Flag
        self.running = True

        # Load handlers
        if 'handlers_path' in self.config['server']:
            handlers_path = self.config['server']['handlers_path']
            self.load_include_path(handlers_path)
        self.load_handlers()

        # Overrides collector config dir
        collector_config_path = os.path.abspath(os.path.dirname(file))
        self.config['server']['collectors_config_path'] = collector_config_path

        # Load config
        self.load_config()

        # Load collectors
        self.load_include_path(os.path.dirname(file))
        collectors = self.load_collectors(os.path.dirname(file), file)

        # Setup Collectors
        for cls in collectors.values():
            # Initialize Collector
            c = self.init_collector(cls)

            # Schedule collector
            self.schedule_collector(c, False)

        # Start main loop
        self.mainloop(False)

    def mainloop(self, reload=True):

        # Start scheduler
        self.scheduler.start()

        # Log
        self.log.info('Started task scheduler.')

        # Initialize reload timer
        time_since_reload = 0

        # Main Loop
        while self.running:
            time.sleep(1)
            time_since_reload += 1

            # Check if its time to reload collectors
            if (reload
                    and time_since_reload
                    > int(self.config['server']['collectors_reload_interval'])):
                self.log.debug("Reloading config.")
                self.load_config()
                # Log
                self.log.debug("Reloading collectors.")
                # Load collectors
                collectors_path = self.config['server']['collectors_path']
                collectors = self.load_collectors(collectors_path)
                # Setup any Collectors that were loaded
                for cls in collectors.values():
                    # Initialize Collector
                    c = self.init_collector(cls)
                    # Schedule Collector
                    self.schedule_collector(c)

                # Reset reload timer
                time_since_reload = 0

            # Is the queue empty and we won't attempt to reload it? Exit
            if not reload and len(self.scheduler.sched._queue) == 0:
                self.running = False

        # Log
        self.log.debug('Stopping task scheduler.')
        # Stop scheduler
        self.scheduler.stop()
        # Log
        self.log.info('Stopped task scheduler.')
        # Log
        self.log.debug("Exiting.")

    def stop(self):
        """
        Close all connections and terminate threads.
        """
        # Set Running Flag
        self.running = False
<commit_msg>Skip collectors and handlers with syntax errors.<commit_after># coding=utf-8

import os
import sys
import time
import logging
import traceback
import configobj
import inspect

# Path Fix
sys.path.append(
    os.path.abspath(
        os.path.join(
            os.path.dirname(__file__), "../")))

import diamond

from diamond.collector import Collector
from diamond.handler.Handler import Handler
from diamond.scheduler import ThreadedScheduler
from diamond.util import load_class_from_name

class Server(object):
    """
    Server class loads and starts Handlers and Collectors
    """

    def __init__(self, config):
        # Initialize Logging
        self.log = logging.getLogger('diamond')
        # Initialize Members
        self.config = config
        self.running = False
        self.handlers = []
        self.modules = {}
        self.tasks = {}
        # Initialize Scheduler
        self.scheduler = ThreadedScheduler()

    def load_config(self):
        """
        Load the full config
        """

        configfile = os.path.abspath(self.config['configfile'])
        config = configobj.ConfigObj(configfile)
        config['configfile'] = self.config['configfile']

        # Merge in handler config files into the main config
        if 'handlers_config_path' in config['server']:
            files = os.listdir(config['server']['handlers_config_path'])
            for filename in files:
                configname = os.path.basename(filename)
                handlername = configname.split('.')[0]
                if handlername not in self.config['handlers']:
                    config['handlers'][handlername] = configobj.ConfigObj()

                configfile = os.path.join(
                    config['server']['handlers_config_path'],
                    configname)
                hconfig = configobj.ConfigObj(configfile)
                if handlername in config['handlers']:
                    config['handlers'][handlername].merge(hconfig)
                else:
                    config['handlers'][handlername] = hconfig

        self.config = config

    def load_handler(self, fqcn):
        """
        Load Handler class named fqcn
        """
        # Load class
        cls = load_class_from_name(fqcn)
        # Check if cls is subclass of Handler
        if cls == Handler or not issubclass(cls, Handler):
            raise TypeError("%s is not a valid Handler" % fqcn)
        # Log
        self.log.debug("Loaded Handler: %s", fqcn)
        return cls

    def load_handlers(self):
        """
        Load handlers
        """
        if type(self.config['server']['handlers']) == str:
            handlers = [self.config['server']['handlers']]
            self.config['server']['handlers'] = handlers

        for h in self.config['server']['handlers']:
            try:
                # Load Handler Class
                cls = self.load_handler(h)

                # Initialize Handler config
                handler_config = configobj.ConfigObj()
                # Merge default Handler default config
                handler_config.merge(self.config['handlers']['default'])
                # Check if Handler config exists
                if cls.__name__ in self.config['handlers']:
                    # Merge Handler config section
                    handler_config.merge(self.config['handlers'][cls.__name__])

                # Initialize Handler class
                self.handlers.append(cls(handler_config))

            except (ImportError, SyntaxError):
                # Log Error
                self.log.debug("Failed to load handler %s. %s", h,
                               traceback.format_exc())
                continue

    def load_collector(self, fqcn):
        """
        Load Collector class named fqcn
        """
        # Load class
        cls = load_class_from_name(fqcn)
        # Check if cls is subclass of Collector
        if cls == Collector or not issubclass(cls, Collector):
            raise TypeError("%s is not a valid Collector" % fqcn)
        # Log
        self.log.debug("Loaded Collector: %s", fqcn)
        return cls

    def load_include_path(self, path):
        """
        Scan for and add paths to the include path
        """
        # Verify the path is valid
        if not os.path.isdir(path):
            return
        # Add path to the system path, to avoid name clashes
        # with mysql-connector for example ...
        sys.path.insert(1, path)
        # Load all the files in path
        for f in os.listdir(path):
            # Are we a directory? If so process down the tree
            fpath = os.path.join(path, f)
            if os.path.isdir(fpath):
                self.load_include_path(fpath)

    def load_collectors(self, path, filter=None):
        """
        Scan for collectors to load from path
        """
        # Initialize return value
        collectors = {}

        # Get a list of files in the directory, if the directory exists
        if not os.path.exists(path):
            raise OSError("Directory does not exist: %s" % path)

        if path.endswith('tests') or path.endswith('fixtures'):
            return collectors

        # Log
        self.log.debug("Loading Collectors from: %s", path)

        # Load all the files in path
        for f in os.listdir(path):

            # Are we a directory? If so process down the tree
            fpath = os.path.join(path, f)
            if os.path.isdir(fpath):
                subcollectors = self.load_collectors(fpath)
                for key in subcollectors:
                    collectors[key] = subcollectors[key]

            # Ignore anything that isn't a .py file
            elif (os.path.isfile(fpath)
                  and len(f) > 3
                  and f[-3:] == '.py'
                  and f[0:4] != 'test'
                  and f[0] != '.'):

                # Check filter
                if filter and os.path.join(path, f) != filter:
                    continue

                modname = f[:-3]

                # Stat module file to get mtime
                st = os.stat(os.path.join(path, f))
                mtime = st.st_mtime
                # Check if module has been loaded before
                if modname in self.modules:
                    # Check if file mtime is newer then the last loaded verison
                    if mtime <= self.modules[modname]:
                        # Module hasn't changed
                        # Log
                        self.log.debug("Found %s, but it hasn't changed.",
                                       modname)
                        continue

                try:
                    # Import the module
                    mod = __import__(modname, globals(), locals(), ['*'])
                except (ImportError, SyntaxError):
                    # Log error
                    self.log.error("Failed to import module: %s. %s", modname,
                                   traceback.format_exc())
                    continue

                # Update module mtime
                self.modules[modname] = mtime
                # Log
                self.log.debug("Loaded Module: %s", modname)

                # Find all classes defined in the module
                for attrname in dir(mod):
                    attr = getattr(mod, attrname)
                    # Only attempt to load classes that are infact classes
                    # are Collectors but are not the base Collector class
                    if (inspect.isclass(attr)
                            and issubclass(attr, Collector)
                            and attr != Collector):
                        if attrname.startswith('parent_'):
                            continue
                        # Get class name
                        fqcn = '.'.join([modname, attrname])
                        try:
                            # Load Collector class
                            cls = self.load_collector(fqcn)
                            # Add Collector class
                            collectors[cls.__name__] = cls
                        except Exception:
                            # Log error
                            self.log.error("Failed to load Collector: %s. %s",
                                           fqcn, traceback.format_exc())
                            continue

        # Return Collector classes
        return collectors

    def init_collector(self, cls):
        """
        Initialize collector
        """
        collector = None
        try:
            # Initialize Collector
            collector = cls(self.config, self.handlers)
            # Log
            self.log.debug("Initialized Collector: %s", cls.__name__)
        except Exception:
            # Log error
            self.log.error("Failed to initialize Collector: %s. %s",
                           cls.__name__, traceback.format_exc())

        # Return collector
        return collector

    def schedule_collector(self, c, interval_task=True):
        """
        Schedule collector
        """
        # Check collector is for realz
        if c is None:
            self.log.warn("Skipped loading invalid Collector: %s",
                          c.__class__.__name__)
            return

        if c.config['enabled'] != True:
            self.log.warn("Skipped loading disabled Collector: %s",
                          c.__class__.__name__)
            return

        # Get collector schedule
        for name, schedule in c.get_schedule().items():
            # Get scheduler args
            func, args, splay, interval = schedule

            # Check if Collecter with same name has already been scheduled
            if name in self.tasks:
                self.scheduler.cancel(self.tasks[name])
                # Log
                self.log.debug("Canceled task: %s", name)

            method = diamond.scheduler.method.sequential

            if 'method' in c.config:
                if c.config['method'] == 'Threaded':
                    method = diamond.scheduler.method.threaded
                elif c.config['method'] == 'Forked':
                    method = diamond.scheduler.method.forked

            # Schedule Collector
            if interval_task:
                task = self.scheduler.add_interval_task(func,
                                                        name,
                                                        splay,
                                                        interval,
                                                        method,
                                                        args,
                                                        None,
                                                        True)
            else:
                task = self.scheduler.add_single_task(func,
                                                      name,
                                                      splay,
                                                      method,
                                                      args,
                                                      None)

            # Log
            self.log.debug("Scheduled task: %s", name)
            # Add task to list
            self.tasks[name] = task

    def run(self):
        """
        Load handler and collector classes and then start collectors
        """

        # Set Running Flag
        self.running = True

        # Load handlers
        if 'handlers_path' in self.config['server']:
            handlers_path = self.config['server']['handlers_path']
            self.load_include_path(handlers_path)
        self.load_handlers()

        # Load config
        self.load_config()

        # Load collectors
        collectors_path = self.config['server']['collectors_path']
        self.load_include_path(collectors_path)
        collectors = self.load_collectors(collectors_path)

        # Setup Collectors
        for cls in collectors.values():
            # Initialize Collector
            c = self.init_collector(cls)
            # Schedule Collector
            self.schedule_collector(c)

        # Start main loop
        self.mainloop()

    def run_one(self, file):
        """
        Run given collector once and then exit
        """
        # Set Running Flag
        self.running = True

        # Load handlers
        if 'handlers_path' in self.config['server']:
            handlers_path = self.config['server']['handlers_path']
            self.load_include_path(handlers_path)
        self.load_handlers()

        # Overrides collector config dir
        collector_config_path = os.path.abspath(os.path.dirname(file))
        self.config['server']['collectors_config_path'] = collector_config_path

        # Load config
        self.load_config()

        # Load collectors
        self.load_include_path(os.path.dirname(file))
        collectors = self.load_collectors(os.path.dirname(file), file)

        # Setup Collectors
        for cls in collectors.values():
            # Initialize Collector
            c = self.init_collector(cls)

            # Schedule collector
            self.schedule_collector(c, False)

        # Start main loop
        self.mainloop(False)

    def mainloop(self, reload=True):

        # Start scheduler
        self.scheduler.start()

        # Log
        self.log.info('Started task scheduler.')

        # Initialize reload timer
        time_since_reload = 0

        # Main Loop
        while self.running:
            time.sleep(1)
            time_since_reload += 1

            # Check if its time to reload collectors
            if (reload
                    and time_since_reload
                    > int(self.config['server']['collectors_reload_interval'])):
                self.log.debug("Reloading config.")
                self.load_config()
                # Log
                self.log.debug("Reloading collectors.")
                # Load collectors
                collectors_path = self.config['server']['collectors_path']
                collectors = self.load_collectors(collectors_path)
                # Setup any Collectors that were loaded
                for cls in collectors.values():
                    # Initialize Collector
                    c = self.init_collector(cls)
                    # Schedule Collector
                    self.schedule_collector(c)

                # Reset reload timer
                time_since_reload = 0

            # Is the queue empty and we won't attempt to reload it? Exit
            if not reload and len(self.scheduler.sched._queue) == 0:
                self.running = False

        # Log
        self.log.debug('Stopping task scheduler.')
        # Stop scheduler
        self.scheduler.stop()
        # Log
        self.log.info('Stopped task scheduler.')
        # Log
        self.log.debug("Exiting.")

    def stop(self):
        """
        Close all connections and terminate threads.
        """
        # Set Running Flag
        self.running = False
<|endoftext|>


<commit_before>from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic

from datetime import datetime

from djapian import utils

class ChangeManager(models.Manager):
    def create(self, object, action, **kwargs):
        ct = ContentType.objects.get_for_model(object.__class__)

        try:
            old_change = self.get(
                content_type=ct,
                object_id=object.pk
            )
            if old_change.action=="add":
                if action=="edit":
                    old_change.save()
                    return old_change
                elif action=="delete":
                    old_change.delete()
                    return None
            old_change.delete()
        except self.model.DoesNotExist:
            old_change = self.model(object=object)

        old_change.action = action
        old_change.save()

        return old_change

class Change(models.Model):
    ACTIOINS = (
        ("add", "object added"),
        ("edit", "object edited"),
        ("delete", "object deleted"),
    )

    content_type = models.ForeignKey(ContentType, db_index=True)
    object_id = models.PositiveIntegerField()
    date = models.DateTimeField(default=datetime.now)
    action = models.CharField(max_length=6, choices=ACTIOINS)

    object = generic.GenericForeignKey()

    objects = ChangeManager()

    def __unicode__(self):
        return u'%s#%d To action:`%s`, added on %s' % (
            self.content_type,
            self.object_id,
            self.action,
            self.date
        )

    def save(self):
        self.date = datetime.now()

        super(Change, self).save()

    class Meta:
        unique_together = [("content_type", "object_id")]
<commit_msg>fixed #58 with object_id type issue<commit_after>from django.db import models
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic

from datetime import datetime

from djapian import utils

class ChangeManager(models.Manager):
    def create(self, object, action, **kwargs):
        ct = ContentType.objects.get_for_model(object.__class__)

        try:
            old_change = self.get(
                content_type=ct,
                object_id=object.pk
            )
            if old_change.action=="add":
                if action=="edit":
                    old_change.save()
                    return old_change
                elif action=="delete":
                    old_change.delete()
                    return None
            old_change.delete()
        except self.model.DoesNotExist:
            old_change = self.model(object=object)

        old_change.action = action
        old_change.save()

        return old_change

class Change(models.Model):
    ACTIONS = (
        ("add", "object added"),
        ("edit", "object edited"),
        ("delete", "object deleted"),
    )

    content_type = models.ForeignKey(ContentType, db_index=True)
    object_id = models.CharField(max_length=150)

    date = models.DateTimeField(default=datetime.now)
    action = models.CharField(max_length=6, choices=ACTIONS)

    object = generic.GenericForeignKey()

    objects = ChangeManager()

    def __unicode__(self):
        return u'%s#%d To action:`%s`, added on %s' % (
            self.content_type,
            self.object_id,
            self.action,
            self.date
        )

    def save(self):
        self.date = datetime.now()

        super(Change, self).save()

    class Meta:
        unique_together = [("content_type", "object_id")]
<|endoftext|>


<commit_before>#!/usr/bin/python -u
#
# Copyright (c) 2015 Peter Ericson <pdericson@gmail.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

import copy
import datetime
import re
import shlex
import sys

import riemann_client.client
import riemann_client.transport

def handle_event(data):
    """Handle event

    create, etc

        >>> data = {'time': 0, 'id': 'abc', 'status': 'create', 'from': 'centos:7'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]

        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'create'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        'centos:7'

    exec_start: ...

        >>> data = {'time': 0, 'id': 'abc', 'status': 'exec_start: <command>', 'from': 'centos:7'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]

        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'exec_start: <command>'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        'centos:7'

    tag, etc

        >>> data = {'time': 0, 'id': 'abc', 'status': 'tag'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'tag'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        ''

    """
    event = {
        'time': data['time'],
        'state': 'ok',
        'service': 'docker',
        'tags': [],
        'ttl': 60,
        'attributes': {
            'event_status': data['status'],
            'event_id': data['id'],
            'event_from': data.get('from', ''),
        },
    }
    return [event]

def handle_log(line, info):
    """Handle a line of log output

        >>> info = {'Id': '', 'Image': '', 'Name': 'foo', 'Config': {'Image': '', 'Cmd': [], 'Entrypoint': ''}}

    Normal

        >>> data = handle_log(b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

        >>> data = handle_log(b"\\x02" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

    Not so normal

    Double magic

        >>> data = handle_log(b"\\x01" + (b"\\x00" * 7) + b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

    No magic

        >>> data = handle_log(b'2015-08-31T14:36:35.179583676Z HERE', info)
        >>> data[0]['attributes']['log']
        'HERE'

    Bad magic

        >>> data = handle_log(b'\\xb02015-08-31T14:41:43.702708748Z HERE', info)
        >>> data[0]['attributes']['log']
        'HERE'

    XXX

        >>> line = b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE"
        >>> info = {'Id': '', 'Image': '', 'Name': 'foo', 'Config': {'Image': '', 'Cmd': [], 'Entrypoint': ''}}

        >>> events = handle_log(line, info)

        >>> len(events)
        1
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        1441032103
        >>> event['state']
        'ok'
        >>> event['service']
        'container foo stdout'
        >>> event['tags']
        []
        >>> event['ttl']
        60

        >>> event['attributes']['container']
        'foo'
        >>> event['attributes']['container_id']
        ''
        >>> event['attributes']['image']
        ''
        >>> event['attributes']['image_id']
        ''
        >>> event['attributes']['log']
        'HERE'
        >>> event['attributes']['stream']
        'stdout'

    """

    try:
        a = {1: 'stdout', 2: 'stderr'}[line[0]]
    except KeyError as exc:
        print("Unable to detect stream: %r" % line, file=sys.stderr)
        a = 'stdout'  # We don't know

    m = re.search(rb'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{9}Z ', line)
    if m is None:
        raise Exception("missing timestamp")

    line = line[m.start():]

    b = line[:30]
    c = line[31:]

    event = {
        'time': int((datetime.datetime.strptime(b.decode('utf-8').split('.')[0], '%Y-%m-%dT%H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds()),
        'state': 'ok',
        'service': 'container %s %s' % (info['Name'].lstrip('/'), a),
        'tags': [],
        'ttl': 60,
        'attributes': {
            'container': info['Name'].lstrip('/'),
            'container_id': info['Id'],
            'image': info['Config']['Image'],
            'image_id': info['Image'],
            'container_cmd': ' '.join([shlex.quote(x) for x in (info['Config']['Cmd'] if info['Config']['Cmd'] is not None else [])]),

            'log': c.decode('utf-8'),
            'stream': a,
        },
    }
    return [event]

def handle_stat(data, info):
    """Handle stat

        >>> blkio_stats = {}
        >>> cpu_stats = {'cpu_usage': {'total_usage': 0, 'usage_in_kernelmode': 0, 'usage_in_usermode': 0, 'percpu_usage': []}}
        >>> memory_stats = {'limit': 0, 'usage': 0}
        >>> network = {}
        >>> data = {'read': '2015-09-23T04:13:56.297129480Z', 'blkio_stats': blkio_stats, 'cpu_stats': cpu_stats, 'memory_stats': memory_stats, 'network': network}
        >>> info = {'Name': 'foo', 'Id': '123', 'Config': {'Image': 'centos:7', 'Cmd': ['true'], 'Entrypoint': ''}, 'Image': 'abc'}
        >>> events = handle_stat(data, info)

        >>> len(events)
        5
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        1442981636
        >>> event['state']
        'ok'
        >>> event['service']
        'container foo cpu total usage'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['container']
        'foo'
        >>> event['attributes']['container_id']
        '123'
        >>> event['attributes']['image']
        'centos:7'
        >>> event['attributes']['image_id']
        'abc'

    """

    events = []

    time_ = int((datetime.datetime.strptime(data['read'].split('.')[0], '%Y-%m-%dT%H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds())

    attributes = {
        'container': info['Name'].lstrip('/'),
        'container_id': info['Id'],
        'image': info['Config']['Image'],
        'image_id': info['Image'],
        'container_cmd': ' '.join([shlex.quote(x) for x in (info['Config']['Cmd'] if info['Config']['Cmd'] is not None else [])]),
    }

    # blkio_stats
    for k, v in data['blkio_stats'].items():
        for x in v:
            event = {
                'time': time_,
                'state': 'ok',
                'service': 'container %s blkio %s %s' % (info['Name'].lstrip('/'), k, x['op'].lower()),
                'tags': [],
                'ttl': 60,
                'attributes': attributes,
                'metric_sint64': x['value'],
            }
            events.append(event)

    # cpu_stats
    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu total usage' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['total_usage'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu usage in kernelmode' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['usage_in_kernelmode'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu usage in usermode' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['usage_in_usermode'],
    }
    events.append(event)

    for i, x in enumerate(data['cpu_stats']['cpu_usage']['percpu_usage']):
        event = {
            'time': time_,
            'state': 'ok',
            'service': 'container %s cpu usage in cpu%i' % (info['Name'].lstrip('/'), i),
            'tags': [],
            'ttl': 60,
            'attributes': attributes,
            'metric_sint64': x,
        }
        events.append(event)

    # memory_stats
    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory limit' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['memory_stats']['limit'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory usage' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['memory_stats']['usage'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory usage percent' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_f': float(data['memory_stats']['usage']) / float(data['memory_stats']['limit']) * 100.0,
    }
    events.append(event)

    # https://www.kernel.org/doc/Documentation/cgroups/memory.txt
    for k in [
            'active_anon',
            'active_file',
            'cache',
            #'hierarchical_memory_limit',
            #'hierarchical_memsw_limit',
            'inactive_anon',
            'inactive_file',
            'mapped_file',
            'pgfault',
            'pgmajfault',
            'pgpgin',
            'pgpgout',
            'rss',
            'rss_huge',
            'swap',
            'unevictable',
            'writeback',
    ]:
        try:
            event = {
                'time': time_,
                'state': 'ok',
                'service': 'container %s memory %s' % (info['Name'].lstrip('/'), k),
                'tags': [],
                'ttl': 60,
                'attributes': attributes,
                'metric_sint64': data['memory_stats']['stats'].get('total_' + k, data['memory_stats']['stats'][k]),
            }
            events.append(event)
        except KeyError:
            # swap
            pass

    # network
    for k, v in data['network'].items():
        event = {
            'time': time_,
            'state': 'ok',
            'service': 'container %s network %s' % (info['Name'].lstrip('/'), k),
            'tags': [],
            'ttl': 60,
            'attributes': attributes,
            'metric_sint64': v,
        }
        events.append(event)

    # precpu_stats

    return events

def write_log(events):
    try:
        with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport("localhost", 5555)) as client:
            for event in events:
                client.event(**event)
                client.flush()
    except Exception as exc:
        pass
<commit_msg>Update reimann.py<commit_after>#!/usr/bin/python -u
#
# Copyright (c) 2015 Peter Ericson <pdericson@gmail.com>
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

import copy
import datetime
import re
import shlex
import sys

import riemann_client.client
import riemann_client.transport

def handle_event(data):
    """Handle event

    create, etc

        >>> data = {'time': 0, 'id': 'abc', 'status': 'create', 'from': 'centos:7'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]

        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'create'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        'centos:7'

    exec_start: ...

        >>> data = {'time': 0, 'id': 'abc', 'status': 'exec_start: <command>', 'from': 'centos:7'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]

        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'exec_start: <command>'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        'centos:7'

    tag, etc

        >>> data = {'time': 0, 'id': 'abc', 'status': 'tag'}
        >>> events = handle_event(data)
        >>> len(events)
        1
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        0
        >>> event['state']
        'ok'
        >>> event['service']
        'docker'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['event_status']
        'tag'
        >>> event['attributes']['event_id']
        'abc'
        >>> event['attributes']['event_from']
        ''

    """
    event = {
        'time': data['time'],
        'state': 'ok',
        'service': 'docker',
        'tags': [],
        'ttl': 60,
        'attributes': {
            'event_status': data['status'],
            'event_id': data['id'],
            'event_from': data.get('from', ''),
        },
    }
    return [event]

def handle_log(line, info):
    """Handle a line of log output

        >>> info = {'Id': '', 'Image': '', 'Name': 'foo', 'Config': {'Image': '', 'Cmd': [], 'Entrypoint': ''}}

    Normal

        >>> data = handle_log(b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

        >>> data = handle_log(b"\\x02" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

    Not so normal

    Double magic

        >>> data = handle_log(b"\\x01" + (b"\\x00" * 7) + b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE", info)
        >>> data[0]['attributes']['log']
        'HERE'

    No magic

        >>> data = handle_log(b'2015-08-31T14:36:35.179583676Z HERE', info)
        >>> data[0]['attributes']['log']
        'HERE'

    Bad magic

        >>> data = handle_log(b'\\xb02015-08-31T14:41:43.702708748Z HERE', info)
        >>> data[0]['attributes']['log']
        'HERE'

    XXX

        >>> line = b"\\x01" + (b"\\x00" * 7) + b"2015-08-31T14:41:43.702708748Z HERE"
        >>> info = {'Id': '', 'Image': '', 'Name': 'foo', 'Config': {'Image': '', 'Cmd': [], 'Entrypoint': ''}}

        >>> events = handle_log(line, info)

        >>> len(events)
        1
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        1441032103
        >>> event['state']
        'ok'
        >>> event['service']
        'container foo stdout'
        >>> event['tags']
        []
        >>> event['ttl']
        60

        >>> event['attributes']['container']
        'foo'
        >>> event['attributes']['container_id']
        ''
        >>> event['attributes']['image']
        ''
        >>> event['attributes']['image_id']
        ''
        >>> event['attributes']['log']
        'HERE'
        >>> event['attributes']['stream']
        'stdout'

    """

    try:
        a = {1: 'stdout', 2: 'stderr'}[line[0]]
    except KeyError as exc:
        print("Unable to detect stream: %r" % line, file=sys.stderr)
        a = 'stdout'  # We don't know

    m = re.search(rb'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{9}Z ', line)
    if m is None:
        raise Exception("missing timestamp")

    line = line[m.start():]

    b = line[:30]
    c = line[31:]

    event = {
        'time': int((datetime.datetime.strptime(b.decode('utf-8').split('.')[0], '%Y-%m-%dT%H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds()),
        'state': 'ok',
        'service': 'container %s %s' % (info['Name'].lstrip('/'), a),
        'tags': [],
        'ttl': 60,
        'attributes': {
            'container': info['Name'].lstrip('/'),
            'container_id': info['Id'],
            'image': info['Config']['Image'],
            'image_id': info['Image'],
            'container_cmd': ' '.join([shlex.quote(x) for x in (info['Config']['Cmd'] if info['Config']['Cmd'] is not None else [])]),

            'log': c.decode('utf-8'),
            'stream': a,
        },
    }
    return [event]

def handle_stat(data, info):
    """Handle stat

        >>> blkio_stats = {}
        >>> cpu_stats = {'cpu_usage': {'total_usage': 0, 'usage_in_kernelmode': 0, 'usage_in_usermode': 0, 'percpu_usage': []}}
        >>> memory_stats = {'limit': 0, 'usage': 0}
        >>> network = {}
        >>> data = {'read': '2015-09-23T04:13:56.297129480Z', 'blkio_stats': blkio_stats, 'cpu_stats': cpu_stats, 'memory_stats': memory_stats, 'network': network}
        >>> info = {'Name': 'foo', 'Id': '123', 'Config': {'Image': 'centos:7', 'Cmd': ['true'], 'Entrypoint': ''}, 'Image': 'abc'}
        >>> events = handle_stat(data, info)

        >>> len(events)
        5
        >>> event = events[0]
        >>> riemann_client.client.Client.create_event(copy.deepcopy(event))  # doctest: +ELLIPSIS
        <google.protobuf...>

        >>> event['time']
        1442981636
        >>> event['state']
        'ok'
        >>> event['service']
        'container foo cpu total usage'
        >>> event['tags']
        []
        >>> event['ttl']
        60
        >>> event['attributes']['container']
        'foo'
        >>> event['attributes']['container_id']
        '123'
        >>> event['attributes']['image']
        'centos:7'
        >>> event['attributes']['image_id']
        'abc'

    """

    events = []

    time_ = int((datetime.datetime.strptime(data['read'].split('.')[0], '%Y-%m-%dT%H:%M:%S') - datetime.datetime(1970,1,1)).total_seconds())

    attributes = {
        'container': info['Name'].lstrip('/'),
        'container_id': info['Id'],
        'image': info['Config']['Image'],
        'image_id': info['Image'],
        'container_cmd': ' '.join([shlex.quote(x) for x in (info['Config']['Cmd'] if info['Config']['Cmd'] is not None else [])]),
    }

    # blkio_stats
    for k, v in data['blkio_stats'].items():
        for x in v:
            event = {
                'time': time_,
                'state': 'ok',
                'service': 'container %s blkio %s %s' % (info['Name'].lstrip('/'), k, x['op'].lower()),
                'tags': [],
                'ttl': 60,
                'attributes': attributes,
                'metric_sint64': x['value'],
            }
            events.append(event)

    # cpu_stats
    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu total usage' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['total_usage'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu usage in kernelmode' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['usage_in_kernelmode'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s cpu usage in usermode' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['cpu_stats']['cpu_usage']['usage_in_usermode'],
    }
    events.append(event)

    for i, x in enumerate(data['cpu_stats']['cpu_usage']['percpu_usage']):
        event = {
            'time': time_,
            'state': 'ok',
            'service': 'container %s cpu usage in cpu%i' % (info['Name'].lstrip('/'), i),
            'tags': [],
            'ttl': 60,
            'attributes': attributes,
            'metric_sint64': x,
        }
        events.append(event)

    # memory_stats
    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory limit' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['memory_stats']['limit'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory usage' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': data['memory_stats']['usage'],
    }
    events.append(event)

    event = {
        'time': time_,
        'state': 'ok',
        'service': 'container %s memory usage percent' % info['Name'].lstrip('/'),
        'tags': [],
        'ttl': 60,
        'attributes': attributes,
        'metric_sint64': int(round(float(data['memory_stats']['usage']) / float(data['memory_stats']['limit']) * 100.0)),
    }
    events.append(event)

    # https://www.kernel.org/doc/Documentation/cgroups/memory.txt
    for k in [
            'active_anon',
            'active_file',
            'cache',
            #'hierarchical_memory_limit',
            #'hierarchical_memsw_limit',
            'inactive_anon',
            'inactive_file',
            'mapped_file',
            'pgfault',
            'pgmajfault',
            'pgpgin',
            'pgpgout',
            'rss',
            'rss_huge',
            'swap',
            'unevictable',
            'writeback',
    ]:
        try:
            event = {
                'time': time_,
                'state': 'ok',
                'service': 'container %s memory %s' % (info['Name'].lstrip('/'), k),
                'tags': [],
                'ttl': 60,
                'attributes': attributes,
                'metric_sint64': data['memory_stats']['stats'].get('total_' + k, data['memory_stats']['stats'][k]),
            }
            events.append(event)
        except KeyError:
            # swap
            pass

    # network
    for k, v in data['network'].items():
        event = {
            'time': time_,
            'state': 'ok',
            'service': 'container %s network %s' % (info['Name'].lstrip('/'), k),
            'tags': [],
            'ttl': 60,
            'attributes': attributes,
            'metric_sint64': v,
        }
        events.append(event)

    # precpu_stats

    return events

def write_log(events):
    try:
        with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport("localhost", 5555)) as client:
            for event in events:
                client.event(**event)
                client.flush()
    except Exception as exc:
        pass
<|endoftext|>


<commit_before>"""
Distribute pipeline processes across the cluster.
"""

import sys
import os
from glob import glob
from mpi4py import MPI
from seqUtils import convert_fasta
from datetime import datetime

def prop_x4 (handle, cutoff, min_count):
	fasta = convert_fasta(handle.readlines())
	total_count = 0
	total_x4_count = 0

	for h, s in fasta:
		# >F00309-IL_variant_0_count_27_fpr_4.0
		tokens = h.split('_')
		try:
			variant = int(tokens[tokens.index('variant')+1])
			count = int(tokens[tokens.index('count')+1])
			fpr = float(tokens[tokens.index('fpr')+1])
		except:
			print 'ERROR, missing variant, count or fpr in header'
			print h
			sys.exit()

		if count <= min_count:
			continue

		total_count += count
		if fpr < cutoff:
			total_x4_count += count

	return (total_x4_count, total_count)

my_rank = MPI.COMM_WORLD.Get_rank()
nprocs = MPI.COMM_WORLD.Get_size()

def timestamp(msg):
	print '[%s] (%d/%d) %s' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
			my_rank,
			nprocs,
			msg)

refpath = '/usr/local/share/miseq/refs/cfe'

# iterative mapping of FASTQs to references
root = sys.argv[1]
files = glob(root + '/*R1*.fastq')

for i in range(len(files)):
	if i % nprocs != my_rank:
		continue

	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 1_mapping.py on %s' % (my_rank, nprocs, filename)
	timestamp('1_mapping %s' % filename)
	os.system('python 1_mapping.py %s %s' % (refpath, files[i]))

MPI.COMM_WORLD.Barrier()

# collect all remapped SAM files
files = glob(root + '/*.remap.sam')

for i in range(len(files)):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 2_sam2fasta on %s' % (my_rank, nprocs, filename)
	timestamp('2_sam2fasta %s' % filename)
	for qcut in [0, 10, 15, 20]:
		os.system('python 2_sam2fasta_with_base_censoring.py %s %d' % (files[i], qcut))

# compute g2p scores for env-mapped FASTAs
files = glob(root + '/*env*.fasta')

for i, file in enumerate(files):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 3_g2p_scoring on %s' % (my_rank, nprocs, filename)
	timestamp('3_g2p_scoring %s' % filename)
	os.system('python 3_g2p_scoring.py %s' % file) # generates *.v3prot and *.badV3

# generate amino acid count CSVs
files = glob(root + '/*.fasta')

for i, file in enumerate(files):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 5_amino_freqs on %s' % (my_rank, nprocs, filename)
	timestamp('5_amino_freqs %s' % filename)
	os.system('python 5_amino_freqs.py %s' % file)

MPI.COMM_WORLD.Barrier()
MPI.Finalize()

# summary stats in serial mode
if my_rank == 0:
	outfile = open(root+'/_g2p.csv', 'w')
	outfile.write('sample,q.cut,fpr.cut,min.count,x4.count,total.count,prop.x4\n')

	files = glob(root + '/*.v3prot')
	for file in files:
		filename = file.split('/')[-1]
		sample, region, qcut = filename.split('.')[:3]
		qcut = int(qcut)
		for mincount in [0, 1, 3, 50]:
			for cutoff in [3.0, 3.5, 4.0, 5.0, 5.75, 7.5]:
				infile = open(file, 'rU')
				x4_count, total_count = prop_x4(infile, cutoff, mincount)
				infile.close()

				outfile.write('%s,%d,%f,%d,%d,%d,%s\n' % (sample, qcut, cutoff,
					mincount, x4_count, total_count,
					str(x4_count/float(total_count)) if total_count > 0 else 'NA'))

	outfile.close()

<commit_msg>allow exception to pass in prop_x4 if FPR missing in header, occasionally sequence is not scored<commit_after>"""
Distribute pipeline processes across the cluster.
"""

import sys
import os
from glob import glob
from mpi4py import MPI
from seqUtils import convert_fasta
from datetime import datetime

def prop_x4 (handle, cutoff, min_count):
	fasta = convert_fasta(handle.readlines())
	total_count = 0
	total_x4_count = 0

	for h, s in fasta:
		# >F00309-IL_variant_0_count_27_fpr_4.0
		tokens = h.split('_')
		try:
			variant = int(tokens[tokens.index('variant')+1])
			count = int(tokens[tokens.index('count')+1])
			fpr = float(tokens[tokens.index('fpr')+1])
		except:
			print 'ERROR, missing variant, count or fpr in header'
			print h, s
			pass

		if count <= min_count:
			continue

		total_count += count
		if fpr < cutoff:
			total_x4_count += count

	return (total_x4_count, total_count)

my_rank = MPI.COMM_WORLD.Get_rank()
nprocs = MPI.COMM_WORLD.Get_size()

def timestamp(msg):
	print '[%s] (%d/%d) %s' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
			my_rank,
			nprocs,
			msg)

refpath = '/usr/local/share/miseq/refs/cfe'

# iterative mapping of FASTQs to references
root = sys.argv[1]
files = glob(root + '/*R1*.fastq')

for i in range(len(files)):
	if i % nprocs != my_rank:
		continue

	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 1_mapping.py on %s' % (my_rank, nprocs, filename)
	timestamp('1_mapping %s' % filename)
	os.system('python 1_mapping.py %s %s' % (refpath, files[i]))

MPI.COMM_WORLD.Barrier()

# collect all remapped SAM files
files = glob(root + '/*.remap.sam')

for i in range(len(files)):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 2_sam2fasta on %s' % (my_rank, nprocs, filename)
	timestamp('2_sam2fasta %s' % filename)
	for qcut in [0, 10, 15, 20]:
		os.system('python 2_sam2fasta_with_base_censoring.py %s %d' % (files[i], qcut))

# compute g2p scores for env-mapped FASTAs
files = glob(root + '/*env*.fasta')

for i, file in enumerate(files):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 3_g2p_scoring on %s' % (my_rank, nprocs, filename)
	timestamp('3_g2p_scoring %s' % filename)
	os.system('python 3_g2p_scoring.py %s' % file) # generates *.v3prot and *.badV3

# generate amino acid count CSVs
files = glob(root + '/*.fasta')

for i, file in enumerate(files):
	if i % nprocs != my_rank:
		continue
	filename = files[i].split('/')[-1]
	#print '... process %d of %d starting task 5_amino_freqs on %s' % (my_rank, nprocs, filename)
	timestamp('5_amino_freqs %s' % filename)
	os.system('python 5_amino_freqs.py %s' % file)

MPI.COMM_WORLD.Barrier()
MPI.Finalize()

# summary stats in serial mode
if my_rank == 0:
	outfile = open(root+'/_g2p.csv', 'w')
	outfile.write('sample,q.cut,fpr.cut,min.count,x4.count,total.count,prop.x4\n')

	files = glob(root + '/*.v3prot')
	for file in files:
		filename = file.split('/')[-1]
		sample, region, qcut = filename.split('.')[:3]
		qcut = int(qcut)
		for mincount in [0, 1, 3, 50]:
			for cutoff in [3.0, 3.5, 4.0, 5.0, 5.75, 7.5]:
				infile = open(file, 'rU')
				x4_count, total_count = prop_x4(infile, cutoff, mincount)
				infile.close()

				outfile.write('%s,%d,%f,%d,%d,%d,%s\n' % (sample, qcut, cutoff,
					mincount, x4_count, total_count,
					str(x4_count/float(total_count)) if total_count > 0 else 'NA'))

	outfile.close()

<|endoftext|>