<commit_before># coding: utf-8

from __future__ import unicode_literals
import json

from django.db.models import FieldDoesNotExist, Q
from django.db.models.query import ValuesListQuerySet
from django.http import HttpResponse
from django.utils.encoding import force_text
from django.utils.http import urlunquote
from django.utils.text import capfirst
from django.views.generic import ListView

class TableView(ListView):
    columns = ()
    columns_widths = {}
    verbose_columns = {}
    search_lookups = ()
    orderings = {}
    filters = {}
    template_name = 'tablature/table_page.html'
    results_per_page = 15

    def get_columns(self):
        if self.columns:
            return self.columns
        return [f.name for f in self.model._meta.fields]

    def get_field(self, column):
        try:
            return self.model._meta.get_field(column)
        except FieldDoesNotExist:
            pass

    def get_verbose_column(self, column):
        if column in self.verbose_columns:
            return self.verbose_columns[column]
        field = self.get_field(column)
        if field is None:
            method = getattr(self.model, column)
            if hasattr(method, 'short_description'):
                column = method.short_description
        else:
            column = field.verbose_name
        return capfirst(column)

    def get_column_width(self, column):
        if column in self.columns_widths:
            return self.columns_widths[column]
        return 'initial'

    def get_context_data(self, **kwargs):
        context = super(TableView, self).get_context_data(**kwargs)
        columns = self.get_columns()
        context.update(
            verbose_name_plural=self.model._meta.verbose_name_plural,
            columns=[self.get_verbose_column(c) for c in columns],
            columns_widths=[self.get_column_width(c) for c in columns],
            search_lookups=self.search_lookups,
            sortables=['true' if self.get_ordering_for_column(c, 1)
                       else 'false' for c in columns],
            filters=[self.get_filter(c) for c in columns],
            results_per_page=self.results_per_page)
        return context

    def get_ordering_for_column(self, column, direction):
        """
        Returns a tuple of lookups to order by for the given column
        and direction. Direction is an integer, either -1, 0 or 1.
        """
        if direction == 0:
            return ()
        if column in self.orderings:
            ordering = self.orderings[column]
        else:
            field = self.get_field(column)
            if field is None:
                return ()
            ordering = column
        if not isinstance(ordering, (tuple, list)):
            ordering = [ordering]
        if direction == 1:
            return ordering
        return [lookup[1:] if lookup[0] == '-' else '-' + lookup
                for lookup in ordering]

    def get_filter(self, column):
        if column not in self.filters:
            return ()
        values = self.filters[column]
        if isinstance(values, ValuesListQuerySet):
            values = tuple(values.all())
        assert len(values[0]) == 2, 'Each filter must be a value/verbose pair.'
        return values

    def search(self, queryset, q):
        if not q:
            return queryset
        filters = Q()
        for lookup in self.search_lookups:
            filters |= Q(**{lookup: q})
        if filters:
            return queryset.filter(filters)
        return queryset.none()

    def get_results_queryset(self):
        qs = self.get_queryset()
        GET = self.request.GET
        qs = self.search(qs, GET.get('q'))
        columns = self.get_columns()

        filter_choices = map(urlunquote, GET.get('choices', '').split(','))
        for column, choice in zip(columns, filter_choices):
            if choice:
                method = getattr(self, 'filter_' + column, None)
                qs = (qs.filter(**{column: choice}) if method is None
                      else method(qs, choice))

        order_directions = map(int, GET.get('orderings', '').split(','))
        order_by = []
        for column, direction in zip(columns, order_directions):
            order_by.extend(self.get_ordering_for_column(column, direction))
        if order_by:
            qs = qs.order_by(*order_by)
        return qs.distinct()

    def get_limited_results_queryset(self):
        qs = self.get_results_queryset()
        current_page = int(self.request.GET.get('page', '0'))
        offset = current_page * self.results_per_page
        return qs[offset:offset + self.results_per_page]

    def get_results(self):
        results = []
        for obj in self.get_limited_results_queryset():
            row = []
            for attr in self.get_columns():
                verbose_attr = 'get_' + attr + '_display'
                if hasattr(obj, verbose_attr):
                    attr = verbose_attr
                v = getattr(obj, attr)
                if callable(v):
                    v = v()
                v = '' if v is None else force_text(v)
                row.append(v)
            results.append(row)
        return results

    def get_data(self):
        return {'results': self.get_results(),
                'count': self.get_results_queryset().count()}

    def get(self, request, *args, **kwargs):
        if request.is_ajax():
            return HttpResponse(json.dumps(self.get_data()),
                                content_type='application/json')
        return super(TableView, self).get(request, *args, **kwargs)
<commit_msg>Fixes Django 1.9 compatibility.<commit_after># coding: utf-8

from __future__ import unicode_literals
import json

from django.db.models import FieldDoesNotExist, Q
from django.db.models.query import QuerySet
from django.http import HttpResponse
from django.utils.encoding import force_text
from django.utils.http import urlunquote
from django.utils.text import capfirst
from django.views.generic import ListView

class TableView(ListView):
    columns = ()
    columns_widths = {}
    verbose_columns = {}
    search_lookups = ()
    orderings = {}
    filters = {}
    template_name = 'tablature/table_page.html'
    results_per_page = 15

    def get_columns(self):
        if self.columns:
            return self.columns
        return [f.name for f in self.model._meta.fields]

    def get_field(self, column):
        try:
            return self.model._meta.get_field(column)
        except FieldDoesNotExist:
            pass

    def get_verbose_column(self, column):
        if column in self.verbose_columns:
            return self.verbose_columns[column]
        field = self.get_field(column)
        if field is None:
            method = getattr(self.model, column)
            if hasattr(method, 'short_description'):
                column = method.short_description
        else:
            column = field.verbose_name
        return capfirst(column)

    def get_column_width(self, column):
        if column in self.columns_widths:
            return self.columns_widths[column]
        return 'initial'

    def get_context_data(self, **kwargs):
        context = super(TableView, self).get_context_data(**kwargs)
        columns = self.get_columns()
        context.update(
            verbose_name_plural=self.model._meta.verbose_name_plural,
            columns=[self.get_verbose_column(c) for c in columns],
            columns_widths=[self.get_column_width(c) for c in columns],
            search_lookups=self.search_lookups,
            sortables=['true' if self.get_ordering_for_column(c, 1)
                       else 'false' for c in columns],
            filters=[self.get_filter(c) for c in columns],
            results_per_page=self.results_per_page)
        return context

    def get_ordering_for_column(self, column, direction):
        """
        Returns a tuple of lookups to order by for the given column
        and direction. Direction is an integer, either -1, 0 or 1.
        """
        if direction == 0:
            return ()
        if column in self.orderings:
            ordering = self.orderings[column]
        else:
            field = self.get_field(column)
            if field is None:
                return ()
            ordering = column
        if not isinstance(ordering, (tuple, list)):
            ordering = [ordering]
        if direction == 1:
            return ordering
        return [lookup[1:] if lookup[0] == '-' else '-' + lookup
                for lookup in ordering]

    def get_filter(self, column):
        if column not in self.filters:
            return ()
        values = self.filters[column]
        if isinstance(values, QuerySet):
            values = tuple(values.all())
        assert len(values[0]) == 2, 'Each filter must be a value/verbose pair.'
        return values

    def search(self, queryset, q):
        if not q:
            return queryset
        filters = Q()
        for lookup in self.search_lookups:
            filters |= Q(**{lookup: q})
        if filters:
            return queryset.filter(filters)
        return queryset.none()

    def get_results_queryset(self):
        qs = self.get_queryset()
        GET = self.request.GET
        qs = self.search(qs, GET.get('q'))
        columns = self.get_columns()

        filter_choices = map(urlunquote, GET.get('choices', '').split(','))
        for column, choice in zip(columns, filter_choices):
            if choice:
                method = getattr(self, 'filter_' + column, None)
                qs = (qs.filter(**{column: choice}) if method is None
                      else method(qs, choice))

        order_directions = map(int, GET.get('orderings', '').split(','))
        order_by = []
        for column, direction in zip(columns, order_directions):
            order_by.extend(self.get_ordering_for_column(column, direction))
        if order_by:
            qs = qs.order_by(*order_by)
        return qs.distinct()

    def get_limited_results_queryset(self):
        qs = self.get_results_queryset()
        current_page = int(self.request.GET.get('page', '0'))
        offset = current_page * self.results_per_page
        return qs[offset:offset + self.results_per_page]

    def get_results(self):
        results = []
        for obj in self.get_limited_results_queryset():
            row = []
            for attr in self.get_columns():
                verbose_attr = 'get_' + attr + '_display'
                if hasattr(obj, verbose_attr):
                    attr = verbose_attr
                v = getattr(obj, attr)
                if callable(v):
                    v = v()
                v = '' if v is None else force_text(v)
                row.append(v)
            results.append(row)
        return results

    def get_data(self):
        return {'results': self.get_results(),
                'count': self.get_results_queryset().count()}

    def get(self, request, *args, **kwargs):
        if request.is_ajax():
            return HttpResponse(json.dumps(self.get_data()),
                                content_type='application/json')
        return super(TableView, self).get(request, *args, **kwargs)
<|endoftext|>


<commit_before>'''
A simple way of setting the output format for data from modules
'''

import json
import pprint

import yaml

import salt.utils

__all__ = ('get_outputter',)

# FIXME: get rid of the JSON constant as it's acutally not needed anymore
JSON = True

def remove_colors():
    '''
    Acces all of the utility colors and change them to empy strings
    '''
    pass

class Outputter(object):
    '''
    Class for outputting data to the screen.
    '''
    supports = None

    @classmethod
    def check(cls, name):
        # Don't advertise Outputter classes for optional modules
        if hasattr(cls, "enabled") and not cls.enabled: # FIXME: class Outputter
            return False                                # has no enabled member
        return cls.supports == name

    def __call__(self, data, **kwargs):
        pprint.pprint(data)

class HighStateOutputter(Outputter):
    '''
    Not a command line option, the HighStateOutputter is only meant to be used
    with the state.highstate function, or a function that returns highstate
    return data
    '''
    supports = 'highstate'

    def __call__(self, data, **kwargs):
        colors = salt.utils.get_colors(kwargs.get('color'))
        for host in data:
            hcolor = colors['GREEN']
            hstrs = []
            if isinstance(data[host], list):
                # Errors have been detected, list them in RED!
                hcolor = colors['RED_BOLD']
                hstrs.append(('    {0}Data failed to compile:{1[ENDC]}'
                              .format(hcolor, colors)))
                for err in data[host]:
                    hstrs.append(('{0}----------\n    {1}{2[ENDC]}'
                                  .format(hcolor, err, colors)))
            if isinstance(data[host], dict):
                # Everything rendered as it should display the output
                for tname, ret in data[host].items():
                    tcolor = colors['GREEN']
                    if not ret['result']:
                        hcolor = colors['RED']
                        tcolor = colors['RED']
                    if ret['changes']:
                        tcolor = colors['CYAN']
                    comps = tname.split('.')
                    hstrs.append(('{0}----------\n    State: - {1}{2[ENDC]}'
                                  .format(tcolor, comps[0], colors)))
                    # FIXME: string formating below should match the
                    # style above
                    hstrs.append('    {0}Name:      {1}{2[ENDC]}'.format(
                        tcolor,
                        '.'.join(comps[1:-1]),
                        colors
                        ))
                    hstrs.append('    {0}Function:  {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[-1],
                        colors
                        ))
                    hstrs.append('        {0}Result:    {1}{2[ENDC]}'.format(
                        tcolor,
                        str(ret['result']),
                        colors
                        ))
                    hstrs.append('        {0}Comment:   {1}{2[ENDC]}'.format(
                        tcolor,
                        ret['comment'],
                        colors
                        ))
                    changes = '        Changes:   '
                    for key in ret['changes']:
                        if isinstance(ret['changes'][key], str):
                            changes += (key + ': ' + ret['changes'][key] +
                                        '\n                   ')
                        elif isinstance(ret['changes'][key], dict):
                            changes += (key + ': ' +
                                        pprint.pformat(ret['changes'][key]) +
                                        '\n                   ')
                        else:
                            changes += (key + ': ' +
                                        pprint.pformat(ret['changes'][key]) +
                                        '\n                   ')
                    hstrs.append(('{0}{1}{2[ENDC]}'
                                  .format(tcolor, changes, colors)))
            print('{0}{1}:{2[ENDC]}'.format(hcolor, host, colors))
            for hstr in hstrs:
                print hstr

class RawOutputter(Outputter):
    '''
    Raw output. This calls repr() on the returned data.
    '''
    supports = "raw"

    def __call__(self, data, **kwargs):
        print data

class TxtOutputter(Outputter):
    '''
    Plain text output. Primarily for returning output from
    shell commands in the exact same way they would output
    on the shell when ran directly.
    '''
    supports = "txt"

    def __call__(self, data, **kwargs):
        if hasattr(data, "keys"):
            for key in data.keys():
                value = data[key]
                for line in value.split('\n'):
                    print "{0}: {1}".format(key, line)
        else:
            # For non-dictionary data, just use print
            RawOutputter()(data)

class JSONOutputter(Outputter):
    '''
    JSON output.
    '''
    supports = "json"
    enabled = JSON

    def __call__(self, data, **kwargs):
        if hasattr(self, 'indent'):
            kwargs.update({'indent': self.indent})
        try:
            # A good kwarg might be: indent=4
            if 'color' in kwargs:
                kwargs.pop('color')
            ret = json.dumps(data, **kwargs)
        except TypeError:
            # Return valid json for unserializable objects
            ret = json.dumps({})
        print ret

class YamlOutputter(Outputter):
    '''
    Yaml output. All of the cool kids are doing it.
    '''
    supports = "yaml"

    def __call__(self, data, **kwargs):
        if 'color' in kwargs:
            kwargs.pop('color')
        print yaml.dump(data, **kwargs)

def get_outputter(name=None):
    '''
    Factory function for returning the right output class.

    Usage:
        printout = get_outputter("txt")
        printout(ret)
    '''
    # Return an actual instance of the correct output class
    for i in Outputter.__subclasses__():  # FIXME: class Outputter has no
        if i.check(name):                 # __subclasses__ member
            return i()
    return Outputter()
<commit_msg>Properly print non-strings with the TxtOutputter<commit_after>'''
A simple way of setting the output format for data from modules
'''

import json
import pprint

import yaml

import salt.utils

__all__ = ('get_outputter',)

# FIXME: get rid of the JSON constant as it's acutally not needed anymore
JSON = True

def remove_colors():
    '''
    Acces all of the utility colors and change them to empy strings
    '''
    pass

class Outputter(object):
    '''
    Class for outputting data to the screen.
    '''
    supports = None

    @classmethod
    def check(cls, name):
        # Don't advertise Outputter classes for optional modules
        if hasattr(cls, "enabled") and not cls.enabled: # FIXME: class Outputter
            return False                                # has no enabled member
        return cls.supports == name

    def __call__(self, data, **kwargs):
        pprint.pprint(data)

class HighStateOutputter(Outputter):
    '''
    Not a command line option, the HighStateOutputter is only meant to be used
    with the state.highstate function, or a function that returns highstate
    return data
    '''
    supports = 'highstate'

    def __call__(self, data, **kwargs):
        colors = salt.utils.get_colors(kwargs.get('color'))
        for host in data:
            hcolor = colors['GREEN']
            hstrs = []
            if isinstance(data[host], list):
                # Errors have been detected, list them in RED!
                hcolor = colors['RED_BOLD']
                hstrs.append(('    {0}Data failed to compile:{1[ENDC]}'
                              .format(hcolor, colors)))
                for err in data[host]:
                    hstrs.append(('{0}----------\n    {1}{2[ENDC]}'
                                  .format(hcolor, err, colors)))
            if isinstance(data[host], dict):
                # Everything rendered as it should display the output
                for tname, ret in data[host].items():
                    tcolor = colors['GREEN']
                    if not ret['result']:
                        hcolor = colors['RED']
                        tcolor = colors['RED']
                    if ret['changes']:
                        tcolor = colors['CYAN']
                    comps = tname.split('.')
                    hstrs.append(('{0}----------\n    State: - {1}{2[ENDC]}'
                                  .format(tcolor, comps[0], colors)))
                    # FIXME: string formating below should match the
                    # style above
                    hstrs.append('    {0}Name:      {1}{2[ENDC]}'.format(
                        tcolor,
                        '.'.join(comps[1:-1]),
                        colors
                        ))
                    hstrs.append('    {0}Function:  {1}{2[ENDC]}'.format(
                        tcolor,
                        comps[-1],
                        colors
                        ))
                    hstrs.append('        {0}Result:    {1}{2[ENDC]}'.format(
                        tcolor,
                        str(ret['result']),
                        colors
                        ))
                    hstrs.append('        {0}Comment:   {1}{2[ENDC]}'.format(
                        tcolor,
                        ret['comment'],
                        colors
                        ))
                    changes = '        Changes:   '
                    for key in ret['changes']:
                        if isinstance(ret['changes'][key], str):
                            changes += (key + ': ' + ret['changes'][key] +
                                        '\n                   ')
                        elif isinstance(ret['changes'][key], dict):
                            changes += (key + ': ' +
                                        pprint.pformat(ret['changes'][key]) +
                                        '\n                   ')
                        else:
                            changes += (key + ': ' +
                                        pprint.pformat(ret['changes'][key]) +
                                        '\n                   ')
                    hstrs.append(('{0}{1}{2[ENDC]}'
                                  .format(tcolor, changes, colors)))
            print('{0}{1}:{2[ENDC]}'.format(hcolor, host, colors))
            for hstr in hstrs:
                print hstr

class RawOutputter(Outputter):
    '''
    Raw output. This calls repr() on the returned data.
    '''
    supports = "raw"

    def __call__(self, data, **kwargs):
        print data

class TxtOutputter(Outputter):
    '''
    Plain text output. Primarily for returning output from
    shell commands in the exact same way they would output
    on the shell when ran directly.
    '''
    supports = "txt"

    def __call__(self, data, **kwargs):
        if hasattr(data, "keys"):
            for key in data.keys():
                value = data[key]
                # Don't blow up on non-strings
                try:
                    for line in value.split('\n'):
                        print "{0}: {1}".format(key, line)
                except AttributeError:
                    print "key: %s" % value
        else:
            # For non-dictionary data, just use print
            RawOutputter()(data)

class JSONOutputter(Outputter):
    '''
    JSON output.
    '''
    supports = "json"
    enabled = JSON

    def __call__(self, data, **kwargs):
        if hasattr(self, 'indent'):
            kwargs.update({'indent': self.indent})
        try:
            # A good kwarg might be: indent=4
            if 'color' in kwargs:
                kwargs.pop('color')
            ret = json.dumps(data, **kwargs)
        except TypeError:
            # Return valid json for unserializable objects
            ret = json.dumps({})
        print ret

class YamlOutputter(Outputter):
    '''
    Yaml output. All of the cool kids are doing it.
    '''
    supports = "yaml"

    def __call__(self, data, **kwargs):
        if 'color' in kwargs:
            kwargs.pop('color')
        print yaml.dump(data, **kwargs)

def get_outputter(name=None):
    '''
    Factory function for returning the right output class.

    Usage:
        printout = get_outputter("txt")
        printout(ret)
    '''
    # Return an actual instance of the correct output class
    for i in Outputter.__subclasses__():  # FIXME: class Outputter has no
        if i.check(name):                 # __subclasses__ member
            return i()
    return Outputter()
<|endoftext|>


<commit_before>"""
sentry.tasks.check_alerts
~~~~~~~~~~~~~~~~~~~~~~~~~

- Store a sorted set per project
- Each sorted set contains the number of events seen in the interval (1 minute)
  - An additional set contains the number of unique events seen
- Every minute we iterate this sorted set (we can exploit the queue just like buffers to avoid crons)
  - We clear the results immediately to no-op any concurrent tasks that might try to run
  - The task fires off a set of subtasks that individually check each project
    - Each project's value is compared to the historic value in the last N minutes (15m for redis counters or
      a period of time using the SQL counters)
    - We only alert if an alert has not been seen on this condition in the last N minutes

Notes:

- Nydus optimizes out multiple writes/gets, so its not as expensive as it looks
- If there are not enough data points in history, alerts are never sent (as they could be false positives)

Last Notified will be tracked by using a new Alert model, which has the following properties:

- users alerted
- datetime
- type
- ident

Type and ident would be similar to how Activity works.

Alert expiration threshold MUST be > MINUTE_NORMALIZATION.

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import division

import time
from datetime import datetime, timedelta
from celery.task import periodic_task, task
from celery.task.schedules import crontab
from django.conf import settings as dj_settings
from django.utils import timezone

def fsteps(start, stop, steps):
    step = (stop - start) / steps
    while start <= stop:
        yield start
        start += step

@periodic_task(ignore_result=True, run_every=crontab(minute='*'))
def check_alerts(**kwargs):
    """
    Iterates all current keys and fires additional tasks to check each individual
    project's alert settings.
    """
    from sentry import app
    from sentry.utils.queue import maybe_delay

    timestamp = time.time() - 60
    when = datetime.fromtimestamp(timestamp)
    if dj_settings.USE_TZ:
        when = when.replace(tzinfo=timezone.utc)

    results = app.counter.extract_counts(prefix='project', when=timestamp)['results']
    for project_id, count in results:
        maybe_delay(check_project_alerts,
            project_id=int(project_id),
            when=when,
            count=int(count),
            expires=120,
        )

@task(ignore_result=True)
def check_project_alerts(project_id, when, count, **kwargs):
    """
    Given 'when' and 'count', which should signify recent times we compare it to historical data for this project
    and if over a given threshold, create an alert.
    """
    from sentry.constants import MINUTE_NORMALIZATION
    from sentry.conf import settings
    from sentry.models import ProjectCountByMinute, ProjectOption, Alert

    try:
        threshold, min_events = ProjectOption.objects.get(project=project_id, key='project_alert_pct')
    except ProjectOption.DoesNotExist:
        threshold, min_events = settings.DEFAULT_ALERT_PROJECT_THRESHOLD

    if not threshold and min_events:
        return

    if min_events > count:
        return

    # number of 15 minute intervals to capture
    intervals = 8

    min_date = when
    max_date = when - timedelta(minutes=(intervals * MINUTE_NORMALIZATION))

    # get historical data
    data = list(ProjectCountByMinute.objects.filter(
        project=project_id,
        date__lt=min_date,
        date__gte=max_date,
    ).values_list('times_seen', flat=True))

    # Bail if we dont have enough data points
    if len(data) != intervals:
        return

    # take a weighted mean, where the oldest value is worth .7 and the newest is 1.0
    # Note: this math is completely made up, and I make no claim to know how math works
    previous = sum((k * v) for k, v in zip(data, fsteps(0.7, 1.0, intervals))) / intervals / MINUTE_NORMALIZATION

    if count / previous * 100 > threshold:
        Alert.maybe_alert(
            project_id=project_id,
            message='Rate of events per minute increased from %d to %d' % (previous, count),
        )
<commit_msg>Simplify baseline checks<commit_after>"""
sentry.tasks.check_alerts
~~~~~~~~~~~~~~~~~~~~~~~~~

- Store a sorted set per project
- Each sorted set contains the number of events seen in the interval (1 minute)
  - An additional set contains the number of unique events seen
- Every minute we iterate this sorted set (we can exploit the queue just like buffers to avoid crons)
  - We clear the results immediately to no-op any concurrent tasks that might try to run
  - The task fires off a set of subtasks that individually check each project
    - Each project's value is compared to the historic value in the last N minutes (15m for redis counters or
      a period of time using the SQL counters)
    - We only alert if an alert has not been seen on this condition in the last N minutes

Notes:

- Nydus optimizes out multiple writes/gets, so its not as expensive as it looks
- If there are not enough data points in history, alerts are never sent (as they could be false positives)

Last Notified will be tracked by using a new Alert model, which has the following properties:

- users alerted
- datetime
- type
- ident

Type and ident would be similar to how Activity works.

Alert expiration threshold MUST be > MINUTE_NORMALIZATION.

:copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from __future__ import division

import time
from datetime import datetime, timedelta
from celery.task import periodic_task, task
from celery.task.schedules import crontab
from django.conf import settings as dj_settings
from django.utils import timezone

def fsteps(start, stop, steps):
    step = (stop - start) / steps
    while start <= stop:
        yield start
        start += step

@periodic_task(ignore_result=True, run_every=crontab(minute='*'))
def check_alerts(**kwargs):
    """
    Iterates all current keys and fires additional tasks to check each individual
    project's alert settings.
    """
    from sentry import app
    from sentry.utils.queue import maybe_delay

    timestamp = time.time() - 60
    when = datetime.fromtimestamp(timestamp)
    if dj_settings.USE_TZ:
        when = when.replace(tzinfo=timezone.utc)

    results = app.counter.extract_counts(prefix='project', when=timestamp)['results']
    for project_id, count in results:
        maybe_delay(check_project_alerts,
            project_id=int(project_id),
            when=when,
            count=int(count),
            expires=120,
        )

@task(ignore_result=True)
def check_project_alerts(project_id, when, count, **kwargs):
    """
    Given 'when' and 'count', which should signify recent times we compare it to historical data for this project
    and if over a given threshold, create an alert.
    """
    from sentry.constants import MINUTE_NORMALIZATION
    from sentry.conf import settings
    from sentry.models import ProjectCountByMinute, ProjectOption, Alert

    try:
        threshold, min_events = ProjectOption.objects.get(project=project_id, key='project_alert_pct')
    except ProjectOption.DoesNotExist:
        threshold, min_events = settings.DEFAULT_ALERT_PROJECT_THRESHOLD

    if not threshold and min_events:
        return

    if min_events > count:
        return

    # number of 15 minute intervals to capture
    intervals = 4

    min_date = when
    max_date = when - timedelta(minutes=(intervals * MINUTE_NORMALIZATION))

    # get historical data
    data = list(ProjectCountByMinute.objects.filter(
        project=project_id,
        date__lt=min_date,
        date__gte=max_date,
    ).values_list('times_seen', flat=True))

    # Bail if we dont have enough data points
    if len(data) != intervals:
        return

    previous = sum(data) / intervals / MINUTE_NORMALIZATION

    if count / previous * 100 > threshold:
        Alert.maybe_alert(
            project_id=project_id,
            message='Rate of events per minute increased from %d to %d' % (previous, count),
        )
<|endoftext|>


<commit_before>import time, sys, os
import numpy as np
from scipy.linalg import LinAlgError

__all__ = ["lnlike_spec", "lnlike_phot", "chi_spec", "chi_phot", "write_log"]

def lnlike_spec(spec_mu, obs=None, spec_noise=None, f_outlier_spec=0.0, **vectors):
        """Calculate the likelihood of the spectroscopic data given the
        spectroscopic model.  Allows for the use of a gaussian process
        covariance matrix for multiplicative residuals.

        :param spec_mu:
            The mean model spectrum, in linear or logarithmic units, including
            e.g. calibration and sky emission.

        :param obs: (optional)
            A dictionary of the observational data, including the keys
            *``spectrum`` a numpy array of the observed spectrum, in linear or
             logarithmic units (same as ``spec_mu``).
            *``unc`` the uncertainty of same length as ``spectrum``
            *``mask`` optional boolean array of same length as ``spectrum``
            *``wavelength`` if using a GP, the metric that is used in the
             kernel generation, of same length as ``spectrum`` and typically
             giving the wavelength array.

        :param spec_noise: (optional)
            A NoiseModel object with the methods `compute` and `lnlikelihood`.
            If ``spec_noise`` is supplied, the `wavelength` entry in the obs
            dictionary must exist.

        :param vectors: (optional)
            A dictionary of vectors of same length as ``wavelength`` giving
            possible weghting functions for the kernels

        :returns lnlikelhood:
            The natural logarithm of the likelihood of the data given the mean
            model spectrum.
        """
        if obs['spectrum'] is None:
            return 0.0

        mask = obs.get('mask', slice(None))
        vectors['mask'] = mask
        vectors['wavelength'] = obs['wavelength']

        delta = (obs['spectrum'] - spec_mu)[mask]
        var = (obs['unc'][mask])**2

        if spec_noise is not None:
            try:
                spec_noise.compute(**vectors)
                if (f_outlier_spec == 0.0):
                    return spec_noise.lnlikelihood(delta)

                # disallow (correlated noise model + mixture model)
                assert spec_noise.Sigma.ndim == 1

                # redefine errors, scaling sigma_bad by (jittered sigma / original sigma)
                var_bad = spec_noise.Sigma * (vectors["nsigma_bad"]**2)
                var = spec_noise.Sigma
            except(LinAlgError):
                return np.nan_to_num(-np.inf)

        lnp = -0.5*( (delta**2/var) + np.log(2*np.pi*var) )
        if (f_outlier_spec == 0.0):
            return lnp.sum()
        else:
            lnp_bad = -0.5*( (delta**2/var_bad) + np.log(2*np.pi*var_bad) )
            lnp_tot = np.logaddexp(lnp + np.log(1-f_outlier_spec), lnp_bad + np.log(f_outlier_spec))

            return lnp_tot.sum()

def lnlike_phot(phot_mu, obs=None, phot_noise=None, **vectors):
    """Calculate the likelihood of the photometric data given the spectroscopic
    model.  Allows for the use of a gaussian process covariance matrix.

    :param phot_mu:
        The mean model sed, in linear flux units (i.e. maggies).

    :param obs: (optional)
        A dictionary of the observational data, including the keys
          *``maggies`` a numpy array of the observed SED, in linear flux
           units
          *``maggies_unc`` the uncertainty of same length as ``maggies``
          *``phot_mask`` optional boolean array of same length as
           ``maggies``
          *``filters`` optional list of sedpy.observate.Filter objects,
           necessary if using fixed filter groups with different gp
           amplitudes for each group.
       If not supplied then the obs dictionary given at initialization will
       be used.

    :param phot_noise: (optional)
        A ``prospect.likelihood.NoiseModel`` object with the methods
        ``compute()`` and ``lnlikelihood()``.  If not supplied a simple chi^2
        likelihood will be evaluated.

    :param vectors:
        A dictionary of possibly relevant vectors of same length as maggies
        that will be passed to the NoiseModel object for constructing weighted
        covariance matrices.

    :returns lnlikelhood:
        The natural logarithm of the likelihood of the data given the mean
        model spectrum.
    """
    if obs['maggies'] is None:
        return 0.0

    mask = obs.get('phot_mask', slice(None))
    delta = (obs['maggies'] - phot_mu)[mask]

    if phot_noise is not None:
        filternames = [f.name for f in obs['filters']]
        vectors['mask'] = mask
        vectors['filternames'] = np.array(filternames)
        try:
            phot_noise.compute(**vectors)
            return phot_noise.lnlikelihood(delta)
        except(LinAlgError):
            return np.nan_to_num(-np.inf)
    else:
        # simple noise model
        var = (obs['maggies_unc'][mask])**2
        lnp = -0.5*( (delta**2/var).sum() + np.log(2*np.pi*var).sum() )
        return lnp

def chi_phot(phot_mu, obs, **extras):
    """Return a vector of chi values, for use in non-linear least-squares
    algorithms.

    :param phot_mu:
        Model photometry, same units as the photometry in `obs`.

    :param obs:
        An observational data dictionary, with the keys ``"maggies"`` and
        ``"maggies_unc"``.  If ``"maggies"`` is None then an empty array is
        returned.

    :returns chi:
        An array of noise weighted residuals, same length as the number of
        unmasked phtometric points.
    """
    if obs['maggies'] is None:
        return np.array([])

    mask = obs.get('phot_mask', slice(None))
    delta = (obs['maggies'] - phot_mu)[mask]
    unc = obs['maggies_unc'][mask]
    chi = delta / unc
    return chi

def chi_spec(spec_mu, obs, **extras):
    """Return a vector of chi values, for use in non-linear least-squares
    algorithms.

    :param spec_mu:
        Model spectroscopy, same units as the photometry in `obs`.

    :param obs:
        An observational data dictionary, with the keys ``"spectrum"`` and
        ``"unc"``.  If ``"spectrum"`` is None then an empty array is returned.
        Optinally a ``"mask"`` boolean vector may be supplied that will be used
        to index the residual vector.

    :returns chi:
        An array of noise weighted residuals, same length as the number of
        unmasked spectroscopic points.
    """
    if obs['spectrum'] is None:
        return np.array([])
    mask = obs.get('mask', slice(None))
    delta = (obs['spectrum'] - spec_mu)[mask]
    unc = obs['unc'][mask]
    chi = delta / unc
    return chi

def write_log(theta, lnp_prior, lnp_spec, lnp_phot, d1, d2):
    """Write all sorts of documentary info for debugging.
    """
    print(theta)
    print('model calc = {0}s, lnlike calc = {1}'.format(d1, d2))
    fstring = 'lnp = {0}, lnp_spec = {1}, lnp_phot = {2}'
    values = [lnp_spec + lnp_phot + lnp_prior, lnp_spec, lnp_phot]
    print(fstring.format(*values))
<commit_msg>allow mixture model without noise model<commit_after>import time, sys, os
import numpy as np
from scipy.linalg import LinAlgError

__all__ = ["lnlike_spec", "lnlike_phot", "chi_spec", "chi_phot", "write_log"]

def lnlike_spec(spec_mu, obs=None, spec_noise=None, f_outlier_spec=0.0, **vectors):
        """Calculate the likelihood of the spectroscopic data given the
        spectroscopic model.  Allows for the use of a gaussian process
        covariance matrix for multiplicative residuals.

        :param spec_mu:
            The mean model spectrum, in linear or logarithmic units, including
            e.g. calibration and sky emission.

        :param obs: (optional)
            A dictionary of the observational data, including the keys
            *``spectrum`` a numpy array of the observed spectrum, in linear or
             logarithmic units (same as ``spec_mu``).
            *``unc`` the uncertainty of same length as ``spectrum``
            *``mask`` optional boolean array of same length as ``spectrum``
            *``wavelength`` if using a GP, the metric that is used in the
             kernel generation, of same length as ``spectrum`` and typically
             giving the wavelength array.

        :param spec_noise: (optional)
            A NoiseModel object with the methods `compute` and `lnlikelihood`.
            If ``spec_noise`` is supplied, the `wavelength` entry in the obs
            dictionary must exist.

        :param vectors: (optional)
            A dictionary of vectors of same length as ``wavelength`` giving
            possible weghting functions for the kernels

        :returns lnlikelhood:
            The natural logarithm of the likelihood of the data given the mean
            model spectrum.
        """
        if obs['spectrum'] is None:
            return 0.0

        mask = obs.get('mask', slice(None))
        vectors['mask'] = mask
        vectors['wavelength'] = obs['wavelength']

        delta = (obs['spectrum'] - spec_mu)[mask]
        var = (obs['unc'][mask])**2

        if spec_noise is not None:
            try:
                spec_noise.compute(**vectors)
                if (f_outlier_spec == 0.0):
                    return spec_noise.lnlikelihood(delta)

                # disallow (correlated noise model + mixture model)
                # and redefine errors
                assert spec_noise.Sigma.ndim == 1
                var = spec_noise.Sigma

            except(LinAlgError):
                return np.nan_to_num(-np.inf)

        lnp = -0.5*( (delta**2/var) + np.log(2*np.pi*var) )
        if (f_outlier_spec == 0.0):
            return lnp.sum()
        else:
            var_bad = var * (vectors["nsigma_bad"]**2)
            lnp_bad = -0.5*( (delta**2/var_bad) + np.log(2*np.pi*var_bad) )
            lnp_tot = np.logaddexp(lnp + np.log(1-f_outlier_spec), lnp_bad + np.log(f_outlier_spec))

            return lnp_tot.sum()

def lnlike_phot(phot_mu, obs=None, phot_noise=None, **vectors):
    """Calculate the likelihood of the photometric data given the spectroscopic
    model.  Allows for the use of a gaussian process covariance matrix.

    :param phot_mu:
        The mean model sed, in linear flux units (i.e. maggies).

    :param obs: (optional)
        A dictionary of the observational data, including the keys
          *``maggies`` a numpy array of the observed SED, in linear flux
           units
          *``maggies_unc`` the uncertainty of same length as ``maggies``
          *``phot_mask`` optional boolean array of same length as
           ``maggies``
          *``filters`` optional list of sedpy.observate.Filter objects,
           necessary if using fixed filter groups with different gp
           amplitudes for each group.
       If not supplied then the obs dictionary given at initialization will
       be used.

    :param phot_noise: (optional)
        A ``prospect.likelihood.NoiseModel`` object with the methods
        ``compute()`` and ``lnlikelihood()``.  If not supplied a simple chi^2
        likelihood will be evaluated.

    :param vectors:
        A dictionary of possibly relevant vectors of same length as maggies
        that will be passed to the NoiseModel object for constructing weighted
        covariance matrices.

    :returns lnlikelhood:
        The natural logarithm of the likelihood of the data given the mean
        model spectrum.
    """
    if obs['maggies'] is None:
        return 0.0

    mask = obs.get('phot_mask', slice(None))
    delta = (obs['maggies'] - phot_mu)[mask]

    if phot_noise is not None:
        filternames = [f.name for f in obs['filters']]
        vectors['mask'] = mask
        vectors['filternames'] = np.array(filternames)
        try:
            phot_noise.compute(**vectors)
            return phot_noise.lnlikelihood(delta)
        except(LinAlgError):
            return np.nan_to_num(-np.inf)
    else:
        # simple noise model
        var = (obs['maggies_unc'][mask])**2
        lnp = -0.5*( (delta**2/var).sum() + np.log(2*np.pi*var).sum() )
        return lnp

def chi_phot(phot_mu, obs, **extras):
    """Return a vector of chi values, for use in non-linear least-squares
    algorithms.

    :param phot_mu:
        Model photometry, same units as the photometry in `obs`.

    :param obs:
        An observational data dictionary, with the keys ``"maggies"`` and
        ``"maggies_unc"``.  If ``"maggies"`` is None then an empty array is
        returned.

    :returns chi:
        An array of noise weighted residuals, same length as the number of
        unmasked phtometric points.
    """
    if obs['maggies'] is None:
        return np.array([])

    mask = obs.get('phot_mask', slice(None))
    delta = (obs['maggies'] - phot_mu)[mask]
    unc = obs['maggies_unc'][mask]
    chi = delta / unc
    return chi

def chi_spec(spec_mu, obs, **extras):
    """Return a vector of chi values, for use in non-linear least-squares
    algorithms.

    :param spec_mu:
        Model spectroscopy, same units as the photometry in `obs`.

    :param obs:
        An observational data dictionary, with the keys ``"spectrum"`` and
        ``"unc"``.  If ``"spectrum"`` is None then an empty array is returned.
        Optinally a ``"mask"`` boolean vector may be supplied that will be used
        to index the residual vector.

    :returns chi:
        An array of noise weighted residuals, same length as the number of
        unmasked spectroscopic points.
    """
    if obs['spectrum'] is None:
        return np.array([])
    mask = obs.get('mask', slice(None))
    delta = (obs['spectrum'] - spec_mu)[mask]
    unc = obs['unc'][mask]
    chi = delta / unc
    return chi

def write_log(theta, lnp_prior, lnp_spec, lnp_phot, d1, d2):
    """Write all sorts of documentary info for debugging.
    """
    print(theta)
    print('model calc = {0}s, lnlike calc = {1}'.format(d1, d2))
    fstring = 'lnp = {0}, lnp_spec = {1}, lnp_phot = {2}'
    values = [lnp_spec + lnp_phot + lnp_prior, lnp_spec, lnp_phot]
    print(fstring.format(*values))
<|endoftext|>


<commit_before>import re
from string import hexdigits

from .chars import UNRESERVED

if isinstance(chr(0), bytes):
    _fromint = chr
else:
    _fromint = lambda i: bytes([i])

# RFC 3986 2.1: For consistency, URI producers and normalizers should
# use uppercase hexadecimal digits for all percent-encodings.
def _pctenc(byte):
    return ('%%%02X' % byte).encode('ascii')

_unreserved = frozenset(memoryview(UNRESERVED.encode('ascii')).tolist())

_encoded = {
    b'': [_fromint(i) if i in _unreserved else _pctenc(i) for i in range(256)]
}

def uriencode(uristring, safe='', encoding='utf-8', errors='strict'):
    """Encode a URI string or string component."""
    if isinstance(uristring, bytes):
        values = memoryview(uristring).tolist()
    else:
        values = memoryview(uristring.encode(encoding, errors)).tolist()
    if not isinstance(safe, bytes):
        safe = safe.encode('ascii')
    try:
        encode = _encoded[safe].__getitem__
    except KeyError:
        enclist = _encoded[b''][:]
        for i in memoryview(safe).tolist():
            enclist[i] = _fromint(i)
        _encoded[safe] = enclist
        encode = enclist.__getitem__
    if isinstance(uristring, bytes):
        return b''.join(map(encode, values))
    else:
        return b''.join(map(encode, values)).decode(encoding)

def uriencode_plus(uristring, safe='', encoding='utf-8', errors='strict'):
    """Encode a URI string or string component. Replace space with plus."""
    if isinstance(safe, bytes):
        safe = safe + b' '
    else:
        safe = safe + ' '
    encode = uriencode(uristring, safe, encoding, errors)
    if isinstance(uristring, bytes):
        return re.sub(b' ', b'+', encode)
    else:
        return re.sub(' ', '+', encode)

def uridecode(uristring, encoding='utf-8', errors='strict'):
    """Decode a URI string or string component."""
    if isinstance(uristring, bytes):
        parts = uristring.split(b'%')
    else:
        parts = uristring.encode('utf-8', errors).split(b'%')
    bary = bytearray(parts[0])
    for s in parts[1:]:
        if len(s) < 2:
            bary.append(b'%'[0])
            bary.extend(s)
        else:
            try:
                raw = int(s[:2], 16)
                if raw < 256:
                    raw_chr = chr(raw)
                    bary.append(raw)
                    bary.extend(s[2:])
                else:
                    bary.append(b'%'[0])
                    bary.extend(s)
            except:
                bary.append(b'%'[0])
                bary.extend(s)
    if isinstance(uristring, bytes):
        #return bytes(bary)
        return bary.decode(encoding, errors).encode(encoding)
    else:
        return bary.decode(encoding, errors)

def uridecode_safe(uristring, encoding='utf-8', errors='replace'):
    """Decode a URI string or string component. Prefer to be safe."""
    if isinstance(uristring, bytes):
        parts = uristring.split(b'%')
    else:
        parts = uristring.encode('utf-8', errors).split(b'%')
    bary = bytearray(parts[0])
    for s in parts[1:]:
        if len(s) < 2:
            bary.append(b'%'[0])
            bary.extend(s)
        else:
            try:
                raw = int(s[:2], 16)
                if raw < 32:
                    bary.append(b'%'[0])
                    bary.extend(s[:2].upper())
                    bary.extend(s[2:])
                elif raw < 256:
                    raw_chr = chr(raw)
                    bary.append(raw)
                    bary.extend(s[2:])
                else:
                    bary.append(b'%'[0])
                    bary.extend(s)
            except:
                bary.append(b'%'[0])
                bary.extend(s)
    if isinstance(uristring, bytes):
        return bary.decode(encoding, errors).encode(encoding)
    else:
        return bary.decode(encoding, errors)

def uridecode_plus(uristring, encoding='utf-8', errors='strict'):
    """Decode a URI string or string component. Replace plus with space."""
    if isinstance(uristring, bytes):
        uristring = re.sub(b'\+', b' ', uristring)
    else:
        uristring = re.sub('\+', ' ', uristring)
    return uridecode(uristring, encoding, errors)

def uridecode_safe_plus(uristring, encoding='utf-8', errors='replace'):
    """
    Decode a URI string or string component. Replace plus with space.
    Prefer to safe.
    """
    if isinstance(uristring, bytes):
        uristring = re.sub(b'\+', b' ', uristring)
    else:
        uristring = re.sub('\+', ' ', uristring)
    return uridecode_plus(uristring, encoding, errors)

def idnencode(domain, encoding='utf-8', errors='strict'):
    """Encode International domain string."""
    if not isinstance(domain, bytes):
        return domain.encode('idna', errors).decode(encoding)
    else:
        return domain.decode(encoding, errors).encode('idna', errors)

def idndecode(domain, encoding='utf-8', errors='strict'):
    """Decode International domain string."""
    if not isinstance(domain, bytes):
        return domain.encode('idna').decode('idna', errors)
    else:
        return domain.decode(encoding, errors).encode('idna').decode('idna', errors).encode(encoding, errors)
<commit_msg>Fix uridecode_safe_plus<commit_after>import re
from string import hexdigits

from .chars import UNRESERVED

if isinstance(chr(0), bytes):
    _fromint = chr
else:
    _fromint = lambda i: bytes([i])

# RFC 3986 2.1: For consistency, URI producers and normalizers should
# use uppercase hexadecimal digits for all percent-encodings.
def _pctenc(byte):
    return ('%%%02X' % byte).encode('ascii')

_unreserved = frozenset(memoryview(UNRESERVED.encode('ascii')).tolist())

_encoded = {
    b'': [_fromint(i) if i in _unreserved else _pctenc(i) for i in range(256)]
}

def uriencode(uristring, safe='', encoding='utf-8', errors='strict'):
    """Encode a URI string or string component."""
    if isinstance(uristring, bytes):
        values = memoryview(uristring).tolist()
    else:
        values = memoryview(uristring.encode(encoding, errors)).tolist()
    if not isinstance(safe, bytes):
        safe = safe.encode('ascii')
    try:
        encode = _encoded[safe].__getitem__
    except KeyError:
        enclist = _encoded[b''][:]
        for i in memoryview(safe).tolist():
            enclist[i] = _fromint(i)
        _encoded[safe] = enclist
        encode = enclist.__getitem__
    if isinstance(uristring, bytes):
        return b''.join(map(encode, values))
    else:
        return b''.join(map(encode, values)).decode(encoding)

def uriencode_plus(uristring, safe='', encoding='utf-8', errors='strict'):
    """Encode a URI string or string component. Replace space with plus."""
    if isinstance(safe, bytes):
        safe = safe + b' '
    else:
        safe = safe + ' '
    encode = uriencode(uristring, safe, encoding, errors)
    if isinstance(uristring, bytes):
        return re.sub(b' ', b'+', encode)
    else:
        return re.sub(' ', '+', encode)

def uridecode(uristring, encoding='utf-8', errors='strict'):
    """Decode a URI string or string component."""
    if isinstance(uristring, bytes):
        parts = uristring.split(b'%')
    else:
        parts = uristring.encode('utf-8', errors).split(b'%')
    bary = bytearray(parts[0])
    for s in parts[1:]:
        if len(s) < 2:
            bary.append(b'%'[0])
            bary.extend(s)
        else:
            try:
                raw = int(s[:2], 16)
                if raw < 256:
                    raw_chr = chr(raw)
                    bary.append(raw)
                    bary.extend(s[2:])
                else:
                    bary.append(b'%'[0])
                    bary.extend(s)
            except:
                bary.append(b'%'[0])
                bary.extend(s)
    if isinstance(uristring, bytes):
        #return bytes(bary)
        return bary.decode(encoding, errors).encode(encoding)
    else:
        return bary.decode(encoding, errors)

def uridecode_safe(uristring, encoding='utf-8', errors='replace'):
    """Decode a URI string or string component. Prefer to be safe."""
    if isinstance(uristring, bytes):
        parts = uristring.split(b'%')
    else:
        parts = uristring.encode('utf-8', errors).split(b'%')
    bary = bytearray(parts[0])
    for s in parts[1:]:
        if len(s) < 2:
            bary.append(b'%'[0])
            bary.extend(s)
        else:
            try:
                raw = int(s[:2], 16)
                if raw < 32:
                    bary.append(b'%'[0])
                    bary.extend(s[:2].upper())
                    bary.extend(s[2:])
                elif raw < 256:
                    raw_chr = chr(raw)
                    bary.append(raw)
                    bary.extend(s[2:])
                else:
                    bary.append(b'%'[0])
                    bary.extend(s)
            except:
                bary.append(b'%'[0])
                bary.extend(s)
    if isinstance(uristring, bytes):
        return bary.decode(encoding, errors).encode(encoding)
    else:
        return bary.decode(encoding, errors)

def uridecode_plus(uristring, encoding='utf-8', errors='strict'):
    """Decode a URI string or string component. Replace plus with space."""
    if isinstance(uristring, bytes):
        uristring = re.sub(b'\+', b' ', uristring)
    else:
        uristring = re.sub('\+', ' ', uristring)
    return uridecode(uristring, encoding, errors)

def uridecode_safe_plus(uristring, encoding='utf-8', errors='replace'):
    """
    Decode a URI string or string component. Replace plus with space.
    Prefer to safe.
    """
    if isinstance(uristring, bytes):
        uristring = re.sub(b'\+', b' ', uristring)
    else:
        uristring = re.sub('\+', ' ', uristring)
    return uridecode_safe(uristring, encoding, errors)

def idnencode(domain, encoding='utf-8', errors='strict'):
    """Encode International domain string."""
    if not isinstance(domain, bytes):
        return domain.encode('idna', errors).decode(encoding)
    else:
        return domain.decode(encoding, errors).encode('idna', errors)

def idndecode(domain, encoding='utf-8', errors='strict'):
    """Decode International domain string."""
    if not isinstance(domain, bytes):
        return domain.encode('idna').decode('idna', errors)
    else:
        return domain.decode(encoding, errors).encode('idna').decode('idna', errors).encode(encoding, errors)
<|endoftext|>


<commit_before>import re

from Message import Field, Union, Message, Header, List, Struct
from message_stream import MessageStream
from primitives import Length

class _Template(object):

    def __init__(self, name, parent):
        self.parent = parent
        self._fields = []
        self.name = name

    def _pretty_print_fields(self, fields):
        return ', '.join('%s:%s' % (key, value) for key, value in fields.items())

    def add(self, field):
        if self._get_field(field.name):
            raise AssertionError('Duplicate field %s in %s' % (field.name, self.name))
        if field.has_length and not field.length.static:
            if not self._get_field_recursive(field.length.field):
                raise AssertionError('Length field %s unknown' % field.length.field)
        self._fields.append(field)

    def _get_field(self, field_name):
        for elem in self._fields:
            if elem.name == field_name:
                return elem
        return None

    def _get_field_recursive(self, field_name):
        return self._get_field(field_name) or self.parent and self.parent._get_field_recursive(field_name)

    def _check_params_empty(self, message_fields, name):
        message_fields.pop('*', None)
        if message_fields:
            raise AssertionError('Unknown fields in %s %s' %
                                 (name, self._pretty_print_fields(message_fields)))

    def _encode_fields(self, struct, params, little_endian=False):
        for field in self._fields:
            # TODO: clean away this ugly hack that makes it possible to skip PDU
            # (now it is a 0 length place holder in header)
            encoded = field.encode(params, struct, little_endian=little_endian)
            if encoded:
                struct[field.name] = encoded
        self._check_params_empty(params, self.name)

    def decode(self, data, parent=None, name=None, little_endian=False):
        message = self._get_struct(name)
        data_index = 0
        for field in self._fields:
            message[field.name] = field.decode(data[data_index:], message, little_endian=little_endian)
            data_index += len(message[field.name])
        return message

    def validate(self, message, message_fields):
        errors = []
        for field in self._fields:
            errors += field.validate(message, message_fields)
        self._check_params_empty(message_fields, self.name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        result = {'*': params['*']} if '*' in params else {}
        name = name if name else self.name
        for key in params.keys():
            prefix, _, ending = key.partition('.')
            if prefix == name:
                result[ending] = params.pop(key)
        return result

class Protocol(_Template):

    def __init__(self, name):
        _Template.__init__(self, name, None)

    def header_length(self):
        length = 0
        for field in self._fields:
            if not field.length.static:
                return length
            length += field.length.value
        return length

    def encode(self, message, header_params, little_endian=False):
        header_params = header_params.copy()
        self._insert_length_to_header_parameters(header_params, message)
        header = Header(self.name)
        self._encode_fields(header, header_params, little_endian=little_endian)
        return header

    def _insert_length_to_header_parameters(self, header_params, message):
        pdu_field = self._get_pdu_field()
        pdu_length = len(message._raw)
        header_params[pdu_field.length.field] = pdu_field.length.solve_parameter(pdu_length)

    def _get_pdu_field(self):
        for field in self._fields:
            if field.type == 'pdu':
                return field
        return None

    # TODO: fields after the pdu
    def read(self, stream, timeout=None):
        data = stream.read(self.header_length(), timeout=timeout)
        data_index = 0
        field_index = 0
        header = Header(self.name)
        while len(data) > data_index:
            field = self._fields[field_index]
            header[field.name] = Field(field.type, field.name, data[data_index:data_index+field.length.value])
            data_index += field.length.value
            field_index +=1
        pdu_field = self._get_pdu_field()
        length_param = header[pdu_field.length.field].int
        pdu_bytes = stream.read(pdu_field.length.calc_value(length_param))
        return (header, pdu_bytes)

    def get_message_stream(self, buffered_stream):
        return MessageStream(buffered_stream, self)

class MessageTemplate(_Template):

    type = 'Message'

    def __init__(self, message_name, protocol, header_params):
        _Template.__init__(self, message_name, None)
        self._protocol = protocol
        self.header_parameters = header_params

    def encode(self, message_params, header_params, little_endian=False):
        message_params = message_params.copy()
        msg = Message(self.name)
        self._encode_fields(msg, message_params, little_endian=little_endian)
        if self._protocol:
            # TODO: little endian support for protocol header
            msg._add_header(self._protocol.encode(msg, self._headers(header_params)))
        return msg

    def _headers(self, header_params):
        result = {}
        result.update(self.header_parameters)
        result.update(header_params)
        return result

    def _get_struct(self, name):
        return Message(self.name)

class StructTemplate(_Template):

    has_length = False

    def __init__(self, type, name, parent, parameters=None):
        self._parameters = parameters if parameters else {}
        self.type = type
        _Template.__init__(self, name, parent)

    def get_static_length(self):
        return sum(field.get_static_length() for field in self._fields)

    def encode(self, message_params, parent=None, name=None, little_endian=False):
        struct = self._get_struct(name)
        self._add_struct_params(message_params)
        self._encode_fields(struct, self._get_params_sub_tree(message_params, name), little_endian=little_endian)
        return struct

    def _get_struct(self, name):
        return Struct(name if name else self.name, self.type)

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        message = parent[name]
        return _Template.validate(self, message, self._get_params_sub_tree(message_fields, name))

    def _add_struct_params(self, params):
        for key in self._parameters.keys():
            params[key] = self._parameters.pop(key) if key not in params else params[key]

class UnionTemplate(_Template):

    has_length = False

    def __init__(self, type, name, parent):
        self.type = type
        _Template.__init__(self, name, parent)

    def add(self, field):
        field.get_static_length()
        self._fields.append(field)

    def get_static_length(self):
        return max(field.get_static_length() for field in self._fields)

    def decode(self, data, parent=None, name=None, little_endian=False):
        union = Union(name if name else self.name, self.get_static_length())
        for field in self._fields:
            union[field.name] = field.decode(data, union, little_endian=little_endian)
        return union

    def encode(self, union_params, parent=None, name=None, little_endian=False):
        name = name if name else self.name
        union = Union(name, self.get_static_length())
        if name not in union_params:
            raise AssertionError('Value not chosen for union %s' % name)
        chosen_one = union_params[name]
        for field in self._fields:
            if field.name == chosen_one:
                union[field.name] = field.encode(self._get_params_sub_tree(union_params, name),
                                                 union,
                                                 little_endian=little_endian)
                return union
        raise Exception('Unknown union field %s' % chosen_one)

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        message = parent[name]
        return _Template.validate(self, message, self._get_params_sub_tree(message_fields, name))

class ListTemplate(_Template):

    param_pattern = re.compile(r'(.*?)\[(.*?)\](.*)')
    has_length = True
    type = 'List'

    def __init__(self, length, name, parent):
        self.length = Length(length)
        _Template.__init__(self, name, parent)

    def get_static_length(self):
        return self.length.value * self.field.get_static_length()

    def encode(self, message_params, parent, name=None, little_endian=False):
        name = name if name else self.name
        params_subtree = self._get_params_sub_tree(message_params, name)
        list = self._get_struct(name)
        for index in range(0, self.length.decode(parent)):
            list[str(index)] = self.field.encode(params_subtree,
                                                 parent,
                                                 name=str(index),
                                                 little_endian=little_endian)
        self._check_params_empty(params_subtree, name)
        return list

    @property
    def field(self):
        return self._fields[0]

    def _get_struct(self, name=None):
        return List(name if name else self.name, self.field.type)

    def decode(self, data, parent, name=None, little_endian=False):
        name = name if name else self.name
        message = self._get_struct(name)
        data_index = 0
        for index in range(0, self.length.decode(parent)):
            message[str(index)] = self.field.decode(data[data_index:], message, name=str(index), little_endian=little_endian)
            data_index += len(message[index])
        return message

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        params_subtree = self._get_params_sub_tree(message_fields, name)
        list = parent[name]
        errors = []
        for index in range(0, self.length.decode(parent)):
            errors += self.field.validate(list, params_subtree, name=str(index))
        self._check_params_empty(params_subtree, name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        # TODO: Test for * syntax in array subfields
        result = {'*': params['*']} if '*' in params else {}
        name = name if name else self.name
        for key in params.keys():
            match = self.param_pattern.match(key)
            if match:
                prefix, child_name, ending = match.groups()
                if prefix == name or prefix == '*':
                    result[child_name + ending] =  params.pop(key)
        return result
<commit_msg>replace list with oredered dict in containers<commit_after>import re

from Message import Field, Union, Message, Header, List, Struct
from message_stream import MessageStream
from primitives import Length
from OrderedDict import OrderedDict

class _Template(object):

    def __init__(self, name, parent):
        self.parent = parent
        self._fields = OrderedDict()
        self.name = name

    def _pretty_print_fields(self, fields):
        return ', '.join('%s:%s' % (key, value) for key, value in fields.items())

    def add(self, field):
        if self._get_field(field.name):
            raise AssertionError('Duplicate field %s in %s' % (field.name, self.name))
        if field.has_length and not field.length.static:
            if not self._get_field_recursive(field.length.field):
                raise AssertionError('Length field %s unknown' % field.length.field)
        self._fields[field.name] = field

    def _get_field(self, field_name):
        return self._fields.get(field_name)

    def _get_field_recursive(self, field_name):
        return self._get_field(field_name) or self.parent and self.parent._get_field_recursive(field_name)

    def _check_params_empty(self, message_fields, name):
        message_fields.pop('*', None)
        if message_fields:
            raise AssertionError('Unknown fields in %s %s' %
                                 (name, self._pretty_print_fields(message_fields)))

    def _encode_fields(self, struct, params, little_endian=False):
        for field in self._fields.values():
            # TODO: clean away this ugly hack that makes it possible to skip PDU
            # (now it is a 0 length place holder in header)
            encoded = field.encode(params, struct, little_endian=little_endian)
            if encoded:
                struct[field.name] = encoded
        self._check_params_empty(params, self.name)

    def decode(self, data, parent=None, name=None, little_endian=False):
        message = self._get_struct(name)
        data_index = 0
        for field in self._fields.values():
            message[field.name] = field.decode(data[data_index:], message, little_endian=little_endian)
            data_index += len(message[field.name])
        return message

    def validate(self, message, message_fields):
        errors = []
        for field in self._fields.values():
            errors += field.validate(message, message_fields)
        self._check_params_empty(message_fields, self.name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        result = {'*': params['*']} if '*' in params else {}
        name = name if name else self.name
        for key in params.keys():
            prefix, _, ending = key.partition('.')
            if prefix == name:
                result[ending] = params.pop(key)
        return result

class Protocol(_Template):

    def __init__(self, name):
        _Template.__init__(self, name, None)

    def header_length(self):
        length = 0
        for field in self._fields.values():
            if not field.length.static:
                return length
            length += field.length.value
        return length

    def encode(self, message, header_params, little_endian=False):
        header_params = header_params.copy()
        self._insert_length_to_header_parameters(header_params, message)
        header = Header(self.name)
        self._encode_fields(header, header_params, little_endian=little_endian)
        return header

    def _insert_length_to_header_parameters(self, header_params, message):
        pdu_field = self._get_pdu_field()
        pdu_length = len(message._raw)
        header_params[pdu_field.length.field] = pdu_field.length.solve_parameter(pdu_length)

    def _get_pdu_field(self):
        for field in self._fields.values():
            if field.type == 'pdu':
                return field
        return None

    # TODO: fields after the pdu
    def read(self, stream, timeout=None):
        data = stream.read(self.header_length(), timeout=timeout)
        data_index = 0
        field_index = 0
        header = Header(self.name)
        values = self._fields.values()
        while len(data) > data_index:
            field = values[field_index]
            header[field.name] = Field(field.type, field.name, data[data_index:data_index+field.length.value])
            data_index += field.length.value
            field_index +=1
        pdu_field = self._get_pdu_field()
        length_param = header[pdu_field.length.field].int
        pdu_bytes = stream.read(pdu_field.length.calc_value(length_param))
        return (header, pdu_bytes)

    def get_message_stream(self, buffered_stream):
        return MessageStream(buffered_stream, self)

class MessageTemplate(_Template):

    type = 'Message'

    def __init__(self, message_name, protocol, header_params):
        _Template.__init__(self, message_name, None)
        self._protocol = protocol
        self.header_parameters = header_params

    def encode(self, message_params, header_params, little_endian=False):
        message_params = message_params.copy()
        msg = Message(self.name)
        self._encode_fields(msg, message_params, little_endian=little_endian)
        if self._protocol:
            # TODO: little endian support for protocol header
            msg._add_header(self._protocol.encode(msg, self._headers(header_params)))
        return msg

    def _headers(self, header_params):
        result = {}
        result.update(self.header_parameters)
        result.update(header_params)
        return result

    def _get_struct(self, name):
        return Message(self.name)

class StructTemplate(_Template):

    has_length = False

    def __init__(self, type, name, parent, parameters=None):
        self._parameters = parameters if parameters else {}
        self.type = type
        _Template.__init__(self, name, parent)

    def get_static_length(self):
        return sum(field.get_static_length() for field in self._fields.values())

    def encode(self, message_params, parent=None, name=None, little_endian=False):
        struct = self._get_struct(name)
        self._add_struct_params(message_params)
        self._encode_fields(struct, self._get_params_sub_tree(message_params, name), little_endian=little_endian)
        return struct

    def _get_struct(self, name):
        return Struct(name if name else self.name, self.type)

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        message = parent[name]
        return _Template.validate(self, message, self._get_params_sub_tree(message_fields, name))

    def _add_struct_params(self, params):
        for key in self._parameters.keys():
            params[key] = self._parameters.pop(key) if key not in params else params[key]

class UnionTemplate(_Template):

    has_length = False

    def __init__(self, type, name, parent):
        self.type = type
        _Template.__init__(self, name, parent)

    def add(self, field):
        field.get_static_length()
        self._fields[field.name] = field

    def get_static_length(self):
        return max(field.get_static_length() for field in self._fields.values())

    def decode(self, data, parent=None, name=None, little_endian=False):
        union = Union(name if name else self.name, self.get_static_length())
        for field in self._fields.values():
            union[field.name] = field.decode(data, union, little_endian=little_endian)
        return union

    def encode(self, union_params, parent=None, name=None, little_endian=False):
        name = name if name else self.name
        union = Union(name, self.get_static_length())
        if name not in union_params:
            raise AssertionError('Value not chosen for union %s' % name)
        chosen_one = union_params[name]
        if chosen_one not in self._fields:
            raise Exception('Unknown union field %s' % chosen_one)
        field = self._fields[chosen_one]
        union[field.name] = field.encode(self._get_params_sub_tree(union_params, name),
                                         union,
                                         little_endian=little_endian)
        return union

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        message = parent[name]
        return _Template.validate(self, message, self._get_params_sub_tree(message_fields, name))

#TODO: check that only one field is added to list
#TODO: list field could be overriden
class ListTemplate(_Template):

    param_pattern = re.compile(r'(.*?)\[(.*?)\](.*)')
    has_length = True
    type = 'List'

    def __init__(self, length, name, parent):
        self.length = Length(length)
        _Template.__init__(self, name, parent)

    def get_static_length(self):
        return self.length.value * self.field.get_static_length()

    def encode(self, message_params, parent, name=None, little_endian=False):
        name = name if name else self.name
        params_subtree = self._get_params_sub_tree(message_params, name)
        list = self._get_struct(name)
        for index in range(0, self.length.decode(parent)):
            list[str(index)] = self.field.encode(params_subtree,
                                                 parent,
                                                 name=str(index),
                                                 little_endian=little_endian)
        self._check_params_empty(params_subtree, name)
        return list

    @property
    def field(self):
        return self._fields.values()[0]

    def _get_struct(self, name=None):
        return List(name if name else self.name, self.field.type)

    def decode(self, data, parent, name=None, little_endian=False):
        name = name if name else self.name
        message = self._get_struct(name)
        data_index = 0
        for index in range(0, self.length.decode(parent)):
            message[str(index)] = self.field.decode(data[data_index:], message, name=str(index), little_endian=little_endian)
            data_index += len(message[index])
        return message

    def validate(self, parent, message_fields, name=None):
        name = name if name else self.name
        params_subtree = self._get_params_sub_tree(message_fields, name)
        list = parent[name]
        errors = []
        for index in range(0, self.length.decode(parent)):
            errors += self.field.validate(list, params_subtree, name=str(index))
        self._check_params_empty(params_subtree, name)
        return errors

    def _get_params_sub_tree(self, params, name=None):
        # TODO: Test for * syntax in array subfields
        result = {'*': params['*']} if '*' in params else {}
        name = name if name else self.name
        for key in params.keys():
            match = self.param_pattern.match(key)
            if match:
                prefix, child_name, ending = match.groups()
                if prefix == name or prefix == '*':
                    result[child_name + ending] =  params.pop(key)
        return result
<|endoftext|>


<commit_before>from .DisplayBib import DisplayBib

_template = """
<!DOCTYPE html>

    <meta charset="UTF-8">
    <title>sdb: TITLE-DIR</title>

Articles under “TITLE-DIR”

           BIB-BODY

"""

class HTMLBib(DisplayBib):
    def tag(self, t):
        return '{}'.format(t)

    def tags_fmt(self):
        if self.entry.tags:
            return '
Tags: ' + ', '.join(self.tag(t) for t in self.entry.tags) + ""

    def attachment(self, a):
        idx = self.entry.fileLabels.index(a)
        key = self.entry.key()

        ext = self.entry.files[idx].split('.')
        if len(ext)>1:
            ext = ext[-1]
        else:
            ext = 'dat'

        fname = "{}-{:04d}.{}".format(key, idx, ext)

        return '{}'.format(fname, a)

    def attachments_fmt(self):
        if len(self.entry.fileLabels) > 1:
            return '
Attachments: ' + ', '.join(self.attachment(a) for a in self.entry.fileLabels[1:]) + ''

    def year(self):
        y = super().year()
        return "" + y + ""

    def booktitle(self):
        t = super().booktitle()
        if t:
            return "" + t + ""

    def volume(self):
        v = super().volume()
        if v:
            return "" + v + ""

    def journal(self):
        j = super().journal()
        if j:
            return "" + j + ""

    def doi(self):
        d = super().doi()
        return 'doi:{d}'.format(d=d)

    def key_fmt(self):
        return '
' + self.key() + "
"

    def bib_fmt(self):
        return ('
{ref}
' +
                ' ' +
                ' ' +
                ' ' +
                '
'
               ).format(ref=super().bib_fmt(), key=super().key())

def htmlBibliography(works, directory):
    return _template.replace('BIB-BODY', '
'.join(w.format(HTMLBib) for w in works)).replace('TITLE-DIR', directory)
<commit_msg>links in new tab<commit_after>from .DisplayBib import DisplayBib

_template = """
<!DOCTYPE html>

    <meta charset="UTF-8">
    <title>sdb: TITLE-DIR</title>

Articles under “TITLE-DIR”

           BIB-BODY

"""

class HTMLBib(DisplayBib):
    def tag(self, t):
        return '{}'.format(t)

    def tags_fmt(self):
        if self.entry.tags:
            return '
Tags: ' + ', '.join(self.tag(t) for t in self.entry.tags) + ""

    def attachment(self, a):
        idx = self.entry.fileLabels.index(a)
        key = self.entry.key()

        ext = self.entry.files[idx].split('.')
        if len(ext)>1:
            ext = ext[-1]
        else:
            ext = 'dat'

        fname = "{}-{:04d}.{}".format(key, idx, ext)

        return '{}'.format(fname, a)

    def attachments_fmt(self):
        if len(self.entry.fileLabels) > 1:
            return '
Attachments: ' + ', '.join(self.attachment(a) for a in self.entry.fileLabels[1:]) + ''

    def year(self):
        y = super().year()
        return "" + y + ""

    def booktitle(self):
        t = super().booktitle()
        if t:
            return "" + t + ""

    def volume(self):
        v = super().volume()
        if v:
            return "" + v + ""

    def journal(self):
        j = super().journal()
        if j:
            return "" + j + ""

    def doi(self):
        d = super().doi()
        return 'doi:{d}'.format(d=d)

    def key_fmt(self):
        return '
' + self.key() + "
"

    def bib_fmt(self):
        return ('
{ref}
' +
                ' ' +
                ' ' +
                ' ' +
                '
'
               ).format(ref=super().bib_fmt(), key=super().key())

def htmlBibliography(works, directory):
    return _template.replace('BIB-BODY', '
'.join(w.format(HTMLBib) for w in works)).replace('TITLE-DIR', directory)
<|endoftext|>


<commit_before># Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors: Jan Safranek <jsafrane@redhat.com>
# -*- coding: utf-8 -*-

import ConfigParser
import os
import socket

class StorageConfiguration(ConfigParser.SafeConfigParser):

    CONFIG_FILE = '/var/lib/openlmi-storage/storage.ini'

    defaults = {
        'namespace' : 'root/cimv2',
        'systemclassname' : 'Linux_ComputerSystem'
    }
    def __init__(self, *args, **kwargs):
        ConfigParser.SafeConfigParser.__init__(self, defaults=self.defaults, *args, **kwargs)

    def load(self):
        """
            Load configuration from CONFIG_FILE. Create default, if the file
            does not exist.
        """
        if os.path.isfile(self.CONFIG_FILE):
            with open(self.CONFIG_FILE, 'wb') as configfile:
                self.write(configfile)
        if not self.has_section('common'):
            self.add_section('common')

    def save(self):
        """
            Save configuration to CONFIG_FILE. Create a directory for it if it
            does not exist.
        """
        d = os.path.dirname(self.CONFIG_FILE)
        if not os.path.isdir(d):
            os.mkdir(d)

        with open(self.CONFIG_FILE, 'wb') as configfile:
            self.write(configfile)

    def getNamespace(self):
        return self.get('common', 'namespace')
    namespace = property(getNamespace)

    def getSystemClassName(self):
        return self.get('common', 'systemclassname')
    systemClassName = property(getSystemClassName)

    def getSystemName(self):
        return socket.getfqdn()
    systemName = property(getSystemName)
<commit_msg>Fixed the StorageConfiguration after some testing.<commit_after># Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors: Jan Safranek <jsafrane@redhat.com>
# -*- coding: utf-8 -*-

import ConfigParser
import os
import socket

class StorageConfiguration(object):

    CONFIG_PATH = '/etc/openlmi/storage/'
    CONFIG_FILE = CONFIG_PATH + 'storage.ini'

    PERSISTENT_PATH = '/var/lib/openlmi-storage/'
    SETTINGS_DIR = 'settings/'

    defaults = {
        'namespace' : 'root/cimv2',
        'systemclassname' : 'Linux_ComputerSystem'
    }

    def __init__(self):
        self.config = ConfigParser.SafeConfigParser(defaults=self.defaults)
        self.load()

    def load(self):
        """
            Load configuration from CONFIG_FILE. The file does not need to
            exist.
        """
        self.config.read(self.CONFIG_FILE)
        if not self.config.has_section('common'):
            self.config.add_section('common')

    def getNamespace(self):
        return self.config.get('common', 'namespace')
    namespace = property(getNamespace)

    def getSystemClassName(self):
        return self.config.get('common', 'systemclassname')
    systemClassName = property(getSystemClassName)

    def getSystemName(self):
        return socket.getfqdn()
    systemName = property(getSystemName)
<|endoftext|>


<commit_before>from scapy.all import *

packetCount = 0
message = False
messageend = False

def customAction(packet):
	global packetCount
	global message
	global messageend
	packetCount += 1
	print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		messageend = False
	if(suffixnum == 23130):
		messageend = True
	if(messageend == True and suffixnum == 23130):
		message = False
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message):
		return "! " + character + " !"

sniff(filter="port 137", prn=customAction)
<commit_msg>update<commit_after>from scapy.all import *

packetCount = 0
message = False
messageend = False

def customAction(packet):
	global packetCount
	global message
	global messageend
	packetCount += 1
	print "Packet #" + str(packetCount) + ": " +packet[0][1].src + "==>" + packet[0][1].dst + ": " + str(packet[0].SUFFIX)
	suffixnum = packet[0].SUFFIX
	if(suffixnum == 23130):
		message = True
		messageend = False
	if(suffixnum == 23130):
		messageend = True
	if(messageend == True and suffixnum == 23130):
		message = False
	asciivalue = hex(suffixnum)[4:]
	asciivalue = int(asciivalue, 16)
	character = chr(asciivalue)
	if(message == True):
		return "! " + character + " !"

sniff(filter="port 137", prn=customAction)
<|endoftext|>


<commit_before># Copyright 2022 The Scenic Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main file for Scenic."""

from absl import flags
from clu import metric_writers
import jax
import jax.numpy as jnp
import ml_collections
from scenic import app
from scenic.model_lib import models
from scenic.train_lib import train_utils
from scenic.train_lib import trainers

FLAGS = flags.FLAGS

def main(rng: jnp.ndarray, config: ml_collections.ConfigDict, workdir: str,
         writer: metric_writers.MetricWriter):
  """Main function for the Scenic."""
  model_cls = models.get_model_cls(config.model_name)
  data_rng, rng = jax.random.split(rng)
  dataset = train_utils.get_dataset(
      config, data_rng, dataset_service_address=FLAGS.dataset_service_address)

  trainers.get_trainer(config.trainer_name)(
      rng=rng,
      config=config,
      model_cls=model_cls,
      dataset=dataset,
      workdir=workdir,
      writer=writer)

if __name__ == '__main__':
  app.run(main=main)
<commit_msg>Add nn.enable_named_call() to main.<commit_after># Copyright 2022 The Scenic Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Main file for Scenic."""

from absl import flags
from clu import metric_writers
import flax.linen as nn
import jax
import jax.numpy as jnp
import ml_collections
from scenic import app
from scenic.model_lib import models
from scenic.train_lib import train_utils
from scenic.train_lib import trainers

FLAGS = flags.FLAGS

def main(rng: jnp.ndarray, config: ml_collections.ConfigDict, workdir: str,
         writer: metric_writers.MetricWriter):
  """Main function for the Scenic."""

  # Enable wrapping of all module calls in a named_call for easier profiling:
  nn.enable_named_call()

  model_cls = models.get_model_cls(config.model_name)
  data_rng, rng = jax.random.split(rng)
  dataset = train_utils.get_dataset(
      config, data_rng, dataset_service_address=FLAGS.dataset_service_address)

  trainers.get_trainer(config.trainer_name)(
      rng=rng,
      config=config,
      model_cls=model_cls,
      dataset=dataset,
      workdir=workdir,
      writer=writer)

if __name__ == '__main__':
  app.run(main=main)
<|endoftext|>