<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import pandas as pd
import os.path
import logging
from nltk.corpus import stopwords
from bs4 import BeautifulSoup
import sys
default_stdout = sys.stdout
default_stderr = sys.stderr
reload(sys)
sys.setdefaultencoding('utf-8')
sys.stdout = default_stdout
sys.stderr = default_stderr

logger = logging.getLogger(__name__)

def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    sys.stdout.write('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix))
    sys.stdout.flush()

class SentenceParser:

    regex_str = [
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs

    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
    ]

    def __init__(self, loggingLevel = 20):
        self.data = None
        logging.basicConfig(level=loggingLevel)
        pass

    def readfile(self, filepath, filetype, encod ='ISO-8859-1', header =None):
        logger.info('Start reading File')
        if not os.path.isfile(filepath):
            logger.error("File Not Exist!")
            sys.exit()
        if filetype == 'csv':
            df = pd.read_csv(filepath, encoding=encod, header =header)
        elif filetype == 'json':
            df = pd.read_json(filepath, encoding=encod, lines=True)
        elif filetype == 'xlsx':
            df = pd.read_excel(filepath, encoding=encod, header =header)
        else:
            logger.error("Extension Type not Accepted!")
            sys.exit()

        logger.debug(df)
        self.data = df

    def importdata(self, data):
        logger.info('Import DataFrame')
        if isinstance(data, pd.core.frame.DataFrame):
            self.data = data
        else:
            logger.error("Data Type not Accepted! Please use pandas.core.frame.DataFrame")
            sys.exit()

    def dfmerge(self, columns, name):
        logger.info('Merge headers %s to %s', str(columns), name)
        self.data[name] = ''
        for header in columns:
            self.data[name] += ' ' + self.data[header]

    def get_all_headers(self):
        return list(self.data.columns.values)

    def get_column(self,column):
        return self.data[column].values.tolist()

    def processtext(self, column, removeSymbol = True, remove_stopwords=False):
        logger.info("Start Data Cleaning...")
        self.data[column] = self.data[column].str.replace(r'[\n\r\t]+', ' ')
        # Remove URLs
        self.data[column] = self.data[column].str.replace(self.regex_str[3],' ')
        tempcol = self.data[column].values.tolist()
        stops = set(stopwords.words("english"))
        # This part takes a lot of times
        printProgressBar(0, len(tempcol), prefix='Progress:', suffix='Complete', length=50)
        for i in range(len(tempcol)):
            row = BeautifulSoup(tempcol[i],'html.parser').get_text()
            if removeSymbol:
                row = re.sub('[^a-zA-Z0-9]', ' ', row)
            words = row.split()
            if remove_stopwords:
                words = [w for w in words if not w in stops and not w.replace('.', '', 1).isdigit()]
            row = ' '.join(words)
            tempcol[i] = row.lower()
            printProgressBar(i, len(tempcol), prefix='Progress:', suffix='Complete', length=50)
        print "\n"
        return tempcol

if __name__ == '__main__':
    SP = SentenceParser(10)
    SP.readfile('./nvbugs.json','json')
    SP.importdata(SP.data)
    SP.dfmerge(['Module','Description','Synopsis'],'X')
    # print SP.processtext('X', True, False)[0]
    text = SP.processtext('X', True, True)[0]
    print text
<commit_msg>Add Counting Method and Get top method<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import pandas as pd
import os.path
import logging
from sklearn.feature_extraction.text import CountVectorizer
from nltk.corpus import stopwords
from bs4 import BeautifulSoup
import sys
default_stdout = sys.stdout
default_stderr = sys.stderr
reload(sys)
sys.setdefaultencoding('utf-8')
sys.stdout = default_stdout
sys.stderr = default_stderr

logger = logging.getLogger(__name__)

def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    sys.stdout.write('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix))
    sys.stdout.flush()

class SentenceParser:

    regex_str = [
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs

    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
    ]

    def __init__(self, loggingLevel = 20):
        self.data = None
        logging.basicConfig(level=loggingLevel)
        pass

    def readfile(self, filepath, filetype, encod ='ISO-8859-1', header =None):
        logger.info('Start reading File')
        if not os.path.isfile(filepath):
            logger.error("File Not Exist!")
            sys.exit()
        if filetype == 'csv':
            df = pd.read_csv(filepath, encoding=encod, header =header)
        elif filetype == 'json':
            df = pd.read_json(filepath, encoding=encod, lines=True)
        elif filetype == 'xlsx':
            df = pd.read_excel(filepath, encoding=encod, header =header)
        else:
            logger.error("Extension Type not Accepted!")
            sys.exit()

        logger.debug(df)
        self.data = df

    def importdata(self, data):
        logger.info('Import DataFrame')
        if isinstance(data, pd.core.frame.DataFrame):
            self.data = data
        else:
            logger.error("Data Type not Accepted! Please use pandas.core.frame.DataFrame")
            sys.exit()

    def dfmerge(self, columns, name):
        logger.info('Merge headers %s to %s', str(columns), name)
        self.data[name] = ''
        for header in columns:
            self.data[name] += ' ' + self.data[header]

    def get_all_headers(self):
        return list(self.data.columns.values)

    def get_column(self,column):
        return self.data[column].values.tolist()

    def processtext(self, column, removeSymbol = True, remove_stopwords=False):
        logger.info("Start Data Cleaning...")
        self.data[column] = self.data[column].str.replace(r'[\n\r\t]+', ' ')
        # Remove URLs
        self.data[column] = self.data[column].str.replace(self.regex_str[3],' ')
        tempcol = self.data[column].values.tolist()
        stops = set(stopwords.words("english"))
        # This part takes a lot of times
        printProgressBar(0, len(tempcol), prefix='Progress:', suffix='Complete', length=50)
        for i in range(len(tempcol)):
            row = BeautifulSoup(tempcol[i],'html.parser').get_text()
            if removeSymbol:
                row = re.sub('[^a-zA-Z0-9]', ' ', row)
            words = row.split()
            if remove_stopwords:
                words = [w for w in words if not w in stops and not w.replace('.', '', 1).isdigit()]
            row = ' '.join(words)
            tempcol[i] = row.lower()
            printProgressBar(i, len(tempcol), prefix='Progress:', suffix='Complete', length=50)
        print "\n"
        return tempcol

    def create_vectorizer(self, text, max_features = 1000):
    	logger.info("Creating Counting Vectorizer...")
    	self.vectorizer = CountVectorizer(analyzer = "word",   \
                             tokenizer = None,    \
                             preprocessor = None, \
                             stop_words = None,   \
                             max_features = max_features)

    	data_vector = self.vectorizer.fit_transform(text)
    	data_vector = data_vector.toarray()
    	vocab = self.vectorizer.get_feature_names()
    	self.data_df = pd.DataFrame(data=data_vector, columns=vocab)
    	return self.data_df

    def get_top(self):
    	return self.data_df.sum().sort_values(ascending=False)

if __name__ == '__main__':
    SP = SentenceParser(10)
    SP.readfile('../NVIDIA_TEMP/dataset/nvbugs.json','json')
    SP.importdata(SP.data)
    SP.dfmerge(['Module','Description','Synopsis'],'X')
    # print SP.processtext('X', True, False)[0]
    text = SP.processtext('X', True, True)
    print SP.create_vectorizer(text)
    print SP.get_top()[0:20]
<|endoftext|>


<commit_before># Copyright (c) 2016 Intel Corporation.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime

from oslo_utils import uuidutils

from neutron.objects import quota
from neutron.tests import base as test_base
from neutron.tests.unit.objects import test_base as obj_test_base
from neutron.tests.unit import testlib_api

class ResourceDeltaObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.ResourceDelta

class ResourceDeltaDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                    testlib_api.SqlTestCase):

    _test_class = quota.ResourceDelta

    def setUp(self):
        super(ResourceDeltaDbObjectTestCase, self).setUp()
        for obj in self.obj_fields:
            self._create_test_reservation(res_id=obj['reservation_id'])

    def _create_test_reservation(self, res_id):
        self._reservation = quota.Reservation(self.context, id=res_id)
        self._reservation.create()

class ReservationObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.Reservation

class ReservationDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                  testlib_api.SqlTestCase):

    _test_class = quota.Reservation

    def _create_test_reservation(self, res=None, exp=None):
        res_id = uuidutils.generate_uuid()
        reservation = self._test_class(self.context,
            id=res_id, resource=res, expiration=exp)
        reservation.create()
        return reservation

    @test_base.unstable_test('bug/1988604')
    def test_delete_expired(self):
        dt = datetime.datetime.utcnow()
        resources = {'goals': 2, 'assists': 1}
        exp_date1 = datetime.datetime(2016, 3, 31, 14, 30)
        res1 = self._create_test_reservation(resources, exp_date1)
        exp_date2 = datetime.datetime(2015, 3, 31, 14, 30)
        res2 = self._create_test_reservation(resources, exp_date2)
        self.assertEqual(2, self._test_class.delete_expired(
            self.context, dt, None))
        objs = self._test_class.get_objects(self.context,
            id=[res1.id, res2.id])
        self.assertEqual([], objs)

    def test_reservation_synthetic_field(self):
        res = self._create_test_reservation()
        resource = 'test-res'
        res_delta = quota.ResourceDelta(self.context,
            resource=resource, reservation_id=res.id, amount='10')
        res_delta.create()
        obj = self._test_class.get_object(self.context, id=res.id)
        self.assertEqual(res_delta, obj.resource_deltas[0])
        res_delta.delete()
        obj.update()
        # NOTE(manjeets) update on reservation should reflect
        # changes on synthetic field when it is deleted.
        obj = self._test_class.get_object(self.context, id=res.id)
        self.assertEqual([], obj.resource_deltas)

class QuotaObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.Quota

class QuotaDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                            testlib_api.SqlTestCase):

    _test_class = quota.Quota

class QuotaUsageObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.QuotaUsage

class QuotaUsageDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                 testlib_api.SqlTestCase):

    _test_class = quota.QuotaUsage

    def _test_get_object_dirty_protected(self, obj, dirty=True):
        obj.create()
        obj.dirty = dirty
        obj.update()
        new = self._test_class.get_object_dirty_protected(
            self.context,
            **obj._get_composite_keys())
        self.assertEqual(obj, new)
        self.assertEqual(dirty, new.dirty)

    def test_get_object_dirty_protected(self):
        obj = self._make_object(self.obj_fields[0])
        obj1 = self._make_object(self.obj_fields[1])
        self._test_get_object_dirty_protected(obj, dirty=False)
        self._test_get_object_dirty_protected(obj1)
<commit_msg>Fix openstack-tox-py39-with-oslo-master job<commit_after># Copyright (c) 2016 Intel Corporation.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import datetime

from neutron_lib.db import api as db_api
from oslo_utils import uuidutils

from neutron.objects import quota
from neutron.tests.unit.objects import test_base as obj_test_base
from neutron.tests.unit import testlib_api

class ResourceDeltaObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.ResourceDelta

class ResourceDeltaDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                    testlib_api.SqlTestCase):

    _test_class = quota.ResourceDelta

    def setUp(self):
        super(ResourceDeltaDbObjectTestCase, self).setUp()
        for obj in self.obj_fields:
            self._create_test_reservation(res_id=obj['reservation_id'])

    def _create_test_reservation(self, res_id):
        self._reservation = quota.Reservation(self.context, id=res_id)
        self._reservation.create()

class ReservationObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.Reservation

class ReservationDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                  testlib_api.SqlTestCase):

    _test_class = quota.Reservation

    def _create_test_reservation(self, res=None, exp=None):
        res_id = uuidutils.generate_uuid()
        reservation = self._test_class(self.context,
            id=res_id, resource=res, expiration=exp)
        reservation.create()
        return reservation

    def test_delete_expired(self):
        dt = datetime.datetime.utcnow()
        resources = {'goals': 2, 'assists': 1}
        exp_date1 = datetime.datetime(2016, 3, 31, 14, 30)
        exp_date2 = datetime.datetime(2015, 3, 31, 14, 30)
        with db_api.CONTEXT_WRITER.using(self.context):
            res1 = self._create_test_reservation(resources, exp_date1)
            res2 = self._create_test_reservation(resources, exp_date2)
        with db_api.CONTEXT_WRITER.using(self.context):
            self.assertEqual(2, self._test_class.delete_expired(
                self.context, dt, None))
        with db_api.CONTEXT_READER.using(self.context):
            objs = self._test_class.get_objects(self.context,
                id=[res1.id, res2.id])
        self.assertEqual([], objs)

    def test_reservation_synthetic_field(self):
        res = self._create_test_reservation()
        resource = 'test-res'
        res_delta = quota.ResourceDelta(self.context,
            resource=resource, reservation_id=res.id, amount='10')
        res_delta.create()
        obj = self._test_class.get_object(self.context, id=res.id)
        self.assertEqual(res_delta, obj.resource_deltas[0])
        res_delta.delete()
        obj.update()
        # NOTE(manjeets) update on reservation should reflect
        # changes on synthetic field when it is deleted.
        obj = self._test_class.get_object(self.context, id=res.id)
        self.assertEqual([], obj.resource_deltas)

class QuotaObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.Quota

class QuotaDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                            testlib_api.SqlTestCase):

    _test_class = quota.Quota

class QuotaUsageObjectIfaceTestCase(obj_test_base.BaseObjectIfaceTestCase):

    _test_class = quota.QuotaUsage

class QuotaUsageDbObjectTestCase(obj_test_base.BaseDbObjectTestCase,
                                 testlib_api.SqlTestCase):

    _test_class = quota.QuotaUsage

    def _test_get_object_dirty_protected(self, obj, dirty=True):
        obj.create()
        obj.dirty = dirty
        obj.update()
        new = self._test_class.get_object_dirty_protected(
            self.context,
            **obj._get_composite_keys())
        self.assertEqual(obj, new)
        self.assertEqual(dirty, new.dirty)

    def test_get_object_dirty_protected(self):
        obj = self._make_object(self.obj_fields[0])
        obj1 = self._make_object(self.obj_fields[1])
        self._test_get_object_dirty_protected(obj, dirty=False)
        self._test_get_object_dirty_protected(obj1)
<|endoftext|>


<commit_before>from django.db import models
from django.utils.translation import ugettext, ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User, Group
from django.conf import settings
from django.template.defaultfilters import date

from ella.core.cache.utils import CachedForeignKey
from ella.core.models import Category
from ella.newman.managers import DenormalizedCategoryUserRoleManager

class DevMessage(models.Model):
    """Development news for ella administrators."""

    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=32)
    summary = models.TextField(_('Summary'))
    details = models.TextField(_('Detail'), blank=True)
    version = models.CharField(_('Version'), max_length=32)

    author = models.ForeignKey(User, editable=False)
    ts = models.DateTimeField(editable=False, auto_now_add=True)

    def __unicode__(self):
        return self.title

    class Meta:
        verbose_name = _('Development message')
        verbose_name_plural = _('Development messages')
        ordering = ('-ts',)
        unique_together = (('slug', 'ts',),)

class AdminHelpItem(models.Model):
    """Help for ella administrators, that extends django help_text functionality."""

    ct = CachedForeignKey(ContentType, verbose_name=_('Model'))
    field = models.CharField(_('Field'), max_length=64, blank=True)
    lang = models.CharField(_('Language'), max_length=5, choices=settings.LANGUAGES)
    short = models.CharField(_('Short help'), max_length=255)
    long = models.TextField(_('Full message'), blank=True)

    def __unicode__(self):
        if self.field:
            return u"%s: %s" % (self.ct, self.field)
        return u"%s: %s" % (self.ct, _('General'))

    class Meta:
        verbose_name = _('Help item')
        verbose_name_plural = _('Help items')
        ordering = ('ct', 'field',)
        unique_together = (('ct', 'field', 'lang',),)

class AdminUserDraft(models.Model):
    """Here is auto-saved objects and user templates."""

    ct = CachedForeignKey(ContentType, verbose_name=_('Model'))
    user = CachedForeignKey(User, verbose_name=_('User'))
    data = models.TextField(_('Data'))

    # Preset title
    title = models.CharField(_('Title'), max_length=64, blank=True)
    ts = models.DateTimeField(editable=False, auto_now=True)

    @property
    def is_preset(self):
        return self.title != ''

    def __unicode__(self):
        if self.is_preset:
            return u"%s (%s)" % (self.title, date(self.ts, 'y-m-d H:i'))
        return u"%s %s (%s)" % (_("Autosaved"), _(self.ct.name), date(self.ts, 'y-m-d H:i'))

    class Meta:
        verbose_name = _('Draft item')
        verbose_name_plural = _('Draft items')
        ordering = ('-ts',)

class AdminSetting(models.Model):
    """Custom settings for newman users/groups"""

    user = CachedForeignKey(User, verbose_name=_('User'), null=True, blank=True)
    group = CachedForeignKey(Group, verbose_name=_('Group'), null=True, blank=True)
    var = models.SlugField(_('Variable'), max_length=64)
    val = models.TextField(_('Value'))

    def __unicode__(self):
        return u"%s: %s for %s" % (self.var, self.val, self.user)

    def __get_val(self):
        return self.val

    def __set_val(self, v):
        self.val = v
    value = property(__get_val, __set_val)

    class Meta:
        unique_together = (('user','var',),('group', 'var',), )
        verbose_name = _('Admin user setting')
        verbose_name_plural = _('Admin user settings')

# ------------------------------------
# Permissions per Category
# ------------------------------------

class CategoryUserRole(models.Model):
    """
    Apply all group's permission for the given user to this category.
    """
    user = models.ForeignKey(User)
    group = models.ForeignKey(Group)
    category = models.ManyToManyField(Category)

    def __unicode__(self):
        try:
            return ugettext(u'User %(user)s is a "%(group)s" for %(category)s.') % {
                    'user' : self.user,
                    'group' : self.group,
                    'category' : [ x['slug'] for x in self.category.values() ],
                }
        except ValueError, ve:
            return ugettext(u'User %(user)s is a "%(group)s" for %(category)s.') % {
                    'user' : self.user,
                    'group' : self.group,
                    'category' : 'unknown'
                }

    def save(self, *args, **kwargs):
        sync = kwargs.pop('sync_role', False)
        super(CategoryUserRole, self).save(*args, **kwargs)
        if sync:
            self.sync_denormalized()

    def sync_denormalized(self):
        from ella.newman.permission import compute_applicable_categories_objects
        denormalized = []
        for p in self.group.permissions.all():
            code = '%s.%s' % (p.content_type.app_label, p.codename)
            cats = compute_applicable_categories_objects(self.user, code)
            # create denormalized roles
            for c in cats:
                root_cat = c.main_parent
                if not root_cat:
                    root_cat = c #c is top category
                elif root_cat.tree_parent_id:
                    root_cat = root_cat.get_tree_parent()
                obj, created = DenormalizedCategoryUserRole.objects.get_or_create(
                    contenttype_id=p.content_type.pk,
                    user_id=self.user.pk,
                    permission_codename=code,
                    permission_id=p.pk,
                    category_id=c.pk,
                    root_category_id=root_cat.pk
                )
                denormalized.append(obj)
        return denormalized

    class Meta:
        verbose_name = _("User role in category")
        verbose_name_plural = _("User roles in categories")

class DenormalizedCategoryUserRole(models.Model):
    """ for better performance """
    user_id = models.IntegerField(db_index=True)
    permission_id = models.IntegerField()
    permission_codename = models.CharField(max_length=100)
    category_id = models.IntegerField()
    root_category_id = models.IntegerField()
    contenttype_id = models.IntegerField()

    objects = DenormalizedCategoryUserRoleManager()

    class Meta:
        unique_together = ('user_id', 'permission_codename', 'permission_id', 'category_id', 'contenttype_id')

<commit_msg>Adding iterators to CategoryUserRole.sync_denormalized() method<commit_after>from django.db import models
from django.utils.translation import ugettext, ugettext_lazy as _
from django.contrib.contenttypes.models import ContentType
from django.contrib.auth.models import User, Group
from django.conf import settings
from django.template.defaultfilters import date

from ella.core.cache.utils import CachedForeignKey
from ella.core.models import Category
from ella.newman.managers import DenormalizedCategoryUserRoleManager

class DevMessage(models.Model):
    """Development news for ella administrators."""

    title = models.CharField(_('Title'), max_length=255)
    slug = models.SlugField(_('Slug'), max_length=32)
    summary = models.TextField(_('Summary'))
    details = models.TextField(_('Detail'), blank=True)
    version = models.CharField(_('Version'), max_length=32)

    author = models.ForeignKey(User, editable=False)
    ts = models.DateTimeField(editable=False, auto_now_add=True)

    def __unicode__(self):
        return self.title

    class Meta:
        verbose_name = _('Development message')
        verbose_name_plural = _('Development messages')
        ordering = ('-ts',)
        unique_together = (('slug', 'ts',),)

class AdminHelpItem(models.Model):
    """Help for ella administrators, that extends django help_text functionality."""

    ct = CachedForeignKey(ContentType, verbose_name=_('Model'))
    field = models.CharField(_('Field'), max_length=64, blank=True)
    lang = models.CharField(_('Language'), max_length=5, choices=settings.LANGUAGES)
    short = models.CharField(_('Short help'), max_length=255)
    long = models.TextField(_('Full message'), blank=True)

    def __unicode__(self):
        if self.field:
            return u"%s: %s" % (self.ct, self.field)
        return u"%s: %s" % (self.ct, _('General'))

    class Meta:
        verbose_name = _('Help item')
        verbose_name_plural = _('Help items')
        ordering = ('ct', 'field',)
        unique_together = (('ct', 'field', 'lang',),)

class AdminUserDraft(models.Model):
    """Here is auto-saved objects and user templates."""

    ct = CachedForeignKey(ContentType, verbose_name=_('Model'))
    user = CachedForeignKey(User, verbose_name=_('User'))
    data = models.TextField(_('Data'))

    # Preset title
    title = models.CharField(_('Title'), max_length=64, blank=True)
    ts = models.DateTimeField(editable=False, auto_now=True)

    @property
    def is_preset(self):
        return self.title != ''

    def __unicode__(self):
        if self.is_preset:
            return u"%s (%s)" % (self.title, date(self.ts, 'y-m-d H:i'))
        return u"%s %s (%s)" % (_("Autosaved"), _(self.ct.name), date(self.ts, 'y-m-d H:i'))

    class Meta:
        verbose_name = _('Draft item')
        verbose_name_plural = _('Draft items')
        ordering = ('-ts',)

class AdminSetting(models.Model):
    """Custom settings for newman users/groups"""

    user = CachedForeignKey(User, verbose_name=_('User'), null=True, blank=True)
    group = CachedForeignKey(Group, verbose_name=_('Group'), null=True, blank=True)
    var = models.SlugField(_('Variable'), max_length=64)
    val = models.TextField(_('Value'))

    def __unicode__(self):
        return u"%s: %s for %s" % (self.var, self.val, self.user)

    def __get_val(self):
        return self.val

    def __set_val(self, v):
        self.val = v
    value = property(__get_val, __set_val)

    class Meta:
        unique_together = (('user','var',),('group', 'var',), )
        verbose_name = _('Admin user setting')
        verbose_name_plural = _('Admin user settings')

# ------------------------------------
# Permissions per Category
# ------------------------------------

class CategoryUserRole(models.Model):
    """
    Apply all group's permission for the given user to this category.
    """
    user = models.ForeignKey(User)
    group = models.ForeignKey(Group)
    category = models.ManyToManyField(Category)

    def __unicode__(self):
        try:
            return ugettext(u'User %(user)s is a "%(group)s" for %(category)s.') % {
                    'user' : self.user,
                    'group' : self.group,
                    'category' : [ x['slug'] for x in self.category.values() ],
                }
        except ValueError, ve:
            return ugettext(u'User %(user)s is a "%(group)s" for %(category)s.') % {
                    'user' : self.user,
                    'group' : self.group,
                    'category' : 'unknown'
                }

    def save(self, *args, **kwargs):
        sync = kwargs.pop('sync_role', False)
        super(CategoryUserRole, self).save(*args, **kwargs)
        if sync:
            self.sync_denormalized()

    def sync_denormalized(self):
        from ella.newman.permission import compute_applicable_categories_objects
        denormalized = []
        #for p in self.group.permissions.all():
        for p in self.group.permissions.all().iterator():
            code = '%s.%s' % (p.content_type.app_label, p.codename)
            cats = compute_applicable_categories_objects(self.user, code)
            # create denormalized roles
            for c in cats:
                root_cat = c.main_parent
                if not root_cat:
                    root_cat = c #c is top category
                elif root_cat.tree_parent_id:
                    root_cat = root_cat.get_tree_parent()
                obj, created = DenormalizedCategoryUserRole.objects.get_or_create(
                    contenttype_id=p.content_type.pk,
                    user_id=self.user.pk,
                    permission_codename=code,
                    permission_id=p.pk,
                    category_id=c.pk,
                    root_category_id=root_cat.pk
                )
                denormalized.append(obj)
        return denormalized

    class Meta:
        verbose_name = _("User role in category")
        verbose_name_plural = _("User roles in categories")

class DenormalizedCategoryUserRole(models.Model):
    """ for better performance """
    user_id = models.IntegerField(db_index=True)
    permission_id = models.IntegerField()
    permission_codename = models.CharField(max_length=100)
    category_id = models.IntegerField()
    root_category_id = models.IntegerField()
    contenttype_id = models.IntegerField()

    objects = DenormalizedCategoryUserRoleManager()

    class Meta:
        unique_together = ('user_id', 'permission_codename', 'permission_id', 'category_id', 'contenttype_id')

<|endoftext|>


<commit_before>'''
Created on Feb 6, 2015

@author: cmccully
'''
import om10
import numpy as np
import re
import json
import os
from lsst.utils import getPackageDir
from lsst.sims.utils import SpecMap
from lsst.sims.catUtils.baseCatalogModels import GalaxyTileCompoundObj
from lsst.sims.photUtils.matchUtils import matchBase
from lsst.sims.photUtils.BandpassDict import BandpassDict
from lsst.sims.photUtils.Sed import Sed
from lsst.sims.utils import radiansFromArcsec

class sprinklerCompound(GalaxyTileCompoundObj):
    objid = 'sprinklerCompound'
    objectTypeID = 1024

    def _final_pass(self, results):
        #From the original GalaxyTileCompoundObj final pass method
        for name in results.dtype.fields:
            if 'raJ2000' in name or 'decJ2000' in name:
                results[name] = np.radians(results[name])

        #Use Sprinkler now
        sp = sprinkler(results, density_param = 0.2)
        results = sp.sprinkle()

        return results

class sprinkler():
    def __init__(self, catsim_cat, om10_cat=str(os.environ['OM10_DIR']+"/data/qso_mock.fits"), density_param = 1.):
        """
        Input:
        catsim_cat:
            The results array from an instance catalog.

        density_param:
            A float between 0. and 1.0 that determines the fraction of eligible agn objects that become lensed.

        Output:
        updated_catalog:
            A new results array with lens systems added.
        """

        self.catalog = catsim_cat
        # ****** THIS ASSUMES THAT THE ENVIRONMENT VARIABLE OM10_DIR IS SET *******
        lensdb = om10.DB(catalog=om10_cat)
        self.lenscat = lensdb.lenses.copy()
        self.density_param = density_param
        self.bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=['i'])

        specFileStart = 'Burst'
        for key, val in sorted(SpecMap.subdir_map.iteritems()):
            if re.match(key, specFileStart):
                galSpecDir = str(val)
        galDir = str(getPackageDir('sims_sed_library') + '/' + galSpecDir + '/')
        self.LRG_name = 'Burst.25E09.1Z.spec'
        self.LRG = Sed()
        self.LRG.readSED_flambda(str(galDir + self.LRG_name))
        #return

    def sprinkle(self):
        # Define a list that we can write out to a text file
        lenslines = []
        # For each galaxy in the catsim catalog
        updated_catalog = self.catalog.copy()
        print "Running sprinkler. Catalog Length: ", len(self.catalog)
        for rowNum, row in enumerate(self.catalog):
            if rowNum == 100 or rowNum % 100000:
                print "Gone through ", rowNum, " lines of catalog."
            if not np.isnan(row['galaxyAgn_magNorm']):
                candidates = self.find_lens_candidates(row['galaxyAgn_redshift'])
                np.random.seed(row['galtileid'] % (2^32 -1))
                pick_value = np.random.uniform()
            # If there aren't any lensed sources at this redshift from OM10 move on the next object
                if ((len(candidates) > 0) and (pick_value <= self.density_param)):
                    # Randomly choose one the lens systems
                    # (can decide with or without replacement)
                    newlens = np.random.choice(candidates)

                    # Append the lens galaxy
                    # For each image, append the lens images
                    for i in range(newlens['NIMG']):
                        lensrow = row.copy()
                        # XIMG and YIMG are in arcseconds
                        # raPhSim and decPhoSim are in radians
                        #Shift all parts of the lensed object, not just its agn part
                        for lensPart in ['galaxyBulge', 'galaxyDisk', 'galaxyAgn']:
                            lensrow[str(lensPart + '_raJ2000')] += np.radians(newlens['XIMG'][i]/(np.cos(np.radians(lensrow[str(lensPart + '_decJ2000')]))*3600.))
                            lensrow[str(lensPart + '_decJ2000')] += np.radians(newlens['YIMG'][i]/3600.)
                        mag_adjust = 2.5*np.log10(np.abs(newlens['MAG'][i]))
                        lensrow['galaxyAgn_magNorm'] -= mag_adjust
                        varString = json.loads(lensrow['galaxyAgn_varParamStr'])
                        varString['pars']['t0Delay'] = newlens['DELAY'][i]
                        varString['varMethodName'] = 'applyAgnTimeDelay'
                        lensrow['galaxyAgn_varParamStr'] = json.dumps(varString)
                        lensrow['galaxyDisk_majorAxis'] = 0.0
                        lensrow['galaxyDisk_minorAxis'] = 0.0
                        lensrow['galaxyDisk_positionAngle'] = 0.0
                        lensrow['galaxyDisk_internalAv'] = 0.0
                        lensrow['galaxyDisk_magNorm'] = np.nan
                        lensrow['galaxyDisk_sedFilename'] = None
                        lensrow['galaxyBulge_majorAxis'] = 0.0
                        lensrow['galaxyBulge_minorAxis'] = 0.0
                        lensrow['galaxyBulge_positionAngle'] = 0.0
                        lensrow['galaxyBulge_internalAv'] = 0.0
                        lensrow['galaxyBulge_magNorm'] = np.nan
                        lensrow['galaxyBulge_sedFilename'] = None
                        lensrow['galaxyBulge_redshift'] = newlens['ZSRC']
                        lensrow['galaxyDisk_redshift'] = newlens['ZSRC']
                        lensrow['galaxyAgn_redshift'] = newlens['ZSRC']
                        lensrow['galtileid'] = np.left_shift(lensrow['galtileid'], 3) + i

                        updated_catalog = np.append(updated_catalog, lensrow)

                    #Now manipulate original entry to be the lens galaxy with desired properties
                    #Start by deleting Disk and AGN properties
                    if not np.isnan(row['galaxyDisk_magNorm']):
                        row['galaxyDisk_majorAxis'] = 0.0
                        row['galaxyDisk_minorAxis'] = 0.0
                        row['galaxyDisk_positionAngle'] = 0.0
                        row['galaxyDisk_internalAv'] = 0.0
                        row['galaxyDisk_magNorm'] = np.nan
                        row['galaxyDisk_sedFilename'] = None
                    row['galaxyAgn_magNorm'] = np.nan
                    row['galaxyAgn_sedFilename'] = None
                    #Now insert desired Bulge properties
                    row['galaxyBulge_sedFilename'] = self.LRG_name
                    row['galaxyBulge_redshift'] = newlens['ZLENS']
                    row['galaxyDisk_redshift'] = newlens['ZLENS']
                    row['galaxyAgn_redshift'] = newlens['ZLENS']
                    row['galaxyBulge_magNorm'] = matchBase().calcMagNorm([newlens['APMAG_I']], self.LRG, self.bandpassDict) #Changed from i band to imsimband
                    newlens['REFF'] = 1.0 #Hard coded for now. See issue in OM10 github.
                    row['galaxyBulge_majorAxis'] = radiansFromArcsec(newlens['REFF'])
                    row['galaxyBulge_minorAxis'] = radiansFromArcsec(newlens['REFF'] * (1 - newlens['ELLIP']))
                    #Convert orientation angle to west of north from east of north by *-1.0 and convert to radians
                    row['galaxyBulge_positionAngle'] = newlens['PHIE']*(-1.0)*np.pi/180.0
                    #Replace original entry with new entry
                    updated_catalog[rowNum] = row

        return updated_catalog

    def find_lens_candidates(self, galz):
        # search the OM10 catalog for all sources +- 0.05 in redshift from the catsim source
        w = np.where(np.abs(self.lenscat['ZSRC'] - galz) <= 0.05)[0]
        lens_candidates = self.lenscat[w]

        return lens_candidates

    def update_catsim(self):
        # Remove the catsim object
        # Add lensed images to the catsim given source brightness and magnifications
        # Add lens galaxy to catsim
        return

    def catsim_to_phosim(self):
        # Pass this catsim to phosim to make images
        return
<commit_msg>Fixed bug 	modified:   sprinkler.py<commit_after>'''
Created on Feb 6, 2015

@author: cmccully
'''
import om10
import numpy as np
import re
import json
import os
from lsst.utils import getPackageDir
from lsst.sims.utils import SpecMap
from lsst.sims.catUtils.baseCatalogModels import GalaxyTileCompoundObj
from lsst.sims.photUtils.matchUtils import matchBase
from lsst.sims.photUtils.BandpassDict import BandpassDict
from lsst.sims.photUtils.Sed import Sed
from lsst.sims.utils import radiansFromArcsec

class sprinklerCompound(GalaxyTileCompoundObj):
    objid = 'sprinklerCompound'
    objectTypeID = 1024

    def _final_pass(self, results):
        #From the original GalaxyTileCompoundObj final pass method
        for name in results.dtype.fields:
            if 'raJ2000' in name or 'decJ2000' in name:
                results[name] = np.radians(results[name])

        #Use Sprinkler now
        sp = sprinkler(results, density_param = 0.2)
        results = sp.sprinkle()

        return results

class sprinkler():
    def __init__(self, catsim_cat, om10_cat=str(os.environ['OM10_DIR']+"/data/qso_mock.fits"), density_param = 1.):
        """
        Input:
        catsim_cat:
            The results array from an instance catalog.

        density_param:
            A float between 0. and 1.0 that determines the fraction of eligible agn objects that become lensed.

        Output:
        updated_catalog:
            A new results array with lens systems added.
        """

        self.catalog = catsim_cat
        # ****** THIS ASSUMES THAT THE ENVIRONMENT VARIABLE OM10_DIR IS SET *******
        lensdb = om10.DB(catalog=om10_cat)
        self.lenscat = lensdb.lenses.copy()
        self.density_param = density_param
        self.bandpassDict = BandpassDict.loadTotalBandpassesFromFiles(bandpassNames=['i'])

        specFileStart = 'Burst'
        for key, val in sorted(SpecMap.subdir_map.iteritems()):
            if re.match(key, specFileStart):
                galSpecDir = str(val)
        galDir = str(getPackageDir('sims_sed_library') + '/' + galSpecDir + '/')
        self.LRG_name = 'Burst.25E09.1Z.spec'
        self.LRG = Sed()
        self.LRG.readSED_flambda(str(galDir + self.LRG_name))
        #return

    def sprinkle(self):
        # Define a list that we can write out to a text file
        lenslines = []
        # For each galaxy in the catsim catalog
        updated_catalog = self.catalog.copy()
        print "Running sprinkler. Catalog Length: ", len(self.catalog)
        for rowNum, row in enumerate(self.catalog):
            if rowNum == 100 or rowNum % 100000==0:
                print "Gone through ", rowNum, " lines of catalog."
            if not np.isnan(row['galaxyAgn_magNorm']):
                candidates = self.find_lens_candidates(row['galaxyAgn_redshift'])
                np.random.seed(row['galtileid'] % (2^32 -1))
                pick_value = np.random.uniform()
            # If there aren't any lensed sources at this redshift from OM10 move on the next object
                if ((len(candidates) > 0) and (pick_value <= self.density_param)):
                    # Randomly choose one the lens systems
                    # (can decide with or without replacement)
                    newlens = np.random.choice(candidates)

                    # Append the lens galaxy
                    # For each image, append the lens images
                    for i in range(newlens['NIMG']):
                        lensrow = row.copy()
                        # XIMG and YIMG are in arcseconds
                        # raPhSim and decPhoSim are in radians
                        #Shift all parts of the lensed object, not just its agn part
                        for lensPart in ['galaxyBulge', 'galaxyDisk', 'galaxyAgn']:
                            lensrow[str(lensPart + '_raJ2000')] += np.radians(newlens['XIMG'][i]/(np.cos(np.radians(lensrow[str(lensPart + '_decJ2000')]))*3600.))
                            lensrow[str(lensPart + '_decJ2000')] += np.radians(newlens['YIMG'][i]/3600.)
                        mag_adjust = 2.5*np.log10(np.abs(newlens['MAG'][i]))
                        lensrow['galaxyAgn_magNorm'] -= mag_adjust
                        varString = json.loads(lensrow['galaxyAgn_varParamStr'])
                        varString['pars']['t0Delay'] = newlens['DELAY'][i]
                        varString['varMethodName'] = 'applyAgnTimeDelay'
                        lensrow['galaxyAgn_varParamStr'] = json.dumps(varString)
                        lensrow['galaxyDisk_majorAxis'] = 0.0
                        lensrow['galaxyDisk_minorAxis'] = 0.0
                        lensrow['galaxyDisk_positionAngle'] = 0.0
                        lensrow['galaxyDisk_internalAv'] = 0.0
                        lensrow['galaxyDisk_magNorm'] = np.nan
                        lensrow['galaxyDisk_sedFilename'] = None
                        lensrow['galaxyBulge_majorAxis'] = 0.0
                        lensrow['galaxyBulge_minorAxis'] = 0.0
                        lensrow['galaxyBulge_positionAngle'] = 0.0
                        lensrow['galaxyBulge_internalAv'] = 0.0
                        lensrow['galaxyBulge_magNorm'] = np.nan
                        lensrow['galaxyBulge_sedFilename'] = None
                        lensrow['galaxyBulge_redshift'] = newlens['ZSRC']
                        lensrow['galaxyDisk_redshift'] = newlens['ZSRC']
                        lensrow['galaxyAgn_redshift'] = newlens['ZSRC']
                        lensrow['galtileid'] = np.left_shift(lensrow['galtileid'], 3) + i

                        updated_catalog = np.append(updated_catalog, lensrow)

                    #Now manipulate original entry to be the lens galaxy with desired properties
                    #Start by deleting Disk and AGN properties
                    if not np.isnan(row['galaxyDisk_magNorm']):
                        row['galaxyDisk_majorAxis'] = 0.0
                        row['galaxyDisk_minorAxis'] = 0.0
                        row['galaxyDisk_positionAngle'] = 0.0
                        row['galaxyDisk_internalAv'] = 0.0
                        row['galaxyDisk_magNorm'] = np.nan
                        row['galaxyDisk_sedFilename'] = None
                    row['galaxyAgn_magNorm'] = np.nan
                    row['galaxyAgn_sedFilename'] = None
                    #Now insert desired Bulge properties
                    row['galaxyBulge_sedFilename'] = self.LRG_name
                    row['galaxyBulge_redshift'] = newlens['ZLENS']
                    row['galaxyDisk_redshift'] = newlens['ZLENS']
                    row['galaxyAgn_redshift'] = newlens['ZLENS']
                    row['galaxyBulge_magNorm'] = matchBase().calcMagNorm([newlens['APMAG_I']], self.LRG, self.bandpassDict) #Changed from i band to imsimband
                    newlens['REFF'] = 1.0 #Hard coded for now. See issue in OM10 github.
                    row['galaxyBulge_majorAxis'] = radiansFromArcsec(newlens['REFF'])
                    row['galaxyBulge_minorAxis'] = radiansFromArcsec(newlens['REFF'] * (1 - newlens['ELLIP']))
                    #Convert orientation angle to west of north from east of north by *-1.0 and convert to radians
                    row['galaxyBulge_positionAngle'] = newlens['PHIE']*(-1.0)*np.pi/180.0
                    #Replace original entry with new entry
                    updated_catalog[rowNum] = row

        return updated_catalog

    def find_lens_candidates(self, galz):
        # search the OM10 catalog for all sources +- 0.05 in redshift from the catsim source
        w = np.where(np.abs(self.lenscat['ZSRC'] - galz) <= 0.05)[0]
        lens_candidates = self.lenscat[w]

        return lens_candidates

    def update_catsim(self):
        # Remove the catsim object
        # Add lensed images to the catsim given source brightness and magnifications
        # Add lens galaxy to catsim
        return

    def catsim_to_phosim(self):
        # Pass this catsim to phosim to make images
        return
<|endoftext|>


<commit_before>import logging
import os
import uuid

import mock
from tornado import gen, testing

from . import base

# os.environ['ASYNC_TEST_TIMEOUT'] = '10'

class AsyncTestCase(base.AsyncTestCase):

    AUTO_CONNECT = False
    CLUSTERING = True

    @property
    def redis_port(self):
        return int(os.environ['NODE1_PORT'])

class ClusterBehaviorTests(AsyncTestCase):

    @testing.gen_test
    def test_connection_move(self):
        yield self.client.connect()
        for offset in range(0, 20):
            key = str(uuid.uuid4())
            yield self.client.set(key, b'1', ex=10)
            value = yield self.client.get(key)
            self.assertEqual(value, b'1')
<commit_msg>Get some debug out put for this issue in Travis<commit_after>import os
import pprint
import uuid

from tornado import testing

import tredis

from . import base

# os.environ['ASYNC_TEST_TIMEOUT'] = '10'

class AsyncTestCase(base.AsyncTestCase):

    AUTO_CONNECT = False
    CLUSTERING = True

    @property
    def redis_port(self):
        return int(os.environ['NODE1_PORT'])

class ClusterBehaviorTests(AsyncTestCase):

    @testing.gen_test
    def test_connection_move(self):
        yield self.client.connect()
        try:
            for offset in range(0, 20):
                key = str(uuid.uuid4())
                yield self.client.set(key, b'1', ex=10)
                value = yield self.client.get(key)
                self.assertEqual(value, b'1')
        except tredis.RedisError as error:
            info = yield self.cluster_info()
            pprint.pprint(info)
            raise

<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 Michael Still and Canonical Inc
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Image cache manager.

The cache manager implements the specification at
http://wiki.openstack.org/nova-image-cache-management.

"""

import hashlib
import os
import re
import time

from nova.compute import task_states
from nova.compute import vm_states
from nova import config
from nova import flags
from nova.openstack.common import cfg
from nova.openstack.common import log as logging
from nova import utils
from nova.virt.libvirt import utils as virtutils

LOG = logging.getLogger(__name__)

imagecache_opts = [
    cfg.BoolOpt('remove_unused_base_images',
                default=True,
                help='Should unused base images be removed?'),
    cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
               default=3600,
               help='Unused resized base images younger than this will not be '
                    'removed'),
    cfg.IntOpt('remove_unused_original_minimum_age_seconds',
               default=(24 * 3600),
               help='Unused unresized base images younger than this will not '
                    'be removed'),
    cfg.BoolOpt('checksum_base_images',
                default=False,
                help='Write a checksum for files in _base to disk'),
    ]

CONF = config.CONF
CONF.import_opt('instances_path', 'nova.compute.manager')
CONF.import_opt('base_dir_name', 'nova.compute.manager')
FLAGS = flags.FLAGS
FLAGS.register_opts(imagecache_opts)

def read_stored_checksum(target):
    """Read the checksum.

    Returns the checksum (as hex) or None.
    """
    return virtutils.read_stored_info(target, field='sha1')

def write_stored_checksum(target):
    """Write a checksum to disk for a file in _base."""

    if not read_stored_checksum(target):
        img_file = open(target, 'r')
        checksum = utils.hash_file(img_file)
        img_file.close()

        virtutils.write_stored_info(target, field='sha1', value=checksum)

class ImageCacheManager(object):
    def __init__(self):
        self._reset_state()

    def _reset_state(self):
        """Reset state variables used for each pass."""

        self.used_images = {}
        self.image_popularity = {}
        self.instance_names = {}

        self.active_base_files = []
        self.corrupt_base_files = []
        self.originals = []
        self.removable_base_files = []
        self.unexplained_images = []

    def _store_image(self, base_dir, ent, original=False):
        """Store a base image for later examination."""
        entpath = os.path.join(base_dir, ent)
        if os.path.isfile(entpath):
            self.unexplained_images.append(entpath)
            if original:
                self.originals.append(entpath)

    def _list_base_images(self, base_dir):
        """Return a list of the images present in _base.

        Determine what images we have on disk. There will be other files in
        this directory (for example kernels) so we only grab the ones which
        are the right length to be disk images.

        Note that this does not return a value. It instead populates a class
        variable with a list of images that we need to try and explain.
        """
        digest_size = hashlib.sha1().digestsize * 2
        for ent in os.listdir(base_dir):
            if len(ent) == digest_size:
                self._store_image(base_dir, ent, original=True)

            elif (len(ent) > digest_size + 2 and
                  ent[digest_size] == '_' and
                  not virtutils.is_valid_info_file(os.path.join(base_dir,
                                                                ent))):
                self._store_image(base_dir, ent, original=False)

    def _list_running_instances(self, context, all_instances):
        """List running instances (on all compute nodes)."""
        self.used_images = {}
        self.image_popularity = {}
        self.instance_names = set()

        for instance in all_instances:
            self.instance_names.add(instance['name'])

            resize_states = [task_states.RESIZE_PREP,
                             task_states.RESIZE_MIGRATING,
                             task_states.RESIZE_MIGRATED,
                             task_states.RESIZE_FINISH]
            if instance['task_state'] in resize_states or \
                instance['vm_state'] == vm_states.RESIZED:
                self.instance_names.add(instance['name'] + '_resize')

            image_ref_str = str(instance['image_ref'])
            local, remote, insts = self.used_images.get(image_ref_str,
                                                        (0, 0, []))
            if instance['host'] == FLAGS.host:
                local += 1
            else:
                remote += 1
            insts.append(instance['name'])
            self.used_images[image_ref_str] = (local, remote, insts)

            self.image_popularity.setdefault(image_ref_str, 0)
            self.image_popularity[image_ref_str] += 1

    def _list_backing_images(self):
        """List the backing images currently in use."""
        inuse_images = []
        for ent in os.listdir(FLAGS.instances_path):
            if ent in self.instance_names:
                LOG.debug(_('%s is a valid instance name'), ent)
                disk_path = os.path.join(FLAGS.instances_path, ent, 'disk')
                if os.path.exists(disk_path):
                    LOG.debug(_('%s has a disk file'), ent)
                    backing_file = virtutils.get_disk_backing_file(disk_path)
                    LOG.debug(_('Instance %(instance)s is backed by '
                                '%(backing)s'),
                              {'instance': ent,
                               'backing': backing_file})

                    if backing_file:
                        backing_path = os.path.join(FLAGS.instances_path,
                                                    FLAGS.base_dir_name,
                                                    backing_file)
                        if not backing_path in inuse_images:
                            inuse_images.append(backing_path)

                        if backing_path in self.unexplained_images:
                            LOG.warning(_('Instance %(instance)s is using a '
                                          'backing file %(backing)s which '
                                          'does not appear in the image '
                                          'service'),
                                        {'instance': ent,
                                         'backing': backing_file})
                            self.unexplained_images.remove(backing_path)

        return inuse_images

    def _find_base_file(self, base_dir, fingerprint):
        """Find the base file matching this fingerprint.

        Yields the name of the base file, a boolean which is True if the image
        is "small", and a boolean which indicates if this is a resized image.
        Note that is is possible for more than one yield to result from this
        check.

        If no base file is found, then nothing is yielded.
        """
        # The original file from glance
        base_file = os.path.join(base_dir, fingerprint)
        if os.path.exists(base_file):
            yield base_file, False, False

        # An older naming style which can be removed sometime after Folsom
        base_file = os.path.join(base_dir, fingerprint + '_sm')
        if os.path.exists(base_file):
            yield base_file, True, False

        # Resized images
        resize_re = re.compile('.*/%s_[0-9]+$' % fingerprint)
        for img in self.unexplained_images:
            m = resize_re.match(img)
            if m:
                yield img, False, True

    def _verify_checksum(self, img_id, base_file, create_if_missing=True):
        """Compare the checksum stored on disk with the current file.

        Note that if the checksum fails to verify this is logged, but no actual
        action occurs. This is something sysadmins should monitor for and
        handle manually when it occurs.
        """

        if not FLAGS.checksum_base_images:
            return None

        stored_checksum = read_stored_checksum(base_file)
        if stored_checksum:
            f = open(base_file, 'r')
            current_checksum = utils.hash_file(f)
            f.close()

            if current_checksum != stored_checksum:
                LOG.error(_('%(id)s (%(base_file)s): image verification '
                            'failed'),
                          {'id': img_id,
                           'base_file': base_file})
                return False

            else:
                return True

        else:
            LOG.info(_('%(id)s (%(base_file)s): image verification skipped, '
                       'no hash stored'),
                     {'id': img_id,
                      'base_file': base_file})

            # NOTE(mikal): If the checksum file is missing, then we should
            # create one. We don't create checksums when we download images
            # from glance because that would delay VM startup.
            if create_if_missing:
                write_stored_checksum(base_file)

            return None

    def _remove_base_file(self, base_file):
        """Remove a single base file if it is old enough.

        Returns nothing.
        """
        if not os.path.exists(base_file):
            LOG.debug(_('Cannot remove %(base_file)s, it does not exist'),
                      base_file)
            return

        mtime = os.path.getmtime(base_file)
        age = time.time() - mtime

        maxage = FLAGS.remove_unused_resized_minimum_age_seconds
        if base_file in self.originals:
            maxage = FLAGS.remove_unused_original_minimum_age_seconds

        if age < maxage:
            LOG.info(_('Base file too young to remove: %s'),
                     base_file)
        else:
            LOG.info(_('Removing base file: %s'), base_file)
            try:
                os.remove(base_file)
                signature = virtutils.get_info_filename(base_file)
                if os.path.exists(signature):
                    os.remove(signature)
            except OSError, e:
                LOG.error(_('Failed to remove %(base_file)s, '
                            'error was %(error)s'),
                          {'base_file': base_file,
                           'error': e})

    def _handle_base_image(self, img_id, base_file):
        """Handle the checks for a single base image."""

        image_bad = False
        image_in_use = False

        LOG.info(_('%(id)s (%(base_file)s): checking'),
                 {'id': img_id,
                  'base_file': base_file})

        if base_file in self.unexplained_images:
            self.unexplained_images.remove(base_file)

        if (base_file and os.path.exists(base_file)
            and os.path.isfile(base_file)):
            # _verify_checksum returns True if the checksum is ok, and None if
            # there is no checksum file
            checksum_result = self._verify_checksum(img_id, base_file)
            if not checksum_result is None:
                image_bad = not checksum_result

        instances = []
        if img_id in self.used_images:
            local, remote, instances = self.used_images[img_id]
            if local > 0:
                LOG.debug(_('%(id)s (%(base_file)s): '
                            'in use: on this node %(local)d local, '
                            '%(remote)d on other nodes'),
                          {'id': img_id,
                           'base_file': base_file,
                           'local': local,
                           'remote': remote})

                image_in_use = True
                self.active_base_files.append(base_file)

                if not base_file:
                    LOG.warning(_('%(id)s (%(base_file)s): warning -- an '
                                  'absent base file is in use! instances: '
                                  '%(instance_list)s'),
                                {'id': img_id,
                                 'base_file': base_file,
                                 'instance_list': ' '.join(instances)})

            else:
                LOG.debug(_('%(id)s (%(base_file)s): in use on (%(remote)d on '
                            'other nodes)'),
                          {'id': img_id,
                           'base_file': base_file,
                           'remote': remote})
        if image_bad:
            self.corrupt_base_files.append(base_file)

        if base_file:
            if not image_in_use:
                LOG.debug(_('%(id)s (%(base_file)s): image is not in use'),
                          {'id': img_id,
                           'base_file': base_file})
                self.removable_base_files.append(base_file)

            else:
                LOG.debug(_('%(id)s (%(base_file)s): image is in use'),
                          {'id': img_id,
                           'base_file': base_file})
                if os.path.exists(base_file):
                    virtutils.chown(base_file, os.getuid())
                    os.utime(base_file, None)

    def verify_base_images(self, context, all_instances):
        """Verify that base images are in a reasonable state."""

        # NOTE(mikal): The new scheme for base images is as follows -- an
        # image is streamed from the image service to _base (filename is the
        # sha1 hash of the image id). If CoW is enabled, that file is then
        # resized to be the correct size for the instance (filename is the
        # same as the original, but with an underscore and the resized size
        # in bytes). This second file is then CoW'd to the instance disk. If
        # CoW is disabled, the resize occurs as part of the copy from the
        # cache to the instance directory. Files ending in _sm are no longer
        # created, but may remain from previous versions.
        self._reset_state()

        base_dir = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name)
        if not os.path.exists(base_dir):
            LOG.debug(_('Skipping verification, no base directory at %s'),
                      base_dir)
            return

        LOG.debug(_('Verify base images'))
        self._list_base_images(base_dir)
        self._list_running_instances(context, all_instances)

        # Determine what images are on disk because they're in use
        for img in self.used_images:
            fingerprint = hashlib.sha1(img).hexdigest()
            LOG.debug(_('Image id %(id)s yields fingerprint %(fingerprint)s'),
                      {'id': img,
                       'fingerprint': fingerprint})
            for result in self._find_base_file(base_dir, fingerprint):
                base_file, image_small, image_resized = result
                self._handle_base_image(img, base_file)

                if not image_small and not image_resized:
                    self.originals.append(base_file)

        # Elements remaining in unexplained_images might be in use
        inuse_backing_images = self._list_backing_images()
        for backing_path in inuse_backing_images:
            if not backing_path in self.active_base_files:
                self.active_base_files.append(backing_path)

        # Anything left is an unknown base image
        for img in self.unexplained_images:
            LOG.warning(_('Unknown base file: %s'), img)
            self.removable_base_files.append(img)

        # Dump these lists
        if self.active_base_files:
            LOG.info(_('Active base files: %s'),
                     ' '.join(self.active_base_files))
        if self.corrupt_base_files:
            LOG.info(_('Corrupt base files: %s'),
                     ' '.join(self.corrupt_base_files))

        if self.removable_base_files:
            LOG.info(_('Removable base files: %s'),
                     ' '.join(self.removable_base_files))

            if FLAGS.remove_unused_base_images:
                for base_file in self.removable_base_files:
                    self._remove_base_file(base_file)

        # That's it
        LOG.debug(_('Verification complete'))
<commit_msg>Yield in between hash runs for the image cache manager.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2012 Michael Still and Canonical Inc
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Image cache manager.

The cache manager implements the specification at
http://wiki.openstack.org/nova-image-cache-management.

"""

import hashlib
import os
import re
import time

from nova.compute import task_states
from nova.compute import vm_states
from nova import config
from nova import flags
from nova.openstack.common import cfg
from nova.openstack.common import log as logging
from nova import utils
from nova.virt.libvirt import utils as virtutils

LOG = logging.getLogger(__name__)

imagecache_opts = [
    cfg.BoolOpt('remove_unused_base_images',
                default=True,
                help='Should unused base images be removed?'),
    cfg.IntOpt('remove_unused_resized_minimum_age_seconds',
               default=3600,
               help='Unused resized base images younger than this will not be '
                    'removed'),
    cfg.IntOpt('remove_unused_original_minimum_age_seconds',
               default=(24 * 3600),
               help='Unused unresized base images younger than this will not '
                    'be removed'),
    cfg.BoolOpt('checksum_base_images',
                default=False,
                help='Write a checksum for files in _base to disk'),
    ]

CONF = config.CONF
CONF.import_opt('instances_path', 'nova.compute.manager')
CONF.import_opt('base_dir_name', 'nova.compute.manager')
FLAGS = flags.FLAGS
FLAGS.register_opts(imagecache_opts)

def read_stored_checksum(target):
    """Read the checksum.

    Returns the checksum (as hex) or None.
    """
    return virtutils.read_stored_info(target, field='sha1')

def write_stored_checksum(target):
    """Write a checksum to disk for a file in _base."""

    if not read_stored_checksum(target):
        img_file = open(target, 'r')
        checksum = utils.hash_file(img_file)
        img_file.close()

        virtutils.write_stored_info(target, field='sha1', value=checksum)

class ImageCacheManager(object):
    def __init__(self):
        self._reset_state()

    def _reset_state(self):
        """Reset state variables used for each pass."""

        self.used_images = {}
        self.image_popularity = {}
        self.instance_names = {}

        self.active_base_files = []
        self.corrupt_base_files = []
        self.originals = []
        self.removable_base_files = []
        self.unexplained_images = []

    def _store_image(self, base_dir, ent, original=False):
        """Store a base image for later examination."""
        entpath = os.path.join(base_dir, ent)
        if os.path.isfile(entpath):
            self.unexplained_images.append(entpath)
            if original:
                self.originals.append(entpath)

    def _list_base_images(self, base_dir):
        """Return a list of the images present in _base.

        Determine what images we have on disk. There will be other files in
        this directory (for example kernels) so we only grab the ones which
        are the right length to be disk images.

        Note that this does not return a value. It instead populates a class
        variable with a list of images that we need to try and explain.
        """
        digest_size = hashlib.sha1().digestsize * 2
        for ent in os.listdir(base_dir):
            if len(ent) == digest_size:
                self._store_image(base_dir, ent, original=True)

            elif (len(ent) > digest_size + 2 and
                  ent[digest_size] == '_' and
                  not virtutils.is_valid_info_file(os.path.join(base_dir,
                                                                ent))):
                self._store_image(base_dir, ent, original=False)

    def _list_running_instances(self, context, all_instances):
        """List running instances (on all compute nodes)."""
        self.used_images = {}
        self.image_popularity = {}
        self.instance_names = set()

        for instance in all_instances:
            self.instance_names.add(instance['name'])

            resize_states = [task_states.RESIZE_PREP,
                             task_states.RESIZE_MIGRATING,
                             task_states.RESIZE_MIGRATED,
                             task_states.RESIZE_FINISH]
            if instance['task_state'] in resize_states or \
                instance['vm_state'] == vm_states.RESIZED:
                self.instance_names.add(instance['name'] + '_resize')

            image_ref_str = str(instance['image_ref'])
            local, remote, insts = self.used_images.get(image_ref_str,
                                                        (0, 0, []))
            if instance['host'] == FLAGS.host:
                local += 1
            else:
                remote += 1
            insts.append(instance['name'])
            self.used_images[image_ref_str] = (local, remote, insts)

            self.image_popularity.setdefault(image_ref_str, 0)
            self.image_popularity[image_ref_str] += 1

    def _list_backing_images(self):
        """List the backing images currently in use."""
        inuse_images = []
        for ent in os.listdir(FLAGS.instances_path):
            if ent in self.instance_names:
                LOG.debug(_('%s is a valid instance name'), ent)
                disk_path = os.path.join(FLAGS.instances_path, ent, 'disk')
                if os.path.exists(disk_path):
                    LOG.debug(_('%s has a disk file'), ent)
                    backing_file = virtutils.get_disk_backing_file(disk_path)
                    LOG.debug(_('Instance %(instance)s is backed by '
                                '%(backing)s'),
                              {'instance': ent,
                               'backing': backing_file})

                    if backing_file:
                        backing_path = os.path.join(FLAGS.instances_path,
                                                    FLAGS.base_dir_name,
                                                    backing_file)
                        if not backing_path in inuse_images:
                            inuse_images.append(backing_path)

                        if backing_path in self.unexplained_images:
                            LOG.warning(_('Instance %(instance)s is using a '
                                          'backing file %(backing)s which '
                                          'does not appear in the image '
                                          'service'),
                                        {'instance': ent,
                                         'backing': backing_file})
                            self.unexplained_images.remove(backing_path)

        return inuse_images

    def _find_base_file(self, base_dir, fingerprint):
        """Find the base file matching this fingerprint.

        Yields the name of the base file, a boolean which is True if the image
        is "small", and a boolean which indicates if this is a resized image.
        Note that is is possible for more than one yield to result from this
        check.

        If no base file is found, then nothing is yielded.
        """
        # The original file from glance
        base_file = os.path.join(base_dir, fingerprint)
        if os.path.exists(base_file):
            yield base_file, False, False

        # An older naming style which can be removed sometime after Folsom
        base_file = os.path.join(base_dir, fingerprint + '_sm')
        if os.path.exists(base_file):
            yield base_file, True, False

        # Resized images
        resize_re = re.compile('.*/%s_[0-9]+$' % fingerprint)
        for img in self.unexplained_images:
            m = resize_re.match(img)
            if m:
                yield img, False, True

    def _verify_checksum(self, img_id, base_file, create_if_missing=True):
        """Compare the checksum stored on disk with the current file.

        Note that if the checksum fails to verify this is logged, but no actual
        action occurs. This is something sysadmins should monitor for and
        handle manually when it occurs.
        """

        if not FLAGS.checksum_base_images:
            return None

        stored_checksum = read_stored_checksum(base_file)
        if stored_checksum:
            f = open(base_file, 'r')
            current_checksum = utils.hash_file(f)
            f.close()

            if current_checksum != stored_checksum:
                LOG.error(_('%(id)s (%(base_file)s): image verification '
                            'failed'),
                          {'id': img_id,
                           'base_file': base_file})
                return False

            else:
                return True

        else:
            LOG.info(_('%(id)s (%(base_file)s): image verification skipped, '
                       'no hash stored'),
                     {'id': img_id,
                      'base_file': base_file})

            # NOTE(mikal): If the checksum file is missing, then we should
            # create one. We don't create checksums when we download images
            # from glance because that would delay VM startup.
            if create_if_missing:
                write_stored_checksum(base_file)

            return None

    def _remove_base_file(self, base_file):
        """Remove a single base file if it is old enough.

        Returns nothing.
        """
        if not os.path.exists(base_file):
            LOG.debug(_('Cannot remove %(base_file)s, it does not exist'),
                      base_file)
            return

        mtime = os.path.getmtime(base_file)
        age = time.time() - mtime

        maxage = FLAGS.remove_unused_resized_minimum_age_seconds
        if base_file in self.originals:
            maxage = FLAGS.remove_unused_original_minimum_age_seconds

        if age < maxage:
            LOG.info(_('Base file too young to remove: %s'),
                     base_file)
        else:
            LOG.info(_('Removing base file: %s'), base_file)
            try:
                os.remove(base_file)
                signature = virtutils.get_info_filename(base_file)
                if os.path.exists(signature):
                    os.remove(signature)
            except OSError, e:
                LOG.error(_('Failed to remove %(base_file)s, '
                            'error was %(error)s'),
                          {'base_file': base_file,
                           'error': e})

    def _handle_base_image(self, img_id, base_file):
        """Handle the checks for a single base image."""

        image_bad = False
        image_in_use = False

        LOG.info(_('%(id)s (%(base_file)s): checking'),
                 {'id': img_id,
                  'base_file': base_file})

        if base_file in self.unexplained_images:
            self.unexplained_images.remove(base_file)

        if (base_file and os.path.exists(base_file)
            and os.path.isfile(base_file)):
            # _verify_checksum returns True if the checksum is ok, and None if
            # there is no checksum file
            checksum_result = self._verify_checksum(img_id, base_file)
            if not checksum_result is None:
                image_bad = not checksum_result

            # Give other threads a chance to run
            time.sleep(0)

        instances = []
        if img_id in self.used_images:
            local, remote, instances = self.used_images[img_id]
            if local > 0:
                LOG.debug(_('%(id)s (%(base_file)s): '
                            'in use: on this node %(local)d local, '
                            '%(remote)d on other nodes'),
                          {'id': img_id,
                           'base_file': base_file,
                           'local': local,
                           'remote': remote})

                image_in_use = True
                self.active_base_files.append(base_file)

                if not base_file:
                    LOG.warning(_('%(id)s (%(base_file)s): warning -- an '
                                  'absent base file is in use! instances: '
                                  '%(instance_list)s'),
                                {'id': img_id,
                                 'base_file': base_file,
                                 'instance_list': ' '.join(instances)})

            else:
                LOG.debug(_('%(id)s (%(base_file)s): in use on (%(remote)d on '
                            'other nodes)'),
                          {'id': img_id,
                           'base_file': base_file,
                           'remote': remote})
        if image_bad:
            self.corrupt_base_files.append(base_file)

        if base_file:
            if not image_in_use:
                LOG.debug(_('%(id)s (%(base_file)s): image is not in use'),
                          {'id': img_id,
                           'base_file': base_file})
                self.removable_base_files.append(base_file)

            else:
                LOG.debug(_('%(id)s (%(base_file)s): image is in use'),
                          {'id': img_id,
                           'base_file': base_file})
                if os.path.exists(base_file):
                    virtutils.chown(base_file, os.getuid())
                    os.utime(base_file, None)

    def verify_base_images(self, context, all_instances):
        """Verify that base images are in a reasonable state."""

        # NOTE(mikal): The new scheme for base images is as follows -- an
        # image is streamed from the image service to _base (filename is the
        # sha1 hash of the image id). If CoW is enabled, that file is then
        # resized to be the correct size for the instance (filename is the
        # same as the original, but with an underscore and the resized size
        # in bytes). This second file is then CoW'd to the instance disk. If
        # CoW is disabled, the resize occurs as part of the copy from the
        # cache to the instance directory. Files ending in _sm are no longer
        # created, but may remain from previous versions.
        self._reset_state()

        base_dir = os.path.join(FLAGS.instances_path, FLAGS.base_dir_name)
        if not os.path.exists(base_dir):
            LOG.debug(_('Skipping verification, no base directory at %s'),
                      base_dir)
            return

        LOG.debug(_('Verify base images'))
        self._list_base_images(base_dir)
        self._list_running_instances(context, all_instances)

        # Determine what images are on disk because they're in use
        for img in self.used_images:
            fingerprint = hashlib.sha1(img).hexdigest()
            LOG.debug(_('Image id %(id)s yields fingerprint %(fingerprint)s'),
                      {'id': img,
                       'fingerprint': fingerprint})
            for result in self._find_base_file(base_dir, fingerprint):
                base_file, image_small, image_resized = result
                self._handle_base_image(img, base_file)

                if not image_small and not image_resized:
                    self.originals.append(base_file)

        # Elements remaining in unexplained_images might be in use
        inuse_backing_images = self._list_backing_images()
        for backing_path in inuse_backing_images:
            if not backing_path in self.active_base_files:
                self.active_base_files.append(backing_path)

        # Anything left is an unknown base image
        for img in self.unexplained_images:
            LOG.warning(_('Unknown base file: %s'), img)
            self.removable_base_files.append(img)

        # Dump these lists
        if self.active_base_files:
            LOG.info(_('Active base files: %s'),
                     ' '.join(self.active_base_files))
        if self.corrupt_base_files:
            LOG.info(_('Corrupt base files: %s'),
                     ' '.join(self.corrupt_base_files))

        if self.removable_base_files:
            LOG.info(_('Removable base files: %s'),
                     ' '.join(self.removable_base_files))

            if FLAGS.remove_unused_base_images:
                for base_file in self.removable_base_files:
                    self._remove_base_file(base_file)

        # That's it
        LOG.debug(_('Verification complete'))
<|endoftext|>


<commit_before>from auditcare.models import AuditEvent
from couchforms.models import XFormInstance
from couchlog.models import ExceptionRecord
from pillowtop.listener import NetworkPillow
from pillowtop.listener import ElasticPillow
from casexml.apps.case.models import CommCareCase
from corehq.apps.domain.models import Domain
import settings

class CasePillow(ElasticPillow):
    couch_db = CommCareCase.get_db()
    couch_filter = "case/casedocs"
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "hqcases"
    es_type = "case"

    def change_transform(self, doc_dict):
        """
        Lighten the load of the search index by removing the data heavy transactional cruft
        """
        if doc_dict.has_key('actions'):
            del doc_dict['actions']
        if doc_dict.has_key('xform_ids'):
            del doc_dict['xform_ids']
        return doc_dict

class AuditcarePillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_AUDITCARE_PORT
    couch_db = AuditEvent.get_db()
    couch_filter = 'auditcare/auditdocs'

class CouchlogPillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_COUCHLOG_PORT
    couch_db = ExceptionRecord.get_db()
    couch_filter = 'couchlog/couchlogs'

class DevicelogPillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_DEVICELOG_PORT
    couch_db = XFormInstance.get_db()
    couch_filter = 'couchforms/devicelogs'

class StrippedXformPillow(ElasticPillow):
    couch_db = XFormInstance.get_db()
    couch_filter = ""
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "xforms_clean"
    es_type = "xform"

    def change_transform(self, doc_dict):
        """
        strip all properties of the form except meta/submit info
        """
        for k in doc_dict['form']:
            if k not in ['meta', 'Meta', '@uiVersion', '@version']:
                del doc_dict['form'][k]
        #todo: add geoip block here
        return doc_dict

    def change_transport(self, doc_dict):
        #not ready yet!
        return None

class ExchangePillow(ElasticPillow):
    couch_db = Domain.get_db()
    couch_filter = "domain/all_domains"
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "cc_exchange"
    es_type = "domain"
    es_meta = {
        "settings" : {
            "analysis" : {
                "analyzer" : {
                    "lowercase_analyzer" : {
                        "type" : "custom",
                        "tokenizer" : "keyword",
                        "filter" : [ "lowercase"]}}}},
        "mappings" : {
            "domain" : {
                "properties" : {
                    "license" : { "type": "string", "index" : "not_analyzed" },
                    "deployment.region": { "type": "string", "analyzer": "lowercase_analyzer" }}}}}
<commit_msg>updating pillow for including the default index metadata<commit_after>from auditcare.models import AuditEvent
from couchforms.models import XFormInstance
from couchlog.models import ExceptionRecord
from pillowtop.listener import NetworkPillow
from pillowtop.listener import ElasticPillow
from casexml.apps.case.models import CommCareCase
from corehq.apps.domain.models import Domain
import settings

class CasePillow(ElasticPillow):
    couch_db = CommCareCase.get_db()
    couch_filter = "case/casedocs"
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "hqcases"
    es_type = "case"
    #the meta here is defined for when the case index is created for the FIRST time
    #subsequent data added to it will be added automatically, but the date_detection is necessary
    # to be false to prevent indexes from not being created due to the way we store dates
    #all will be strings EXCEPT the core case properties which we need to explicitly define below.
    #that way date sort and ranges will work with canonical date formats for queries.
    es_meta = { "case": {
            "date_detection": False,
            "properties": {
                "name": {
                    "type": "string"
                },
                "domain": {
                    "type": "string"
                },
                "modified_on": {
                    "format": "dateOptionalTime",
                    "type": "date"
                },
                "closed_on": {
                    "format": "dateOptionalTime",
                    "type": "date"
                },
                "opened_on": {
                    "format": "dateOptionalTime",
                    "type": "date"
                },
                "user_id": {
                    "type": "string"
                },
                "closed": {
                    "type": "boolean"
                },
                "type": {
                    "type": "string"
                },
                "owner_id": {
                    "type": "string"
                }
            }
        }
    }

    def change_transform(self, doc_dict):
        """
        Lighten the load of the search index by removing the data heavy transactional cruft
        """
        if doc_dict.has_key('actions'):
            del doc_dict['actions']
        if doc_dict.has_key('xform_ids'):
            del doc_dict['xform_ids']
        return doc_dict

class AuditcarePillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_AUDITCARE_PORT
    couch_db = AuditEvent.get_db()
    couch_filter = 'auditcare/auditdocs'

class CouchlogPillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_COUCHLOG_PORT
    couch_db = ExceptionRecord.get_db()
    couch_filter = 'couchlog/couchlogs'

class DevicelogPillow(NetworkPillow):
    endpoint_host = settings.LOGSTASH_HOST
    endpoint_port = settings.LOGSTASH_DEVICELOG_PORT
    couch_db = XFormInstance.get_db()
    couch_filter = 'couchforms/devicelogs'

class StrippedXformPillow(ElasticPillow):
    couch_db = XFormInstance.get_db()
    couch_filter = ""
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "xforms_clean"
    es_type = "xform"

    def change_transform(self, doc_dict):
        """
        strip all properties of the form except meta/submit info
        """
        for k in doc_dict['form']:
            if k not in ['meta', 'Meta', '@uiVersion', '@version']:
                del doc_dict['form'][k]
            #todo: add geoip block here
        return doc_dict

    def change_transport(self, doc_dict):
        #not ready yet!
        return None

class ExchangePillow(ElasticPillow):
    couch_db = Domain.get_db()
    couch_filter = "domain/all_domains"
    es_host = settings.ELASTICSEARCH_HOST
    es_port = settings.ELASTICSEARCH_PORT
    es_index = "cc_exchange"
    es_type = "domain"
    es_meta = {
        "settings": {
            "analysis": {
                "analyzer": {
                    "lowercase_analyzer": {
                        "type": "custom",
                        "tokenizer": "keyword",
                        "filter": ["lowercase"]}}}},
        "mappings": {
            "domain": {
                "properties": {
                    "license": {"type": "string", "index": "not_analyzed"},
                    "deployment.region": {"type": "string", "analyzer": "lowercase_analyzer"}}}}}
<|endoftext|>


<commit_before>import sublime, sublime_plugin
import os

class JasmineCoffeeScriptDetectFileTypeCommand(sublime_plugin.EventListener):
  """ Detects current file type if the file's extension isn't conclusive """

  def on_load(self, view):
    filename = view.file_name()
    if not filename: # buffer has never been saved
      return

    name = os.path.basename(filename.lower())
    if name.endswith("spec.js.coffee") or name.endswith("spec.coffee"):
      set_syntax(view, "Jasmine CoffeeScript", "jasmine-coffeescript/Syntaxes")

def set_syntax(view, syntax, path=None):
  if path is None:
    path = syntax
  view.settings().set('syntax', 'Packages/'+ path + '/' + syntax + '.tmLanguage')
  print("Switched syntax to: " + syntax)
<commit_msg>Adjust syntax detector to account for new ST3 default package name<commit_after>import sublime, sublime_plugin
import os

class JasmineCoffeeScriptDetectFileTypeCommand(sublime_plugin.EventListener):
  """ Detects current file type if the file's extension isn't conclusive """

  def on_load(self, view):
    filename = view.file_name()
    if not filename: # buffer has never been saved
      return

    name = os.path.basename(filename.lower())
    if name.endswith("spec.js.coffee") or name.endswith("spec.coffee"):
      set_syntax(view, "Jasmine CoffeeScript", "Jasmine CoffeeScript/Syntaxes")

def set_syntax(view, syntax, path=None):
  if path is None:
    path = syntax
  view.settings().set('syntax', 'Packages/'+ path + '/' + syntax + '.tmLanguage')
  print("Switched syntax to: " + syntax)
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
# ! python3

import sys
import time
from contextlib import ContextDecorator

__all__ = ["Timer"]

class Timer(ContextDecorator):
    """
    Timer class that can be used both as a context manager and function/method decorator.

    Usage:

    >>> import math
    >>> import time
    >>>
    >>> with Timer():
    ...    for i in range(42):
    ...        print("{}! = {:.5}...".format(i**2, str(math.factorial(i**2))))
    >>>
    >>> @Timer(title="Second")
    ... def some_func():
    ...     time.sleep(1)
    >>>
    >>> with Timer(title="Some title") as t:
    ...    for i in range(42):
    ...        print("{}! = {:.5}...".format(i**2, str(math.factorial(i**2))))
    >>>
    >>> print(t.elapsed)
    """

    def __init__(self, title: str = "", print_title = True):
        self._title = title
        self._print = sys.stdout if print_title else print_title
        self._elapsed = 0

    def __float__(self) -> float:
        return float(self.elapsed)

    def __str__(self) -> str:
        """
        “informal” or nicely printable string representation of an object
        """
        return "Elapsed {}".format(self.__repr__())

    def __repr__(self) -> str:
        """
        “official” string representation of an object.
        """
        return str(float(self))

    def __enter__(self) -> 'Timer':
        self.start = time.perf_counter()

        return self

    def __exit__(self, *args):
        self._elapsed = time.perf_counter() - self.start
        if self._print:
            title = "[{}] ".format(self._title) if self._title else ""
            print('{title}Total time {total_seconds:.5f} seconds.'.format(title=title, total_seconds=self._elapsed),
                  file=self._print)

    @property
    def elapsed(self) -> float:
        return self._elapsed
<commit_msg>formatting<commit_after># -*- encoding: utf-8 -*-
# ! python3

import sys
import time
from contextlib import ContextDecorator

__all__ = ["Timer"]

class Timer(ContextDecorator):
    """
    Timer class that can be used both as a context manager and function/method decorator.

    Usage:

    >>> import math
    >>> import time
    >>>
    >>> with Timer():
    ...    for i in range(42):
    ...        print("{}! = {:.5}...".format(i**2, str(math.factorial(i**2))))
    >>>
    >>> @Timer(title="Second")
    ... def some_func():
    ...     time.sleep(1)
    >>>
    >>> with Timer(title="Some title") as t:
    ...    for i in range(42):
    ...        print("{}! = {:.5}...".format(i**2, str(math.factorial(i**2))))
    >>>
    >>> print(t.elapsed)
    """

    def __init__(self, title: str = "", print_title=True):
        self._title = title
        self._print = sys.stdout if print_title else print_title
        self._elapsed = 0

    def __float__(self) -> float:
        return float(self.elapsed)

    def __str__(self) -> str:
        """
        “informal” or nicely printable string representation of an object
        """
        return "Elapsed {}".format(self.__repr__())

    def __repr__(self) -> str:
        """
        “official” string representation of an object.
        """
        return str(float(self))

    def __enter__(self) -> 'Timer':
        self.start = time.perf_counter()

        return self

    def __exit__(self, *args):
        self._elapsed = time.perf_counter() - self.start
        if self._print:
            title = "[{}] ".format(self._title) if self._title else ""
            print('{title}Total time {total_seconds:.5f} seconds.'.format(title=title, total_seconds=self._elapsed),
                  file=self._print)

    @property
    def elapsed(self) -> float:
        return self._elapsed
<|endoftext|>