<commit_before>import Image
from datetime import datetime
from os import path
from imageop import ImageStretch, detect_img_type
import os

from django.db import models, transaction, IntegrityError
from django.conf import settings
from django.utils.translation import ugettext, ugettext_lazy as _
from django.contrib.contenttypes import generic
from django.contrib.sites.models import Site
from django.utils.safestring import mark_safe

from ella.core.models import Author, Source
from ella.core.managers import RelatedManager
from ella.core.box import Box
from ella.core.cache.utils import get_cached_object
from ella.utils.filemanipulation import file_rename
from ella.tagging.models import TaggedItem

# settings default
PHOTOS_FORMAT_QUALITY_DEFAULT = (
    (45, _('Low')),
    (65, _('Medium')),
    (75, _('Good')),
    (85, _('Better')),
    (95, _('High')),
)

PHOTOS_THUMB_DIMENSION_DEFAULT = (80,80)

PHOTOS_FORMAT_QUALITY = getattr(settings, 'PHOTOS_FORMAT_QUALITY', PHOTOS_FORMAT_QUALITY_DEFAULT)
PHOTOS_THUMB_DIMENSION = getattr(settings, 'PHOTOS_THUMB_DIMENSION', PHOTOS_THUMB_DIMENSION_DEFAULT)
PHOTOS_DO_URL_CHECK = getattr(settings, 'PHOTOS_DO_URL_CHECK', False)
CUSTOM_SUBDIR = getattr(settings, 'PHOTOS_CUSTOM_SUBDIR', '')
UPLOAD_TO = CUSTOM_SUBDIR and 'photos/%s/%%Y/%%m/%%d' % CUSTOM_SUBDIR or 'photos/%Y/%m/%d'

PHOTOS_TYPE_EXTENSION = {
    'JPEG': '.jpg',
    'PNG': '.png',
    'GIF': '.gif'
}

class PhotoBox(Box):
    def get_context(self):
        "Updates box context with photo-specific variables."
        cont = super(PhotoBox, self).get_context()
        cont.update({
                'title' : self.params.get('title', self.obj.title),
                'alt' : self.params.get('alt', ''),
                'description' : self.params.get('description', self.obj.description),
                'show_title' : self.params.get('show_title', ''),
                'show_description' : self.params.get('show_description', ''),
                'show_authors' : self.params.get('show_authors', ''),
                'show_detail' : self.params.get('show_detail', ''),
                'link_url': self.params.get('link_url', ''),
})
        return cont

class Photo(models.Model):
    "Represents original (unformated) photo."
    title = models.CharField(_('Title'), max_length=200)
    description = models.TextField(_('Description'), blank=True)
    slug = models.SlugField(_('Slug'), max_length=255)
    image = models.ImageField(upload_to=UPLOAD_TO , height_field='height', width_field='width') # save it to YYYY/MM/DD structure
    width = models.PositiveIntegerField(editable=False)
    height = models.PositiveIntegerField(editable=False)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors') , related_name='photo_set')
    source = models.ForeignKey(Source, blank=True, null=True, verbose_name=_('Source'))

    created = models.DateTimeField(default=datetime.now, editable=False)

    if 'ella.tagging' in settings.INSTALLED_APPS:
        tags = generic.GenericRelation(TaggedItem)

    def thumb(self):
        """
        Generates html and thumbnails for admin site.
        """
        thumbUrl = self.thumb_url()
        if not thumbUrl:
            return mark_safe("""%s""" % ugettext('Thumbnail not available'))
        return mark_safe("""""" % (self.image.url, settings.MEDIA_URL, thumbUrl, self.title))
    thumb.allow_tags = True

    def thumb_url(self):

        type = detect_img_type(self.image.path)
        if not type:
            return None

        # photos/2008/12/31/foo.jpg => photos/2008/12/31/thumb-foo.jpg
        thumb_name = path.dirname(self.image.name) + "/" + 'thumb-%s%s' % path.splitext(path.basename(self.image.name))

        storage = self.image.storage
        if not storage.exists(thumb_name):
            try:
                im = Image.open(self.image.path)
                im.thumbnail(PHOTOS_THUMB_DIMENSION, Image.ANTIALIAS)
                im.save(storage.path(thumb_name), type)
            except IOError:
                # TODO Logging something wrong
                return None
        return thumb_name

    def Box(self, box_type, nodelist):
        return PhotoBox(self, box_type, nodelist)

    @transaction.commit_on_success
    def save(self, force_insert=False, force_update=False):
        """Overrides models.Model.save.

        - Generates slug.
        - Saves image file.
        """
        # prefill the slug with the ID, it requires double save
        if not self.id:
            super(Photo, self).save(force_insert, force_update)
            self.slug = str(self.id) + '-' + self.slug
            force_insert, force_update = False, True
            image_changed = False
        else:
            old = Photo.objects.get(pk = self.pk)
            image_changed = old.image != self.image
        # rename image by slug
        imageType = detect_img_type(path.join(settings.MEDIA_ROOT, self.image.name))
        if imageType is not None:
            self.image = file_rename(self.image.name, self.slug, PHOTOS_TYPE_EXTENSION[ imageType ])
        # delete formatedphotos if new image was uploaded
        if image_changed:
            for f_photo in self.formatedphoto_set.all():
                f_photo.delete()
        super(Photo, self).save(force_insert, force_update)

    def ratio(self):
        "Return photo's width to height ratio"
        if self.height:
            return float(self.width) / self.height
        else:
            return None

    def get_formated_photo(self, format):
        "Return formated photo"
        format_object = Format.objects.get(name=format, site=settings.SITE_ID)
        try:
            formated_photo = get_cached_object(FormatedPhoto, photo=self, format=format_object)
        except FormatedPhoto.DoesNotExist:
            try:
                formated_photo = FormatedPhoto.objects.create(photo=self, format=format_object)
            except (IOError, SystemError, IntegrityError):
                return None

        return formated_photo

    def __unicode__(self):
        return self.title

    def get_absolute_url(self):
        return self.get_image_url()

    class Meta:
        verbose_name = _('Photo')
        verbose_name_plural = _('Photos')
        ordering = ('-created',)

class Format(models.Model):
    "Defines per-site photo sizes"
    name = models.CharField(_('Name'), max_length=80)
    max_width = models.PositiveIntegerField(_('Max width'))
    max_height = models.PositiveIntegerField(_('Max height'))
    flexible_height = models.BooleanField(_('Flexible height'), help_text=_(('Determines whether max_height is an absolute maximum, '
                                                                                'or the formatted photo can vary from max_height for flexible_max_height.'))
)
    flexible_max_height = models.PositiveIntegerField(_('Flexible max height'), blank=True, null=True)
    stretch = models.BooleanField(_('Stretch'))
    nocrop = models.BooleanField(_('Do not crop'))
    resample_quality = models.IntegerField(_('Resample quality'), choices=PHOTOS_FORMAT_QUALITY, default=85)
    site = models.ForeignKey(Site)

    def get_blank_img(self):
        """
        Return fake FormatedPhoto object to be used in templates when an error occurs in image generation.
        """
        out = {
            'width' : self.max_width,
            'height' : self.max_height,
            'filename' : 'img/empty/%s.png' % (self.name),
            'format' : self,
            'url' : settings.MEDIA_URL + 'img/empty/%s.png' % (self.name),
}
        return out

    def ratio(self):
        "Return photo's width to height ratio"
        return float(self.max_width) / self.max_height

    def __unicode__(self):
        return  u"%s (%sx%s) " % (self.name, self.max_width, self.max_height)

    class Meta:
        verbose_name = _('Format')
        verbose_name_plural = _('Formats')
        ordering = ('name', '-max_width',)

class FormatedPhoto(models.Model):
    "Specific photo of specific format."
    photo = models.ForeignKey(Photo)
    format = models.ForeignKey(Format)
    filename = models.CharField(max_length=300, editable=False) # derive local filename and url
    crop_left = models.PositiveIntegerField()
    crop_top = models.PositiveIntegerField()
    crop_width = models.PositiveIntegerField()
    crop_height = models.PositiveIntegerField()
    width = models.PositiveIntegerField(editable=False)
    height = models.PositiveIntegerField(editable=False)

    objects = RelatedManager()

    @property
    def url(self):
        "Returns url of the photo file."
        if not PHOTOS_DO_URL_CHECK:
            return settings.MEDIA_URL + self.filename

        if not path.exists(settings.MEDIA_ROOT):
            # NFS not available - we have no chance of creating it
            return self.format.get_blank_img()['url']

        if path.exists(path.join(settings.MEDIA_ROOT, self.filename)):
            # image exists
            return settings.MEDIA_URL + self.filename

        # image does not exist - try and create it
        try:
            self.generate()
            return settings.MEDIA_URL + self.filename
        except (IOError, SystemError):
            return self.format.get_blank_img()['url']

    def generate(self):
        "Generates photo file in current format"
        i = ImageStretch(filename=self.photo.image.path, formated_photo=self)
        stretched_photo = i.stretch_image()
        self.width, self.height = stretched_photo.size
        self.filename = self.file(relative=True)
        stretched_photo.save(self.file(), quality=self.format.resample_quality)

    def save(self, force_insert=False, force_update=False):
        """Overrides models.Model.save

        - Removes old file from the FS
        - Generates new file.
        """
        self.remove_file()
        self.generate()
        super(FormatedPhoto, self).save(force_insert, force_update)

    def delete(self):
        self.remove_file()
        super(FormatedPhoto, self).delete()

    def remove_file(self):
        if path.exists(path.join(settings.MEDIA_ROOT, self.file(relative=True))):
            os.remove(path.join(settings.MEDIA_ROOT, self.file(relative=True)))

    def file(self, relative=False):
        """ Method returns formated photo path - derived from format.id and source Photo filename """
        if relative:
            source_file = path.split(self.photo.image.name)
        else:
            source_file = path.split(self.photo.image.path)
        return path.join(source_file[0],  str (self.format.id) + '-' + source_file[1])

    def __unicode__(self):
        return u"%s - %s" % (self.filename, self.format)

    class Meta:
        verbose_name = _('Formated photo')
        verbose_name_plural = _('Formated photos')
        unique_together = (('photo','format'),)

<commit_msg>Code cleanup<commit_after>import Image
from datetime import datetime
from os import path
from imageop import ImageStretch, detect_img_type
import os

from django.db import models, transaction, IntegrityError
from django.conf import settings
from django.utils.translation import ugettext, ugettext_lazy as _
from django.contrib.contenttypes import generic
from django.contrib.sites.models import Site
from django.utils.safestring import mark_safe

from ella.core.models import Author, Source
from ella.core.managers import RelatedManager
from ella.core.box import Box
from ella.core.cache.utils import get_cached_object
from ella.utils.filemanipulation import file_rename
from ella.tagging.models import TaggedItem

# settings default
PHOTOS_FORMAT_QUALITY_DEFAULT = (
    (45, _('Low')),
    (65, _('Medium')),
    (75, _('Good')),
    (85, _('Better')),
    (95, _('High')),
)

PHOTOS_THUMB_DIMENSION_DEFAULT = (80,80)

PHOTOS_FORMAT_QUALITY = getattr(settings, 'PHOTOS_FORMAT_QUALITY', PHOTOS_FORMAT_QUALITY_DEFAULT)
PHOTOS_THUMB_DIMENSION = getattr(settings, 'PHOTOS_THUMB_DIMENSION', PHOTOS_THUMB_DIMENSION_DEFAULT)
PHOTOS_DO_URL_CHECK = getattr(settings, 'PHOTOS_DO_URL_CHECK', False)
CUSTOM_SUBDIR = getattr(settings, 'PHOTOS_CUSTOM_SUBDIR', '')
UPLOAD_TO = CUSTOM_SUBDIR and 'photos/%s/%%Y/%%m/%%d' % CUSTOM_SUBDIR or 'photos/%Y/%m/%d'

PHOTOS_TYPE_EXTENSION = {
    'JPEG': '.jpg',
    'PNG': '.png',
    'GIF': '.gif'
}

class PhotoBox(Box):
    def get_context(self):
        "Updates box context with photo-specific variables."
        cont = super(PhotoBox, self).get_context()
        cont.update({
                'title' : self.params.get('title', self.obj.title),
                'alt' : self.params.get('alt', ''),
                'description' : self.params.get('description', self.obj.description),
                'show_title' : self.params.get('show_title', ''),
                'show_description' : self.params.get('show_description', ''),
                'show_authors' : self.params.get('show_authors', ''),
                'show_detail' : self.params.get('show_detail', ''),
                'link_url': self.params.get('link_url', ''),
})
        return cont

class Photo(models.Model):
    "Represents original (unformated) photo."
    title = models.CharField(_('Title'), max_length=200)
    description = models.TextField(_('Description'), blank=True)
    slug = models.SlugField(_('Slug'), max_length=255)
    image = models.ImageField(upload_to=UPLOAD_TO , height_field='height', width_field='width') # save it to YYYY/MM/DD structure
    width = models.PositiveIntegerField(editable=False)
    height = models.PositiveIntegerField(editable=False)

    # Authors and Sources
    authors = models.ManyToManyField(Author, verbose_name=_('Authors') , related_name='photo_set')
    source = models.ForeignKey(Source, blank=True, null=True, verbose_name=_('Source'))

    created = models.DateTimeField(default=datetime.now, editable=False)

    if 'ella.tagging' in settings.INSTALLED_APPS:
        tags = generic.GenericRelation(TaggedItem)

    def thumb(self):
        """
        Generates html and thumbnails for admin site.
        """
        thumbUrl = self.thumb_url()
        if not thumbUrl:
            return mark_safe("""%s""" % ugettext('Thumbnail not available'))
        return mark_safe("""""" % (self.image.url, thumbUrl, self.title))
    thumb.allow_tags = True

    def thumb_url(self):
        """
        Generates thumbnail for admin site and returns its url
        """
        type = detect_img_type(self.image.path)
        if not type:
            return None

        # photos/2008/12/31/foo.jpg => photos/2008/12/31/thumb-foo.jpg
        thumb_name = path.dirname(self.image.name) + "/" + 'thumb-%s' % path.basename(self.image.name)

        storage = self.image.storage
        if not storage.exists(thumb_name):
            try:
                im = Image.open(self.image.path)
                im.thumbnail(PHOTOS_THUMB_DIMENSION, Image.ANTIALIAS)
                im.save(storage.path(thumb_name), type)
            except IOError:
                # TODO Logging something wrong
                return None
        return storage.url(thumb_name)

    def Box(self, box_type, nodelist):
        return PhotoBox(self, box_type, nodelist)

    @transaction.commit_on_success
    def save(self, force_insert=False, force_update=False):
        """Overrides models.Model.save.

        - Generates slug.
        - Saves image file.
        """
        # prefill the slug with the ID, it requires double save
        if not self.id:
            super(Photo, self).save(force_insert, force_update)
            self.slug = str(self.id) + '-' + self.slug
            force_insert, force_update = False, True
            image_changed = False
        else:
            old = Photo.objects.get(pk = self.pk)
            image_changed = old.image != self.image
        # rename image by slug
        imageType = detect_img_type(path.join(settings.MEDIA_ROOT, self.image.name))
        if imageType is not None:
            self.image = file_rename(self.image.name, self.slug, PHOTOS_TYPE_EXTENSION[ imageType ])
        # delete formatedphotos if new image was uploaded
        if image_changed:
            for f_photo in self.formatedphoto_set.all():
                f_photo.delete()
        super(Photo, self).save(force_insert, force_update)

    def ratio(self):
        "Return photo's width to height ratio"
        if self.height:
            return float(self.width) / self.height
        else:
            return None

    def get_formated_photo(self, format):
        "Return formated photo"
        format_object = Format.objects.get(name=format, site=settings.SITE_ID)
        try:
            formated_photo = get_cached_object(FormatedPhoto, photo=self, format=format_object)
        except FormatedPhoto.DoesNotExist:
            try:
                formated_photo = FormatedPhoto.objects.create(photo=self, format=format_object)
            except (IOError, SystemError, IntegrityError):
                return None

        return formated_photo

    def __unicode__(self):
        return self.title

    def get_absolute_url(self):
        return self.get_image_url()

    class Meta:
        verbose_name = _('Photo')
        verbose_name_plural = _('Photos')
        ordering = ('-created',)

class Format(models.Model):
    "Defines per-site photo sizes"
    name = models.CharField(_('Name'), max_length=80)
    max_width = models.PositiveIntegerField(_('Max width'))
    max_height = models.PositiveIntegerField(_('Max height'))
    flexible_height = models.BooleanField(_('Flexible height'), help_text=_(('Determines whether max_height is an absolute maximum, '
                                                                                'or the formatted photo can vary from max_height for flexible_max_height.'))
)
    flexible_max_height = models.PositiveIntegerField(_('Flexible max height'), blank=True, null=True)
    stretch = models.BooleanField(_('Stretch'))
    nocrop = models.BooleanField(_('Do not crop'))
    resample_quality = models.IntegerField(_('Resample quality'), choices=PHOTOS_FORMAT_QUALITY, default=85)
    site = models.ForeignKey(Site)

    def get_blank_img(self):
        """
        Return fake FormatedPhoto object to be used in templates when an error occurs in image generation.
        """
        out = {
            'width' : self.max_width,
            'height' : self.max_height,
            'filename' : 'img/empty/%s.png' % (self.name),
            'format' : self,
            'url' : settings.MEDIA_URL + 'img/empty/%s.png' % (self.name),
}
        return out

    def ratio(self):
        "Return photo's width to height ratio"
        return float(self.max_width) / self.max_height

    def __unicode__(self):
        return  u"%s (%sx%s) " % (self.name, self.max_width, self.max_height)

    class Meta:
        verbose_name = _('Format')
        verbose_name_plural = _('Formats')
        ordering = ('name', '-max_width',)

class FormatedPhoto(models.Model):
    "Specific photo of specific format."
    photo = models.ForeignKey(Photo)
    format = models.ForeignKey(Format)
    filename = models.CharField(max_length=300, editable=False) # derive local filename and url
    crop_left = models.PositiveIntegerField()
    crop_top = models.PositiveIntegerField()
    crop_width = models.PositiveIntegerField()
    crop_height = models.PositiveIntegerField()
    width = models.PositiveIntegerField(editable=False)
    height = models.PositiveIntegerField(editable=False)

    objects = RelatedManager()

    @property
    def url(self):
        "Returns url of the photo file."
        if not PHOTOS_DO_URL_CHECK:
            return settings.MEDIA_URL + self.filename

        if not path.exists(settings.MEDIA_ROOT):
            # NFS not available - we have no chance of creating it
            return self.format.get_blank_img()['url']

        if path.exists(path.join(settings.MEDIA_ROOT, self.filename)):
            # image exists
            return settings.MEDIA_URL + self.filename

        # image does not exist - try and create it
        try:
            self.generate()
            return settings.MEDIA_URL + self.filename
        except (IOError, SystemError):
            return self.format.get_blank_img()['url']

    def generate(self):
        "Generates photo file in current format"
        i = ImageStretch(filename=self.photo.image.path, formated_photo=self)
        stretched_photo = i.stretch_image()
        self.width, self.height = stretched_photo.size
        self.filename = self.file(relative=True)
        stretched_photo.save(self.file(), quality=self.format.resample_quality)

    def save(self, force_insert=False, force_update=False):
        """Overrides models.Model.save

        - Removes old file from the FS
        - Generates new file.
        """
        self.remove_file()
        self.generate()
        super(FormatedPhoto, self).save(force_insert, force_update)

    def delete(self):
        self.remove_file()
        super(FormatedPhoto, self).delete()

    def remove_file(self):
        if path.exists(path.join(settings.MEDIA_ROOT, self.file(relative=True))):
            os.remove(path.join(settings.MEDIA_ROOT, self.file(relative=True)))

    def file(self, relative=False):
        """ Method returns formated photo path - derived from format.id and source Photo filename """
        if relative:
            source_file = path.split(self.photo.image.name)
        else:
            source_file = path.split(self.photo.image.path)
        return path.join(source_file[0],  str (self.format.id) + '-' + source_file[1])

    def __unicode__(self):
        return u"%s - %s" % (self.filename, self.format)

    class Meta:
        verbose_name = _('Formated photo')
        verbose_name_plural = _('Formated photos')
        unique_together = (('photo','format'),)

<|endoftext|>


<commit_before>import unittest
from collections import OrderedDict
import sys
import os
import datetime
import hashlib
import time
import subprocess
sys.path.insert(0, "../common")

import argparse
import tempfile
import json

import leds
import beeper
from wbmqtt import WBMQTT

from test_ms_common import TestCO2, TestBuzzer, TestIlluminance, TestSPL, TestTH, TestHStrict, TestEEPROMPersistence, parse_comma_separated_set, SerialDeviceHandler, SerialDriverHandler, ModbusDeviceTestLog, MSTesterBase, Test1Wire
import test_ms_common

class TestIlluminanceMSW2(TestIlluminance):
    MAX_AMBIENT = 50
    ILLUMINATED_DIFF = 4390
    ILLUMINATED_DIFF_ERR = 0.08

class TestSPLMSW(TesterstSPL):
    SOUND_LEVEL_MIN = 75.4
    SOUND_LEVEL_MAX = 81.5
    AMBIENT_MAX = 63

class Tester(MSTesterBase):
    MQTT_DEVICE_ID = 'wbmsw2-test'
    CONFIG_FNAME = "wbmsw2.conf"

    def init_mapping(self):
        self.mapping = OrderedDict([
                (TestSPLMSW, 2),
                (TestIlluminanceMSW2, 3),
                (TestTH, 4),
                (TestHStrict, 8),
                (TestCO2, 5),
                (TestBuzzer, 6),
                (TestEEPROMPersistence, 1),
        ])

if __name__ == '__main__':
    while 1:
        try:
            test_ms_common.wbmqtt = WBMQTT()
            wbmqtt = test_ms_common.wbmqtt
            wbmqtt.watch_device('am2320')
            Tester().main()
        finally:
            wbmqtt.close()

        while 1:
            e = raw_input("press Enter to continue or Control+C to exit")
            if e == '':
                break

        print "\n" * 5

<commit_msg>test-suite: wider allowed lux range for WB-MSW2<commit_after>import unittest
from collections import OrderedDict
import sys
import os
import datetime
import hashlib
import time
import subprocess
sys.path.insert(0, "../common")

import argparse
import tempfile
import json

import leds
import beeper
from wbmqtt import WBMQTT

from test_ms_common import TestCO2, TestBuzzer, TestIlluminance, TestSPL, TestTH, TestHStrict, TestEEPROMPersistence, parse_comma_separated_set, SerialDeviceHandler, SerialDriverHandler, ModbusDeviceTestLog, MSTesterBase, Test1Wire
import test_ms_common

class TestIlluminanceMSW2(TestIlluminance):
    MAX_AMBIENT = 50
    ILLUMINATED_DIFF = 4400
    ILLUMINATED_DIFF_ERR = 0.10

class TestSPLMSW(TestSPL):
    SOUND_LEVEL_MIN = 75.4
    SOUND_LEVEL_MAX = 81.5
    AMBIENT_MAX = 63

class Tester(MSTesterBase):
    MQTT_DEVICE_ID = 'wbmsw2-test'
    CONFIG_FNAME = "wbmsw2.conf"

    def init_mapping(self):
        self.mapping = OrderedDict([
                (TestSPLMSW, 2),
                (TestIlluminanceMSW2, 3),
                (TestTH, 4),
                (TestHStrict, 8),
                (TestCO2, 5),
                (TestBuzzer, 6),
                (TestEEPROMPersistence, 1),
        ])

if __name__ == '__main__':
    while 1:
        try:
            test_ms_common.wbmqtt = WBMQTT()
            wbmqtt = test_ms_common.wbmqtt
            wbmqtt.watch_device('am2320')
            Tester().main()
        finally:
            wbmqtt.close()

        while 1:
            e = raw_input("press Enter to continue or Control+C to exit")
            if e == '':
                break

        print "\n" * 5

<|endoftext|>


<commit_before>from django.contrib import admin
from django.contrib.contenttypes import generic
from django.forms import models as modelforms
from django.forms import model_to_dict
from django import forms
from ella.tagging.models import Tag, TaggedItem
from django.utils.translation import ugettext_lazy as _
from ella.tagging.fields import SuggestTagAdminField, TagPriorityAdminField
from django.contrib.contenttypes.models import ContentType
from ella.ellaadmin.options import EllaAdminOptionsMixin

class TagInlineForm(modelforms.ModelForm):
    class Meta:
        model = TaggedItem

    def __init__(self, *args, **kwargs):
        super(TagInlineForm, self).__init__(*args, **kwargs)

class TagInlineFormset(modelforms.BaseModelFormSet):
    def __init__(self, data=None, files=None, instance=None, save_as_new=None):
        self.queryset = None
        self.instance = instance
        tags_by_prio = {}
        initial = []
        if instance:
            qs = TaggedItem.objects.get_for_object(instance)
            for ti in qs:
                if not ti.priority in tags_by_prio:
                    tags_by_prio[ ti.priority ] = []
                tags_by_prio[ ti.priority ].append(ti.tag)
            # sort aplhabeticaly
            map(lambda x: x.sort(), tags_by_prio.values())
            for k, v in tags_by_prio.items():
                initial.append({'priority': k, 'tag': v})
        super(modelforms.BaseModelFormSet, self).__init__(
            data=data,
            files=files,
            prefix='tagged_item_',
            initial=initial
)

    def save(self):
        self.new_objects = []
        self.changed_objects = []
        self.deleted_objects = []
        for d in self.cleaned_data:
            if 'priority' not in d or 'tag' not in d:
                continue
            obj = self.instance
            ct = ContentType.objects.get_for_model(obj)
            if hasattr(obj, 'category'):
                tis = TaggedItem.objects.filter(
                    content_type=ct,
                    object_id=obj._get_pk_val(),
                    category=obj.category,
                    priority=d['priority']
)
            else:
                tis = TaggedItem.objects.filter(
                    content_type=ct,
                    object_id=obj._get_pk_val(),
                    priority=d['priority']
)

            tags_before = set(map(lambda x: x.tag, tis))
            saved_tags = []
            for t in d['tag']:
                if t in saved_tags:
                    continue
                ti, created = Tag.objects.add_tag(obj, t.name, int(d['priority']))
                saved_tags.append(ti.tag)
            st = set(saved_tags)
            if tags_before == st:
                continue
            diff = tags_before - st
            while diff:
                ti = TaggedItem.objects.get(
                    tag=diff.pop(),
                    content_type=ct,
                    object_id=obj._get_pk_val(),
#                    category=obj.category,
                    priority=d['priority']
)
                ti.delete()

class TaggingInlineOptionsSimple(admin.TabularInline):
    model = TaggedItem
    extra = 0

class TaggingInlineOptions(EllaAdminOptionsMixin, generic.GenericTabularInline):
    fields = ('tag', 'priority',)
    raw_id_fields = ('tag',)
    model = TaggedItem
    extra = 2
    id_field_name = 'object_id'
    ct_field_name = 'content_type'
    formset = TagInlineFormset

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'tag':
            return SuggestTagAdminField(db_field, **kwargs)
        elif db_field.name == 'priority':
            return TagPriorityAdminField(db_field, **kwargs)
        return super(TaggingInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class TagOptions(admin.ModelAdmin):
    ordernig = ('name',)
    list_display = ('name',)
    inlines = (TaggingInlineOptionsSimple,)
    search_fields = ('name',)

class TaggedItemOptions(admin.ModelAdmin):
    list_display = ('tag', 'content_type',)

admin.site.register(Tag, TagOptions)
admin.site.register(TaggedItem, TaggedItemOptions)
<commit_msg>removed priority field from TaggingInlineOptions, priority is automatically set to PRIMARY_TAG, fixes #776<commit_after>from django.contrib import admin
from django.contrib.contenttypes import generic
from django.forms import models as modelforms
from django.forms import model_to_dict
from django import forms
from ella.tagging.models import Tag, TaggedItem
from django.utils.translation import ugettext_lazy as _
from ella.tagging.fields import SuggestTagAdminField, TagPriorityAdminField
from ella.tagging.utils import PRIMARY_TAG
from django.contrib.contenttypes.models import ContentType
from ella.ellaadmin.options import EllaAdminOptionsMixin

class TagInlineForm(modelforms.ModelForm):
    class Meta:
        model = TaggedItem

    def __init__(self, *args, **kwargs):
        super(TagInlineForm, self).__init__(*args, **kwargs)

class TagInlineFormset(modelforms.BaseModelFormSet):
    def __init__(self, data=None, files=None, instance=None, save_as_new=None):
        self.queryset = None
        self.instance = instance
        tags_by_prio = {}
        initial = []
        if instance:
            qs = TaggedItem.objects.get_for_object(instance)
            for ti in qs:
                if not ti.priority in tags_by_prio:
                    tags_by_prio[ ti.priority ] = []
                tags_by_prio[ ti.priority ].append(ti.tag)
            # sort aplhabeticaly
            map(lambda x: x.sort(), tags_by_prio.values())
            for k, v in tags_by_prio.items():
                initial.append({'priority': k, 'tag': v})
        super(modelforms.BaseModelFormSet, self).__init__(
            data=data,
            files=files,
            prefix='tagged_item_',
            initial=initial
)

    def save(self):
        self.new_objects = []
        self.changed_objects = []
        self.deleted_objects = []
        for d in self.cleaned_data:
            if 'tag' not in d:
                continue
            if 'priority' not in d:
                d['priority']=PRIMARY_TAG
            obj = self.instance
            ct = ContentType.objects.get_for_model(obj)
            if hasattr(obj, 'category'):
                tis = TaggedItem.objects.filter(
                    content_type=ct,
                    object_id=obj._get_pk_val(),
                    category=obj.category,
                    priority=d['priority']
)
            else:
                tis = TaggedItem.objects.filter(
                    content_type=ct,
                    object_id=obj._get_pk_val(),
                    priority=d['priority']
)

            tags_before = set(map(lambda x: x.tag, tis))
            saved_tags = []
            for t in d['tag']:
                if t in saved_tags:
                    continue
                ti, created = Tag.objects.add_tag(obj, t.name, int(d['priority']))
                saved_tags.append(ti.tag)
            st = set(saved_tags)
            if tags_before == st:
                continue
            diff = tags_before - st
            while diff:
                ti = TaggedItem.objects.get(
                    tag=diff.pop(),
                    content_type=ct,
                    object_id=obj._get_pk_val(),
#                    category=obj.category,
                    priority=d['priority']
)
                ti.delete()

class TaggingInlineOptionsSimple(admin.TabularInline):
    model = TaggedItem
    extra = 0

class TaggingInlineOptions(EllaAdminOptionsMixin, generic.GenericTabularInline):
    fields = ('tag',)# 'priority',)
    raw_id_fields = ('tag',)
    model = TaggedItem
    extra = 1
    id_field_name = 'object_id'
    ct_field_name = 'content_type'
    formset = TagInlineFormset

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'tag':
            return SuggestTagAdminField(db_field, **kwargs)
    #    elif db_field.name == 'priority':
    #       return TagPriorityAdminField(db_field, **kwargs)
        return super(TaggingInlineOptions, self).formfield_for_dbfield(db_field, **kwargs)

class TagOptions(admin.ModelAdmin):
    ordernig = ('name',)
    list_display = ('name',)
    inlines = (TaggingInlineOptionsSimple,)
    search_fields = ('name',)

class TaggedItemOptions(admin.ModelAdmin):
    list_display = ('tag', 'content_type',)

admin.site.register(Tag, TagOptions)
admin.site.register(TaggedItem, TaggedItemOptions)
<|endoftext|>


<commit_before>import numpy as np
from scipy.spatial.distance import cdist
from time import time
import mrpt

# Generate synthetic test data
k = 10; n_queries = 100
data = np.dot(np.random.rand(int(1e5),5), np.random.rand(5,100)).astype('float32')
queries = np.dot(np.random.rand(n_queries,5), np.random.rand(5,100)).astype('float32')

# Solve exact nearest neighbors with standard methods from scipy and numpy for reference
exact_search_time = time()
exact_neighbors = np.zeros((n_queries, k))
for i in range(n_queries):
    exact_neighbors[i] = np.argsort(cdist([queries[i]], data))[0,:k]
exact_search_time = time() - exact_search_time

# Offline phase: Indexing the data. This might take some time.
indexing_time = time()
index = mrpt.MRPTIndex(data, depth=5, n_trees=100)
index.build()
indexing_time = time() - indexing_time

# Online phase: Finding nearest neighbors stupendously fast.
approximate_search_time = time()
approximate_neighbors = np.zeros((n_queries, k))
for i in range(n_queries):
    approximate_neighbors[i] = index.ann(queries[i], k, votes_required=4)
approximate_search_time = time() - approximate_search_time

# Print some stats
print ('Indexing time: %1.3f seconds' %indexing_time)
print ('%d approximate queries time: %1.3f seconds' %(n_queries, approximate_search_time))
print ('%d exact queries time: %1.3f seconds' %(n_queries, exact_search_time))

correct_neighbors = 0
for i in range(n_queries):
    correct_neighbors += len(np.intersect1d(exact_neighbors[i], approximate_neighbors[i]))
print ('Average recall: %1.2f.' %(float(correct_neighbors)/(n_queries*k)))
<commit_msg>Delete demo.py<commit_after><|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
import json

from opensearchsdk.client import Client

def list_app(client):
    apps = client.app.list(page=1, page_size=1)
    print(apps)
    '''
    {
        u'status': u'OK',
        u'total': u'1',
        u'result': [
                        {
                            u'description': u'test',
                            u'created': u'1447753400',
                            u'id': u'119631',
                            u'name': u'test'
                        }
                    ],
        u'request_id': u'1447866418002866600607068'
    }
    '''

def create_app(client):
    app = client.app.create('test_app', 'a')
    print(app)
    '''
    {
        u'status': u'OK',
        u'result': {
                        u'index_name': u'test_app'
                    },
        u'request_id': u'1449146731021309800470027'
    }

    '''

def creat_data_process(client):
    table_name = 'test_table'
    items = [
        {
            "cmd": "add",
            "timestamp": 1401342874777,
            "fields": {
                "id": "1",
                "title": "This is the title",
                "body": "This is the body"
            }
        },
        {
            "cmd": "update",
            "timestamp": 1401342874778,
            "fields": {
                "id": "2",
                "title": "This is the new title"
            }
        },
        {
            "cmd": "delete",
            "fields": {
                "id": "3"
            }
        }
    ]
    items = json.dumps(items)
    data_ret = client.data.create(table_name, items)
    print(data_ret)
    '''
        {
            u'status': u'OK',
            u'total': u'1',
            u'result': [
                            {
                                u'description': u'test',
                                u'created': u'1447753400',
                                u'id': u'119631',
                                u'name': u'test'
                            }
                        ],
            u'request_id': u'1448079259020634800692900'
        }
    '''

def search(client):
    # simple search
    body = dict(index_name='test_app',
                query='query=id:"0"',
                fetch_fields='testname',
                first_formula_name='default',
                formula_name='default',
                summary='summary_snipped:1,summary_field:title,summ'
                        'ary_element:high,summary_len:32,summary_el'
                        'lipsis:...;summary_snipped:2,summary_field'
                        ':body,summary_element:high,summary_len:60,s'
                        'ummary_ellipsis:...')

    def simple_search():
        res = client.search.search(**body)
        print(res)
        '''
        {
            u'status': u'OK',
            u'tracer': u'',
            u'errors': [],
            u'result': {
                            u'searchtime': 0.005986,
                            u'items': [
                                        {
                                            u'index_name': u'test_app',
                                            u'testname': u'0',
                                        }
                                    ],
                            u'facet': [],
                            u'viewtotal': 1,
                            u'num': 1,
                            u'total': 1
                        },
            u'request_id': u'144932743917790790023680'
        }
        '''

    def combine_search_1():
        # first combine search
        body['scroll'] = '1h'
        body['search_type'] = 'scan'
        res = client.search.search(**body)
        print(res)
        '''
        {
            u'status': u'OK',
            u'tracer': u'',
            u'errors': [],
            u'result': {
                            u'scroll_id': u'eJyNjk1LxDAQhn9NcizdZNnSQw+rXkTUk
                            +dQk2k7ki8zqW7315sWhRUVhAwZnnmHZ45aA9EdLLemexpz87
                            Ab8f7t0b2fw5kr21NWek4UUjdgoozeQyKdgrVouCqdVXCKKqO
                            Dbrfft1KKRtZt2/ABsp7UgGANdRko+94BR2/gpNZ2Y6qPkb/O
                            kJZOBz/gyOTNEJLrM5PHFwqeiUN5W6KM0BTMhKhLcZqd68vi5
                            6/IY4ywJnZMXH/R7YDCMmYLFxwsOPCrZsJxuhhY8AVK8S1rMR
                            JS4VVVMXn1Uyl+UT4Hs/zfeKj/NH4A34SVcg==',
                            u'searchtime': 0.002997,
                            u'items': [],
                            u'facet': [],
                            u'viewtotal': 1,
                            u'num': 0,
                            u'total': 1
                        },
            u'request_id': u'144932913017790788000125'
        }
        '''
        return res.get('result').get('scroll_id')

    def combine_search_2(scroll_id):
        del body['scroll']
        body['scroll_id'] = scroll_id
        res = client.search.search(**body)
        print(res)
        '''
        the same as simple search
        '''

    simple_search()
    scroll_id = combine_search_1()
    combine_search_2(scroll_id)

def suggest(client):
    res = client.suggest.suggest('的', 'test_app', 's2')
    print(res)
    '''
    {
        u'errors': [
            {
                u'message': u'nosuggesterdataonline',
                u'code': 2422
            }
        ],
        u'suggestions': [],
        u'searchtime': 0.034475,
        u'request_id': u'144937817217790789042028'
    }
    '''

def index_refactor(client):
    # only refactor index
    def refactor_only():
        res = client.index.refactor('test_app')
        print(res)
        '''
        {
            u'status': u'OK',
            u'result': {
                u'task_id': u'159469'
            },
            u'request_id': u'1449378626069143700426870'
        }
        '''
    def refactor_import_data():
        #  refactor index and import data
        res = client.index.refactor('test_app', 'import', 'id')
        print(res)

    # refactor_only()
    refactor_import_data()

def get_error_log(client):
    res = client.log.get('test_app', '1', 1, 'ASC')
    print(res)
    '''
    {
        u'status': u'OK',
        u'result': {
            u'count': 0,
            u'items': [],
            u'page': 1,
            u'page_size': u'1'
        },
        u'request_id': u'1449379073037249300677090'
    }
    '''

if __name__ == '__main__':
    from opensearchsdk import mykey
    import logging
    LOG = logging.basicConfig(level=logging.DEBUG,
                              format="[%(asctime)s] %(name)s:"
                                     "%(levelname)s: %(message)s")
    url = 'http://opensearch-cn-hangzhou.aliyuncs.com'
    key = mykey.KEY['key_secrete']
    key_id = mykey.KEY['key_id']
    client = Client(url, key, key_id)
    # list_app(client)
    # creat_data_process(client)
    # create_app(client)
    # search(client)
    # suggest(client)
    # index_refactor(client)
    # get_error_log(client)
<commit_msg>fix nits in demo<commit_after># -*- encoding: utf-8 -*-
import json

from opensearchsdk.client import Client

def list_app(client):
    apps = client.app.list(page=1, page_size=1)
    print(apps)
    '''
    {
        u'status': u'OK',
        u'total': u'1',
        u'result': [
                        {
                            u'description': u'test',
                            u'created': u'1447753400',
                            u'id': u'119631',
                            u'name': u'test'
                        }
                    ],
        u'request_id': u'1447866418002866600607068'
    }
    '''

def create_app(client):
    app = client.app.create('test_app', 'a')
    print(app)
    '''
    {
        u'status': u'OK',
        u'result': {
                        u'index_name': u'test_app'
                    },
        u'request_id': u'1449146731021309800470027'
    }

    '''

def creat_data_process(client):
    table_name = 'test_table'
    items = [
        {
            "cmd": "add",
            "timestamp": 1401342874777,
            "fields": {
                "id": "1",
                "title": "This is the title",
                "body": "This is the body"
            }
        },
        {
            "cmd": "update",
            "timestamp": 1401342874778,
            "fields": {
                "id": "2",
                "title": "This is the new title"
            }
        },
        {
            "cmd": "delete",
            "fields": {
                "id": "3"
            }
        }
    ]
    items = json.dumps(items)
    data_ret = client.data.create(table_name, items)
    print(data_ret)
    '''
        {
            u'status': u'OK',
            u'total': u'1',
            u'result': [
                            {
                                u'description': u'test',
                                u'created': u'1447753400',
                                u'id': u'119631',
                                u'name': u'test'
                            }
                        ],
            u'request_id': u'1448079259020634800692900'
        }
    '''

def search(client):
    # simple search
    body = dict(index_name='test_app',
                query='query=id:"0"',
                fetch_fields='testname',
                first_formula_name='default',
                formula_name='default',
                summary='summary_snipped:1,summary_field:title,summ'
                        'ary_element:high,summary_len:32,summary_el'
                        'lipsis:...;summary_snipped:2,summary_field'
                        ':body,summary_element:high,summary_len:60,s'
                        'ummary_ellipsis:...')

    def simple_search():
        res = client.search.search(**body)
        print(res)
        '''
        {
            u'status': u'OK',
            u'tracer': u'',
            u'errors': [],
            u'result': {
                            u'searchtime': 0.005986,
                            u'items': [
                                        {
                                            u'index_name': u'test_app',
                                            u'testname': u'0',
                                        }
                                    ],
                            u'facet': [],
                            u'viewtotal': 1,
                            u'num': 1,
                            u'total': 1
                        },
            u'request_id': u'144932743917790790023680'
        }
        '''

    def combine_search_1():
        # first combine search
        body['scroll'] = '1h'
        body['search_type'] = 'scan'
        res = client.search.search(**body)
        print(res)
        '''
        {
            u'status': u'OK',
            u'tracer': u'',
            u'errors': [],
            u'result': {
                            u'scroll_id': u'eJyNjk1LxDAQhn9NcizdZNnSQw+rXkTUk
                            +dQk2k7ki8zqW7315sWhRUVhAwZnnmHZ45aA9EdLLemexpz87
                            Ab8f7t0b2fw5kr21NWek4UUjdgoozeQyKdgrVouCqdVXCKKqO
                            Dbrfft1KKRtZt2/ABsp7UgGANdRko+94BR2/gpNZ2Y6qPkb/O
                            kJZOBz/gyOTNEJLrM5PHFwqeiUN5W6KM0BTMhKhLcZqd68vi5
                            6/IY4ywJnZMXH/R7YDCMmYLFxwsOPCrZsJxuhhY8AVK8S1rMR
                            JS4VVVMXn1Uyl+UT4Hs/zfeKj/NH4A34SVcg==',
                            u'searchtime': 0.002997,
                            u'items': [],
                            u'facet': [],
                            u'viewtotal': 1,
                            u'num': 0,
                            u'total': 1
                        },
            u'request_id': u'144932913017790788000125'
        }
        '''
        return res.get('result').get('scroll_id')

    def combine_search_2(scroll_id):
        del body['scroll']
        body['scroll_id'] = scroll_id
        res = client.search.search(**body)
        print(res)
        '''
        the same as simple search
        '''

    simple_search()
    scroll_id = combine_search_1()
    combine_search_2(scroll_id)

def suggest(client):
    res = client.suggest.suggest('的', 'test_app', 's2')
    print(res)
    '''
    {
        u'errors': [
            {
                u'message': u'nosuggesterdataonline',
                u'code': 2422
            }
        ],
        u'suggestions': [],
        u'searchtime': 0.034475,
        u'request_id': u'144937817217790789042028'
    }
    '''

def index_refactor(client):
    # only refactor index
    def refactor_only():
        res = client.index.refactor('test_app')
        print(res)
        '''
        {
            u'status': u'OK',
            u'result': {
                u'task_id': u'159469'
            },
            u'request_id': u'1449378626069143700426870'
        }
        '''
    def refactor_import_data():
        #  refactor index and import data
        res = client.index.refactor('test_app', 'import', 'id')
        print(res)

    # refactor_only()
    refactor_import_data()

def get_error_log(client):
    res = client.log.get('test_app', '1', 1, 'ASC')
    print(res)
    '''
    {
        u'status': u'OK',
        u'result': {
            u'count': 0,
            u'items': [],
            u'page': 1,
            u'page_size': u'1'
        },
        u'request_id': u'1449379073037249300677090'
    }
    '''

if __name__ == '__main__':
    import mykey
    import logging
    LOG = logging.basicConfig(level=logging.DEBUG,
                              format="[%(asctime)s] %(name)s:"
                                     "%(levelname)s: %(message)s")
    url = 'http://opensearch-cn-hangzhou.aliyuncs.com'
    key = mykey.KEY['key_secrete']
    key_id = mykey.KEY['key_id']
    client = Client(url, key, key_id)
    # list_app(client)
    # creat_data_process(client)
    # create_app(client)
    # search(client)
    # suggest(client)
    # index_refactor(client)
    # get_error_log(client)
<|endoftext|>


<commit_before>"""
A simple grid world environment
"""
import numpy as np
import random
from ..common import MDP
from ..exceptions import EpisodeDoneError, InvalidActionError
from ..actions import LEFT, RIGHT, UP, DOWN
from .helper_utilities import flatten_state, unflatten_state

class GridWorldMDP(MDP):
    def __init__(self, P, R, gamma, p0, terminal_states, size, seed=1337, skip_check=False,
                 convert_terminal_states_to_ints=False):
        """
        (!) if terminal_states is not empty then there will be an absorbing state. So
            the actual number of states will be size x size + 1
            if there is a terminal state, it should be the last one.
        :param P: Transition matrix |S| x |A| x |S|
        :param R: Transition matrix |S| x |A|
        :param gamma: discount factor
        :param p0: initial starting distribution
        :param terminal_states: Must be a list of (x,y) tuples.  use skip_terminal_state_conversion if giving ints
        :param size: the size of the grid world (i.e there are size x size (+ 1)= |S| states)
        :param seed:
        :param skip_check:
        """
        if not convert_terminal_states_to_ints:
            terminal_states = list(map(lambda tupl: int(size * tupl[0] + tupl[1]), terminal_states))
        super().__init__(P, R, gamma, p0, terminal_states, seed=seed, skip_check=skip_check)
        self.size =  size
        self.human_state = (None, None)
        self.reset()
        self.has_absorbing_state = len(terminal_states) > 0
        self.human_state = self.unflatten_state(self.current_state)

    def flatten_state(self, state):
        """Flatten state (x,y) into a one hot vector"""
        return flatten_state(state, self.size, self.state_space)

    def unflatten_state(self, onehot):
        """Unflatten a one hot vector into a (x,y) pair"""
        return unflatten_state(onehot, self.size, self.has_absorbing_state)

    def step(self, action):
        state, reward, done, info = super().step(action)
        self.human_state = self.unflatten_state(self.current_state)
        return state, reward, done, info

    def set_current_state_to(self, tuple_state):
        return super().set_current_state_to(self.flatten_state(tuple_state).argmax())<commit_msg>Make sure reset sets the human state on every reset.<commit_after>"""
A simple grid world environment
"""
import numpy as np
import random
from ..common import MDP
from ..exceptions import EpisodeDoneError, InvalidActionError
from ..actions import LEFT, RIGHT, UP, DOWN
from .helper_utilities import flatten_state, unflatten_state

class GridWorldMDP(MDP):
    def __init__(self, P, R, gamma, p0, terminal_states, size, seed=1337, skip_check=False,
                 convert_terminal_states_to_ints=False):
        """
        (!) if terminal_states is not empty then there will be an absorbing state. So
            the actual number of states will be size x size + 1
            if there is a terminal state, it should be the last one.
        :param P: Transition matrix |S| x |A| x |S|
        :param R: Transition matrix |S| x |A|
        :param gamma: discount factor
        :param p0: initial starting distribution
        :param terminal_states: Must be a list of (x,y) tuples.  use skip_terminal_state_conversion if giving ints
        :param size: the size of the grid world (i.e there are size x size (+ 1)= |S| states)
        :param seed:
        :param skip_check:
        """
        if not convert_terminal_states_to_ints:
            terminal_states = list(map(lambda tupl: int(size * tupl[0] + tupl[1]), terminal_states))
        super().__init__(P, R, gamma, p0, terminal_states, seed=seed, skip_check=skip_check)
        self.size =  size
        self.human_state = (None, None)
        self.reset()
        self.has_absorbing_state = len(terminal_states) > 0

    def reset(self):
        super().reset()
        self.human_state = self.unflatten_state(self.current_state)

    def flatten_state(self, state):
        """Flatten state (x,y) into a one hot vector"""
        return flatten_state(state, self.size, self.state_space)

    def unflatten_state(self, onehot):
        """Unflatten a one hot vector into a (x,y) pair"""
        return unflatten_state(onehot, self.size, self.has_absorbing_state)

    def step(self, action):
        state, reward, done, info = super().step(action)
        self.human_state = self.unflatten_state(self.current_state)
        return state, reward, done, info

    def set_current_state_to(self, tuple_state):
        return super().set_current_state_to(self.flatten_state(tuple_state).argmax())
<|endoftext|>


<commit_before>import pystan
import cPickle as pickle
import numpy as np

def init_fn():
    n_sne = stan_data["n_sne"]
    n_samples = stan_data["n_samples"]

    three_simplex = np.random.random(size = 3)
    three_simplex /= sum(three_simplex)

    obs_x1s = np.array([item[1] for item in stan_data["obs_mBx1c"]])
    obs_colors = np.array([item[2] for item in stan_data["obs_mBx1c"]])

    return {"MB": np.random.random()*0.2 - 19.1,
            "Om": np.random.random()*0.4 + 0.1,
            "alpha_angle_low": np.arctan(np.random.random()*0.2),
            "alpha_angle_high": np.arctan(np.random.random()*0.2),
            "beta_angle_blue": np.arctan(np.random.random()*0.5 + 2.5),
            "beta_angle_red": np.arctan(np.random.random()*0.5 + 2.5),
            "delta_0": np.random.random()*0.05,
            "delta_h": 0.5,
            "calibs": np.random.normal(size = stan_data["n_calib"])*0.5,

            "log10_sigma_int": np.log10(np.random.random(size = n_samples)*0.1 + 0.1),
            "mBx1c_int_variance": three_simplex,
            "Lmat": [[1.0, 0.0, 0.0],
                     [np.random.random()*0.1 - 0.05, np.random.random()*0.1 + 0.7, 0.0],
                     [np.random.random()*0.1 - 0.05, np.random.random()*0.1 - 0.05, np.random.random()*0.1 + 0.7]],

            "true_c": np.random.random(size = n_sne)*0.02 - 0.01 + np.clip(obs_colors, -0.2, 1.0),
            "true_x1": np.random.random(size = n_sne)*0.2 - 0.1 + obs_x1s,

            "x1_star": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.05,
            "c_star": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.05,
            "delta_c": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.2 - 0.1,
            "log10_R_x1": np.random.random(size = n_samples)*0.5 - 0.25,
            "log10_R_c": np.random.random(size = n_samples)*0.4 - 1.2,

            "outl_frac": np.random.random()*0.02 + 0.01
        }

################################################# Main Program ###################################################

stan_data = pickle.load(open("stan_data.pickle", 'rb'))

fit = pystan.stan(file="stan_code.txt", data=stan_data,
                  iter=50, chains=1, n_jobs = 1, init = init_fn)

print fit
<commit_msg>Update demo.py<commit_after>import pystan
import pickle
import numpy as np

def init_fn():
    n_sne = stan_data["n_sne"]
    n_samples = stan_data["n_samples"]

    three_simplex = np.random.random(size = 3)
    three_simplex /= sum(three_simplex)

    obs_x1s = np.array([item[1] for item in stan_data["obs_mBx1c"]])
    obs_colors = np.array([item[2] for item in stan_data["obs_mBx1c"]])

    return {"MB": np.random.random()*0.2 - 19.1,
            "Om": np.random.random()*0.4 + 0.1,
            "alpha_angle_low": np.arctan(np.random.random()*0.2),
            "alpha_angle_high": np.arctan(np.random.random()*0.2),
            "beta_angle_blue": np.arctan(np.random.random()*0.5 + 2.5),
            "beta_angle_red": np.arctan(np.random.random()*0.5 + 2.5),
            "delta_0": np.random.random()*0.05,
            "delta_h": 0.5,
            "calibs": np.random.normal(size = stan_data["n_calib"])*0.5,

            "log10_sigma_int": np.log10(np.random.random(size = n_samples)*0.1 + 0.1),
            "mBx1c_int_variance": three_simplex,
            "Lmat": [[1.0, 0.0, 0.0],
                     [np.random.random()*0.1 - 0.05, np.random.random()*0.1 + 0.7, 0.0],
                     [np.random.random()*0.1 - 0.05, np.random.random()*0.1 - 0.05, np.random.random()*0.1 + 0.7]],

            "true_c": np.random.random(size = n_sne)*0.02 - 0.01 + np.clip(obs_colors, -0.2, 1.0),
            "true_x1": np.random.random(size = n_sne)*0.2 - 0.1 + obs_x1s,

            "x1_star": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.05,
            "c_star": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.05,
            "delta_c": np.random.random(size = [n_samples, stan_data["n_x1c_star"]])*0.2 - 0.1,
            "log10_R_x1": np.random.random(size = n_samples)*0.5 - 0.25,
            "log10_R_c": np.random.random(size = n_samples)*0.4 - 1.2,

            "outl_frac": np.random.random()*0.02 + 0.01
        }

################################################# Main Program ###################################################

with open("stan_data.pickle", 'rb') as f:
    u = pickle._Unpickler(f)
    u.encoding = 'latin1'
    stan_data = u.load()

fit = pystan.stan(file="stan_code.txt", data=stan_data,
                  iter=50, chains=1, n_jobs = 1, init = init_fn)

print fit
<|endoftext|>


<commit_before>ProcName = raw_input("Process name: ")
MyMem = PyHadesMem.MemoryMgr(ProcName)
RegionIter = PyHadesMem.RegionIter(MyMem)
for MyRegion in RegionIter:
  print("")
  print("GetBase: " + hex(MyRegion.GetBase()))
  print("GetAllocBase: " + hex(MyRegion.GetAllocBase()))
  print("GetAllocProtect: " + hex(MyRegion.GetAllocProtect()))
  print("GetSize: " + hex(MyRegion.GetSize()))
  print("GetState: " + hex(MyRegion.GetState()))
  print("GetProtect: " + hex(MyRegion.GetProtect()))
  print("GetType: " + hex(MyRegion.GetType()))
<commit_msg>* Output fix.<commit_after>ProcName = raw_input("Process name: ")
MyMem = PyHadesMem.MemoryMgr(ProcName)
RegionIter = PyHadesMem.RegionIter(MyMem)
for MyRegion in RegionIter:
  print("")
  print("Base: " + hex(MyRegion.GetBase()))
  print("AllocBase: " + hex(MyRegion.GetAllocBase()))
  print("AllocProtect: " + hex(MyRegion.GetAllocProtect()))
  print("Size: " + hex(MyRegion.GetSize()))
  print("State: " + hex(MyRegion.GetState()))
  print("Protect: " + hex(MyRegion.GetProtect()))
  print("Type: " + hex(MyRegion.GetType()))
<|endoftext|>


<commit_before>#   Copyright Alexander Baranin 2016

import sys
import os.path
import time
import sfml.window

_import_modules = (
    ('EngineCore', 'engine.EngineCore'),
    ('Logging', 'engine.Logging'),
    ('IOBroker', 'engine.IOBroker'),
    ('EngineConsole', 'engine.EngineConsole'))

def onLoad(core):
    Logging.logMessage('ModuleStamp is loading')
    IOBroker.register_handler(handle_focus, sfml.window.FocusEvent)
    EngineConsole.register_extension(toggle_command, 'stamp')
    initialize()

def onUnload():
    Logging.logMessage('ModuleStamp is unloading')
    EngineConsole.unregister_extension(toggle_command, 'stamp')
    IOBroker.unregister_handler(handle_focus, sfml.window.FocusEvent)

_active = True

module_hash = {}

def initialize():
    for mdl in EngineCore.loaded_modules:
        module_hash[mdl] = time.ctime(os.path.getmtime(EngineCore.loaded_modules[mdl].__file__))

def handle_focus(event, wnd):
    if _active and event.gained:
        # let's loop over loaded modules and reload those with new file stamps
        for mdl in EngineCore.loaded_modules:
            new_time = time.ctime(os.path.getmtime(EngineCore.loaded_modules[mdl].__file__))
            if not mdl in module_hash:
                module_hash[mdl] = new_time
            else:
                old_time = module_hash[mdl]
                if new_time > old_time:
                    # let's reload module
                    EngineCore.reloadModule(mdl)
                    module_hash[mdl] = new_time

def toggle_command(cmds):
    global active
    if cmds[1] in ('on', '1'):
        _active = True
    elif cmds[1] in ('off', '0'):
        _active = False
<commit_msg>report of stamper<commit_after>#   Copyright Alexander Baranin 2016

import sys
import os.path
import time
import sfml.window

_import_modules = (
    ('EngineCore', 'engine.EngineCore'),
    ('Logging', 'engine.Logging'),
    ('IOBroker', 'engine.IOBroker'),
    ('EngineConsole', 'engine.EngineConsole'))

def onLoad(core):
    Logging.logMessage('ModuleStamp is loading')
    IOBroker.register_handler(handle_focus, sfml.window.FocusEvent)
    EngineConsole.register_extension(toggle_command, 'stamp')
    initialize()

def onUnload():
    Logging.logMessage('ModuleStamp is unloading')
    EngineConsole.unregister_extension(toggle_command, 'stamp')
    IOBroker.unregister_handler(handle_focus, sfml.window.FocusEvent)

_active = True

module_hash = {}

def initialize():
    for mdl in EngineCore.loaded_modules:
        module_hash[mdl] = time.ctime(os.path.getmtime(EngineCore.loaded_modules[mdl].__file__))

def handle_focus(event, wnd):
    if _active and event.gained:
        Logging.logMessage('ModuleStamp handling files')
        # let's loop over loaded modules and reload those with new file stamps
        for mdl in EngineCore.loaded_modules:
            new_time = time.ctime(os.path.getmtime(EngineCore.loaded_modules[mdl].__file__))
            if not mdl in module_hash:
                module_hash[mdl] = new_time
            else:
                old_time = module_hash[mdl]
                if new_time > old_time:
                    # let's reload module
                    EngineCore.reloadModule(mdl)
                    module_hash[mdl] = new_time

def toggle_command(cmds):
    global active
    if cmds[1] in ('on', '1'):
        _active = True
    elif cmds[1] in ('off', '0'):
        _active = False
<|endoftext|>


<commit_before>#!/usr/bin/env python

import socket
import sys
from thread import *
from time import sleep

if len(sys.argv) < 2:
	print 'Usage python light_server.py [LIGHT THRESHOLD]'
	sys.exit(-1)

MY_TCP_IP = ''
MY_TCP_PORT = 5000
BUFFER_SIZE = 8

ARD_TCP_IP = '192.168.1.10'
ARD_TCP_PORT = 5001

THRS = int(sys.argv[1])

def writeLog(message):
	log = open('interactions.log', 'a')
	log.write(message)
	log.close()

def num(s):
	try:
		return int(s)
	except ValueError:
		return float(s)

def sendmessage(mes):
	writeLog('[ARD-S] Sending '+mes+' to arduino\n')
	global ARD_TCP_IP
	global ARD_TCP_PORT
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.connect((ARD_TCP_IP, ARD_TCP_PORT))
	s.send(mes)
	data = s.recv(BUFFER_SIZE)
	s.close()
	writeLog('[ARD-R] Received '+data+' from arduino\n')
	return data

def getlight():
	return num(sendmessage('REQ'))

def setlight(val):
	return sendmessage(val)

def blink(times):
	for x in range(0, times):
		setlight('ON')
		sleep(0.2)
		setlight('OFF')
		sleep(0.2)

def getval(val):
	temp = val
	if temp > 999:
		temp = 999
	u = temp%10
	temp = (temp-u)/10
	d = temp%10
	c = (temp-d)/10
	ret = (c,d,u)
	writeLog('\n[GAM-V] Value for game is '+str(ret)+'\n')
	return ret

def guessval(guess):
	writeLog('[GAM-G] Value of guess is '+str(guess)+'\n')
	global LHT_V
	blinks = 0
	for x in range (0,3):
		if LHT_V[x] == guess[x]:
			blinks = blinks + 1
	blink(blinks)

def recvfromclient(conn, addr):
	global THRS
	global LHT_V
	while True:
		data = conn.recv(BUFFER_SIZE)
		if not data: break
		tokens = data.split()
		data = data.rstrip()
		writeLog('\n[CLI-R] Received '+data+' from client\n')
		if data == 'STS':
			data = sendmessage('STS')
			writeLog('[CLI-S] Sending '+data+' to client\n');
			conn.send(data+'\n')
		elif data == 'OFF':
			data = setlight(data)
			writeLog('[CLI-S] Sending OK to client\n');
			conn.send('OK\n')
		elif data == 'ON':
			if getlight() < THRS:
				data = setlight(data)
				writeLog('[CLI-S] Sending OK to client\n');
				conn.send('OK\n')
			else:
				writeLog('[CLI-S] Sending NOP to client\n');
				conn.send('NOP\n')
		elif data == 'RES':
			LHT_V = getval(getlight())
			conn.send('OK\n')
		elif tokens[0] == 'G':
			guessval((int(tokens[1]), int(tokens[2]), int(tokens[3])))
			conn.send('OK\n')
		else:
			writeLog('[CLI-S] Sending NOK to client\n');
			conn.send('NOK\n')
		break
	conn.close()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((MY_TCP_IP, MY_TCP_PORT))
s.listen(1)

writeLog('\n[INF-S] Light Server started on port 5000\n\n')

LHT_V = getval(getlight())

while 1:
	conn, addr = s.accept()
	recvfromclient(conn,addr)
<commit_msg>Changed game blink to turn light off if the light is on<commit_after>#!/usr/bin/env python

import socket
import sys
from thread import *
from time import sleep

if len(sys.argv) < 2:
	print 'Usage python light_server.py [LIGHT THRESHOLD]'
	sys.exit(-1)

MY_TCP_IP = ''
MY_TCP_PORT = 5000
BUFFER_SIZE = 8

ARD_TCP_IP = '192.168.1.10'
ARD_TCP_PORT = 5001

THRS = int(sys.argv[1])

def writeLog(message):
	log = open('interactions.log', 'a')
	log.write(message)
	log.close()

def num(s):
	try:
		return int(s)
	except ValueError:
		return float(s)

def sendmessage(mes):
	writeLog('[ARD-S] Sending '+mes+' to arduino\n')
	global ARD_TCP_IP
	global ARD_TCP_PORT
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	s.connect((ARD_TCP_IP, ARD_TCP_PORT))
	s.send(mes)
	data = s.recv(BUFFER_SIZE)
	s.close()
	writeLog('[ARD-R] Received '+data+' from arduino\n')
	return data

def getlight():
	return num(sendmessage('REQ'))

def setlight(val):
	return sendmessage(val)

def blink(times):
	data = sendmessage('STS')
	if data == '0':
		for x in range(0, times):
			setlight('ON')
			sleep(0.2)
			setlight('OFF')
			sleep(0.2)
	else:
		for x in range(0, times):
			setlight('OFF')
			sleep(0.2)
			setlight('ON')
			sleep(0.2)

def getval(val):
	temp = val
	if temp > 999:
		temp = 999
	u = temp%10
	temp = (temp-u)/10
	d = temp%10
	c = (temp-d)/10
	ret = (c,d,u)
	writeLog('\n[GAM-V] Value for game is '+str(ret)+'\n')
	return ret

def guessval(guess):
	writeLog('[GAM-G] Value of guess is '+str(guess)+'\n')
	global LHT_V
	blinks = 0
	for x in range (0,3):
		if LHT_V[x] == guess[x]:
			blinks = blinks + 1
	blink(blinks)

def recvfromclient(conn, addr):
	global THRS
	global LHT_V
	while True:
		data = conn.recv(BUFFER_SIZE)
		if not data: break
		tokens = data.split()
		data = data.rstrip()
		writeLog('\n[CLI-R] Received '+data+' from client\n')
		if data == 'STS':
			data = sendmessage('STS')
			writeLog('[CLI-S] Sending '+data+' to client\n');
			conn.send(data+'\n')
		elif data == 'OFF':
			data = setlight(data)
			writeLog('[CLI-S] Sending OK to client\n');
			conn.send('OK\n')
		elif data == 'ON':
			if getlight() < THRS:
				data = setlight(data)
				writeLog('[CLI-S] Sending OK to client\n');
				conn.send('OK\n')
			else:
				writeLog('[CLI-S] Sending NOP to client\n');
				conn.send('NOP\n')
		elif data == 'RES':
			LHT_V = getval(getlight())
			conn.send('OK\n')
		elif tokens[0] == 'G':
			guessval((int(tokens[1]), int(tokens[2]), int(tokens[3])))
			conn.send('OK\n')
		else:
			writeLog('[CLI-S] Sending NOK to client\n');
			conn.send('NOK\n')
		break
	conn.close()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((MY_TCP_IP, MY_TCP_PORT))
s.listen(1)

writeLog('\n[INF-S] Light Server started on port 5000\n\n')

LHT_V = getval(getlight())

while 1:
	conn, addr = s.accept()
	recvfromclient(conn,addr)
<|endoftext|>


<commit_before># https://www.w3.org/TR/filter-effects/#funcdef-hue-rotate
# https://www.w3.org/TR/SVG/filters.html#TransferFunctionElementAttributes

import numpy as np

from . import rgb_clamp

CONST_PART = np.array([[0.213, 0.715, 0.072],
                       [0.213, 0.715, 0.072],
                       [0.213, 0.715, 0.072]])

COS_PART = np.array([[0.787, -0.715, -0.072],
                     [-0.213, 0.285, -0.072],
                     [-0.213, -0.715, 0.928]])

SIN_PART = np.array([[-0.213, -0.715, 0.928],
                     [0.143, 0.140, -0.283],
                     [-0.787, 0.715, 0.072]])

def calc_rgb(rgb, deg):
    return rgb_clamp((_calc_rotation(deg) * np.array(rgb)).sum(1))

def _deg_to_rad(deg):
    return np.pi * deg / 180

def _calc_rotation(deg):
    rad = _deg_to_rad(deg)
    cos_part = COS_PART * np.cos(rad)
    sin_part = SIN_PART * np.sin(rad)
    return CONST_PART + cos_part + sin_part
<commit_msg>add docstring to converters.hue_rotate.calc_rgb()<commit_after># https://www.w3.org/TR/filter-effects/#funcdef-hue-rotate
# https://www.w3.org/TR/SVG/filters.html#TransferFunctionElementAttributes

import numpy as np

from . import rgb_clamp

CONST_PART = np.array([[0.213, 0.715, 0.072],
                       [0.213, 0.715, 0.072],
                       [0.213, 0.715, 0.072]])

COS_PART = np.array([[0.787, -0.715, -0.072],
                     [-0.213, 0.285, -0.072],
                     [-0.213, -0.715, 0.928]])

SIN_PART = np.array([[-0.213, -0.715, 0.928],
                     [0.143, 0.140, -0.283],
                     [-0.787, 0.715, 0.072]])

def calc_rgb(rgb, deg):
    """Return a hue rotation applied RGB value of passed color.

    THe calculation is based on the definition found at
    https://www.w3.org/TR/filter-effects/#funcdef-hue-rotate
    :param rgb: The Original RGB value before the rotation.
    :type rgb: (int, int, int)
    :param deg: Degrees of rotation (0 to 360)
    :type deg: float
    :return: Hue rotation applied RGB value
    :rtype: (int, int, int)
    """
    return rgb_clamp((_calc_rotation(deg) * np.array(rgb)).sum(1))

def _deg_to_rad(deg):
    return np.pi * deg / 180

def _calc_rotation(deg):
    rad = _deg_to_rad(deg)
    cos_part = COS_PART * np.cos(rad)
    sin_part = SIN_PART * np.sin(rad)
    return CONST_PART + cos_part + sin_part
<|endoftext|>


<commit_before># coding=utf-8

"""
Runs third party scripts and collects their output.

Scripts need to be +x and should output metrics in the form of

```
metric.path.a 1
metric.path.b 2
metric.path.c 3
```

They are not passed any arguments and if they return an error code,
no metrics are collected.

#### Dependencies

 * [subprocess](http://docs.python.org/library/subprocess.html)

"""

import diamond.collector
import diamond.convertor
import os
import subprocess

class UserScriptsCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(UserScriptsCollector,
                            self).get_default_config_help()
        config_help.update({
            'scripts_path': "Path to find the scripts to run",
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(UserScriptsCollector, self).get_default_config()
        config.update({
            'path':         '.',
            'scripts_path': '/etc/diamond/user_scripts/',
            'floatprecision': 4,
        })
        return config

    def collect(self):
        scripts_path = self.config['scripts_path']
        if not os.access(scripts_path, os.R_OK):
            return None
        for script in os.listdir(scripts_path):
            absolutescriptpath = os.path.join(scripts_path, script)
            executable = os.access(absolutescriptpath, os.X_OK)
            is_file = os.path.isfile(absolutescriptpath)
            if is_file:
                if not executable:
                    self.log.info("%s is not executable" % absolutescriptpath)
                    continue
            else:
                # Don't bother logging skipped non-file files (typically
                # directories)
                continue
            out = None
            self.log.debug("Executing %s" % absolutescriptpath)
            try:
                proc = subprocess.Popen([absolutescriptpath],
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
                (out, err) = proc.communicate()
            except subprocess.CalledProcessError, e:
                self.log.error("%s error launching: %s; skipping" %
                               (absolutescriptpath, e))
                continue
            if proc.returncode:
                self.log.error("%s return exit value %s; skipping" %
                               (absolutescriptpath, proc.returncode))
            if not out:
                self.log.info("%s return no output" % absolutescriptpath)
                continue
            if err:
                self.log.error("%s return error output: %s" %
                               (absolutescriptpath, err))
            # Use filter to remove empty lines of output
            for line in filter(None, out.split('\n')):
                # Ignore invalid lines
                try:
                    name, value = line.split()
                    float(value)
                except ValueError:
                    self.log.error("%s returned error output: %s" %
                                   (absolutescriptpath, line))
                    continue
                name, value = line.split()
                floatprecision = 0
                if "." in value:
                    floatprecision = self.config['floatprecision']
                self.publish(name, value, precision=floatprecision)
<commit_msg>improve userscripts collector error messages<commit_after># coding=utf-8

"""
Runs third party scripts and collects their output.

Scripts need to be +x and should output metrics in the form of

```
metric.path.a 1
metric.path.b 2
metric.path.c 3
```

They are not passed any arguments and if they return an error code,
no metrics are collected.

#### Dependencies

 * [subprocess](http://docs.python.org/library/subprocess.html)

"""

import diamond.collector
import diamond.convertor
import os
import subprocess

class UserScriptsCollector(diamond.collector.Collector):

    def get_default_config_help(self):
        config_help = super(UserScriptsCollector,
                            self).get_default_config_help()
        config_help.update({
            'scripts_path': "Path to find the scripts to run",
        })
        return config_help

    def get_default_config(self):
        """
        Returns the default collector settings
        """
        config = super(UserScriptsCollector, self).get_default_config()
        config.update({
            'path':         '.',
            'scripts_path': '/etc/diamond/user_scripts/',
            'floatprecision': 4,
        })
        return config

    def collect(self):
        scripts_path = self.config['scripts_path']
        if not os.access(scripts_path, os.R_OK):
            return None
        for script in os.listdir(scripts_path):
            absolutescriptpath = os.path.join(scripts_path, script)
            executable = os.access(absolutescriptpath, os.X_OK)
            is_file = os.path.isfile(absolutescriptpath)
            if is_file:
                if not executable:
                    self.log.info("%s is not executable" % absolutescriptpath)
                    continue
            else:
                # Don't bother logging skipped non-file files (typically
                # directories)
                continue
            out = None
            self.log.debug("Executing %s" % absolutescriptpath)
            try:
                proc = subprocess.Popen([absolutescriptpath],
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
                (out, err) = proc.communicate()
            except subprocess.CalledProcessError, e:
                self.log.error("%s error launching: %s; skipping" %
                               (absolutescriptpath, e))
                continue
            if proc.returncode:
                self.log.error("%s return exit value %s; skipping" %
                               (absolutescriptpath, proc.returncode))
            if not out:
                self.log.info("%s return no output" % absolutescriptpath)
                continue
            if err:
                self.log.error("%s returned error output (stderr): %s" %
                               (absolutescriptpath, err))
            # Use filter to remove empty lines of output
            for line in filter(None, out.split('\n')):
                # Ignore invalid lines
                try:
                    name, value = line.split()
                    float(value)
                except ValueError:
                    self.log.error("%s returned invalid/unparsable output: %s" %
                                   (absolutescriptpath, line))
                    continue
                name, value = line.split()
                floatprecision = 0
                if "." in value:
                    floatprecision = self.config['floatprecision']
                self.publish(name, value, precision=floatprecision)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This work was created by participants in the DataONE project, and is
# jointly copyrighted by participating institutions in DataONE. For
# more information on DataONE, see our web site at http://dataone.org.
#
#   Copyright 2009-2016 DataONE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Django template context processors

Before rendering a template, Django calls context processors as set up in
settings_default.TEMPLATE_CONTEXT_PROCESSORS. The context processors are
functions that are expected to return a dict which will be merged into the
environment available to the template.
"""

import django.conf

def global_settings(request):
  """Expose some values from settings.py to templates
  """
  return {
    'BASE_URL': django.conf.settings.BASE_URL,
  }

<commit_msg>Add dynamic links to GMN home page from 404 and 500 HTML templates<commit_after># -*- coding: utf-8 -*-

# This work was created by participants in the DataONE project, and is
# jointly copyrighted by participating institutions in DataONE. For
# more information on DataONE, see our web site at http://dataone.org.
#
#   Copyright 2009-2016 DataONE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Django template context processors

Before rendering a template, Django calls context processors as set up in
settings_default.TEMPLATE_CONTEXT_PROCESSORS. The context processors are
functions that are expected to return a dict which will be merged into the
environment available to the template.
"""

import django.conf

def global_settings(request):
  """Expose some values from settings.py to templates
  """
  return {
    'BASE_URL': django.conf.settings.BASE_URL,
  }
<|endoftext|>


<commit_before>from falafel.mappers import autofs_conf
from falafel.tests import context_wrap

AUTOFS_CONF = """
#
# Define default options for autofs.
# Heavily edited for brevity by removing most of the useless comments
#
[ autofs ]
#
# timeout - set the default mount timeout in secons. The internal
#	    program default is 10 minutes, but the default installed
#	    configuration overrides this and sets the timeout to 5
#	    minutes to be consistent with earlier autofs releases.
#
timeout = 300
#
# browse_mode - maps are browsable by default.
#
browse_mode = no
#
# mount_nfs_default_protocol - specify the default protocol used by
# 			       mount.nfs(8). Since we can't identify
# 			       the default automatically we need to
# 			       set it in our configuration.
#
#mount_nfs_default_protocol = 3
mount_nfs_default_protocol = 4
#
# Define global options for the amd parser within autofs.
#
[ amd ]
#
# Override the internal default with the same timeout that
# is used by the override in the autofs configuration, sanity
# only change.
#
dismount_interval = 300
#
# map_type = file
#
"""

class TestAutoFSConf():
    def test_standard_autofs_conf(self):
        cfg = autofs_conf.AutoFSConf(context_wrap(AUTOFS_CONF))

        assert cfg.get("autofs").get("timeout") == '300'
        assert cfg.get_key("autofs","timeout") == '300'
        assert cfg.get_key("autofs","browse_mode") == 'no'
        assert cfg.get_key("autofs","mount_nfs_default_protocol") == '4'
        assert cfg.get_key("amd","dismount_interval") == '300'

        # Check that things set in comments do not appear
        assert cfg.get_key("amd", "map_type") == None
        # Check that nonexistent sections do not appear
        assert cfg.get("nfs") == None
<commit_msg>flake8 improvements to tests<commit_after>from falafel.mappers import autofs_conf
from falafel.tests import context_wrap

AUTOFS_CONF = """
#
# Define default options for autofs.
# Heavily edited for brevity by removing most of the useless comments
#
[ autofs ]
#
# timeout - set the default mount timeout in secons. The internal
#	    program default is 10 minutes, but the default installed
#	    configuration overrides this and sets the timeout to 5
#	    minutes to be consistent with earlier autofs releases.
#
timeout = 300
#
# browse_mode - maps are browsable by default.
#
browse_mode = no
#
# mount_nfs_default_protocol - specify the default protocol used by
# 			       mount.nfs(8). Since we can't identify
# 			       the default automatically we need to
# 			       set it in our configuration.
#
#mount_nfs_default_protocol = 3
mount_nfs_default_protocol = 4
#
# Define global options for the amd parser within autofs.
#
[ amd ]
#
# Override the internal default with the same timeout that
# is used by the override in the autofs configuration, sanity
# only change.
#
dismount_interval = 300
#
# map_type = file
#
"""

class TestAutoFSConf():
    def test_standard_autofs_conf(self):
        cfg = autofs_conf.AutoFSConf(context_wrap(AUTOFS_CONF))

        assert cfg.get("autofs").get("timeout") == '300'
        assert cfg.get_key("autofs", "timeout") == '300'
        assert cfg.get_key("autofs", "browse_mode") == 'no'
        assert cfg.get_key("autofs", "mount_nfs_default_protocol") == '4'
        assert cfg.get_key("amd", "dismount_interval") == '300'

        # Check that things set in comments do not appear
        assert cfg.get_key("amd", "map_type") is None
        # Check that nonexistent sections do not appear
        assert cfg.get("nfs") is None
<|endoftext|>


<commit_before><commit_msg>Improve logging in openstack poller.<commit_after><|endoftext|>


<commit_before>from typing import Callable

import jax
import jax.tree_util as jtu

from .filters import combine, is_array, partition
from .module import Module, Static

def _is_struct(x):
    return is_array(x) or (hasattr(x, "shape") and hasattr(x, "dtype"))

class _MakeJaxpr(Module):
    fn: Callable

    def __call__(self, *args, **kwargs):
        dynamic, static = partition((args, kwargs), _is_struct)
        dynamic_flat, dynamic_treedef = jtu.tree_flatten(dynamic)

        def _fn(*_dynamic_flat):
            _dynamic = jtu.tree_unflatten(dynamic_treedef, _dynamic_flat)
            _args, _kwargs = combine(_dynamic, static)
            _out = self.fn(*_args, **_kwargs)
            _out_dynamic, _out_static = partition(_out, is_array)
            return _out_dynamic, Static(_out_static)

        jaxpr, out_struct = jax.make_jaxpr(_fn, return_shape=True)(*dynamic_flat)
        dynamic_out_struct, static_out = out_struct
        static_out = static_out.value
        return jaxpr, dynamic_out_struct, static_out

def filter_make_jaxpr(fun):
    """As `jax.make_jaxpr`, but accepts arbitrary PyTrees as input and output.

    **Arguments:**

    - `fun`: The function `fun(*arg, **kwargs)` whose jaxpr is to be computed. Its
        positional and keyword arguments may be anything, as can its return value.

    **Returns:**

    A wrapped version of `fun`, that when applied to example arguments
    `*args, **kwargs`, will return a 3-tuple of:
    - A `ClosedJaxpr` representing the evaluation of that function on those arguments.
    - A `PyTree[jax.ShapeDtypeStruct]` representing the output shape and dtype of the
        result.
    - A `PyTree[Any]` representing any non-array outputs from `fun`.

    The example arguments may be either JAX/NumPy arrays, or anything with `.shape` and
    `.dtype` fields (typically `jax.ShapeDtypeStruct`s). Python builtins (`int`,
    `float`, `bool`, `complex`) are treated as static inputs; wrap them in JAX/NumPy
    arrays if you would like them to be traced.
    """
    return _MakeJaxpr(fun)
<commit_msg>Doc fix<commit_after>from typing import Callable

import jax
import jax.tree_util as jtu

from .filters import combine, is_array, partition
from .module import Module, Static

def _is_struct(x):
    return is_array(x) or (hasattr(x, "shape") and hasattr(x, "dtype"))

class _MakeJaxpr(Module):
    fn: Callable

    def __call__(self, *args, **kwargs):
        dynamic, static = partition((args, kwargs), _is_struct)
        dynamic_flat, dynamic_treedef = jtu.tree_flatten(dynamic)

        def _fn(*_dynamic_flat):
            _dynamic = jtu.tree_unflatten(dynamic_treedef, _dynamic_flat)
            _args, _kwargs = combine(_dynamic, static)
            _out = self.fn(*_args, **_kwargs)
            _out_dynamic, _out_static = partition(_out, is_array)
            return _out_dynamic, Static(_out_static)

        jaxpr, out_struct = jax.make_jaxpr(_fn, return_shape=True)(*dynamic_flat)
        dynamic_out_struct, static_out = out_struct
        static_out = static_out.value
        return jaxpr, dynamic_out_struct, static_out

def filter_make_jaxpr(fun):
    """As `jax.make_jaxpr`, but accepts arbitrary PyTrees as input and output.

    **Arguments:**

    - `fun`: The function `fun(*arg, **kwargs)` whose jaxpr is to be computed. Its
        positional and keyword arguments may be anything, as can its return value.

    **Returns:**

    A wrapped version of `fun`, that when applied to example arguments
    `*args, **kwargs`, will return a 3-tuple of:

    - A `ClosedJaxpr` representing the evaluation of that function on those arguments.
    - A `PyTree[jax.ShapeDtypeStruct]` representing the output shape and dtype of the
        result.
    - A `PyTree[Any]` representing any non-array outputs from `fun`.

    The example arguments to be traced may be anything with `.shape` and `.dtype`
    fields (typically JAX arrays, NumPy arrays, of `jax.ShapeDtypeStruct`s). All
    other argments are treated statically. In particular, Python builtins (`bool`,
    `int`, `float`, `complex`) are treated as static inputs; wrap them in JAX/NumPy
    arrays if you would like them to be traced.
    """
    return _MakeJaxpr(fun)
<|endoftext|>