<commit_before>#!/usr/bin/python

import unittest
import math
import argparse
import sys
import copy

class result:
	def __init__(self, histogram, frequencies, passed):
		self.histogram = histogram
		self.frequencies = frequencies
		self.passed = passed

	def __str__(self):
		output = self.get_format()
		bfd = benfords_frequncy_distribution()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(self.histogram[digit]) + "\t"
			output += str(self.frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(self.passed[digit]) + "\n"
		return output

	def get_format(self):
		return "d:\th\tf\tb\tp\n"

def digits():
	return {x: 0 for x in range(1, 10)}

def benfords_frequncy_distribution():
	return {x: math.log(float(1) + (float(1)/float(x)), float(10)) for x in range (1, 10)}

def calculate_frequncies_from_histogram(histogram):
	count = 0
	for c in histogram.itervalues():
		count += c
	frequencies  = dict()
	for k,v in histogram.iteritems():
		frequencies[k] = float(v) * float(100) / float(count)
	return frequencies

def skip_minus(word):
	if word[0] == '-': return 1
	return 0

def skip_zeros(word):
	i = 0;
	while i < len(word) and word[i].isdigit() and int(word[i]) == 0: i += 1
	return i

def calculate_benfords_histogram_from_text(text):
	digits_histogram = digits()
	for word in text.split():
		i = 0
		i += skip_minus(word)
		i += skip_zeros(word)
		if i < len(word) and word[i].isdigit(): # TODO is not a date
			digits_histogram[int(word[i])] += 1
	return digits_histogram

def expected_distribution_test_for_digits_frequencies(expected, frequencies, treshold = 0):
	return {x: math.fabs(frequencies[x] - expected[x]) <= treshold for x in range(1, 10)}

class benfords_law:

	def __init__(self, text, treshold = 0):
		self.text = text
		self.treshold = treshold

	def get_result(self):
		bfd = benfords_frequncy_distribution()
		histogram = calculate_benfords_histogram_from_text(self.text)
		frequencies = calculate_frequncies_from_histogram(histogram)
		marks = expected_distribution_test_for_digits_frequencies(bfd, frequencies,	self.treshold)
		return str(result(histogram, frequencies, marks))

class test_calculate_benfords_histogram_from_text(unittest.TestCase):
	def test_one_number(self):
		text = "12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_negative_number(self):
		text = "-12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_two_numbers_separated_by_space(self):
		text = "12345 234"
		digits_histogram = {1:1, 2:1, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_among_words(self):
		text = "abc 987 zdf"
		digits_histogram = {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:1}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_with_dot_colon(self):
		text = "2,34 5.34"
		digits_histogram = {1:0, 2:1, 3:0, 4:0, 5:1, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_starting_from_zeros(self):
		text = "00345"
		digits_histogram = {1:0, 2:0, 3:1, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

class test_result_str(unittest.TestCase):
	def test_print(self):
		digits_histogram = {x: x for x in range(1, 10)}
		digits_frequencies = {x: float(10) - float(x) for x in range(1, 10)}
		digits_passed = {x: (x % 2) == 0 for x in range(1, 10)}
		r = result(digits_histogram, digits_frequencies, digits_passed)
		bfd = benfords_frequncy_distribution()
		output = r.get_format()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(digits_histogram[digit]) + "\t"
			output += str(digits_frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(digits_passed[digit]) + "\n"
		self.assertEqual(output, str(r))

class test_calculate_frequncies_from_histogram(unittest.TestCase):
	def test_two_positions(self):
		histogram = {1: 2, 2: 18}
		frequencies = {1: 10, 2: 90}
		self.assertEqual(frequencies, calculate_frequncies_from_histogram(histogram))

class test_expected_distribution_test_for_digits_frequencies(unittest.TestCase):
	def test_all_failed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 2 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies))

	def test_all_passed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		marks = {x: True for x in range(1, 10)}
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(frequencies, frequencies))

	def test_two_passed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 2 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		for i in [2, 7]:
			frequencies[i] = expected[i]
			marks[i] = True
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies))

	def test_two_passed_with_treshold(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 3 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		frequencies[2] = expected[2] + 1
		marks[2] = True
		frequencies[7] = expected[7] - 1
		marks[7] = True
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies, 1))

initial_argv = copy.deepcopy(sys.argv)
test_program_option = "--run-tests"
parser = argparse.ArgumentParser(description="Check a text document against Benford's law.")
parser.add_argument(test_program_option, dest='test', action='store_true',
		help='run unit tests')

args = parser.parse_args()

if args.test:
	initial_argv.remove(test_program_option)
	unittest.main(argv = initial_argv)
else:
	bl = benfords_law(sys.stdin.read(), 0)
	print bl.get_result()
<commit_msg>frequency as a decimal not percentage<commit_after>#!/usr/bin/python

import unittest
import math
import argparse
import sys
import copy

class result:
	def __init__(self, histogram, frequencies, passed):
		self.histogram = histogram
		self.frequencies = frequencies
		self.passed = passed

	def __str__(self):
		output = self.get_format()
		bfd = benfords_frequncy_distribution()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(self.histogram[digit]) + "\t"
			output += str(self.frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(self.passed[digit]) + "\n"
		return output

	def get_format(self):
		return "d:\th\tf\tb\tp\n"

def digits():
	return {x: 0 for x in range(1, 10)}

def benfords_frequncy_distribution():
	return {x: math.log(float(1) + (float(1)/float(x)), float(10)) for x in range (1, 10)}

def calculate_frequncies_from_histogram(histogram):
	count = 0
	for c in histogram.itervalues():
		count += c
	frequencies  = dict()
	for k,v in histogram.iteritems():
		frequencies[k] = float(v) / float(count)
	return frequencies

def skip_minus(word):
	if word[0] == '-': return 1
	return 0

def skip_zeros(word):
	i = 0;
	while i < len(word) and word[i].isdigit() and int(word[i]) == 0: i += 1
	return i

def calculate_benfords_histogram_from_text(text):
	digits_histogram = digits()
	for word in text.split():
		i = 0
		i += skip_minus(word)
		i += skip_zeros(word)
		if i < len(word) and word[i].isdigit(): # TODO is not a date
			digits_histogram[int(word[i])] += 1
	return digits_histogram

def expected_distribution_test_for_digits_frequencies(expected, frequencies, treshold = 0):
	return {x: math.fabs(frequencies[x] - expected[x]) <= treshold for x in range(1, 10)}

class benfords_law:

	def __init__(self, text, treshold = 0):
		self.text = text
		self.treshold = treshold

	def get_result(self):
		bfd = benfords_frequncy_distribution()
		histogram = calculate_benfords_histogram_from_text(self.text)
		frequencies = calculate_frequncies_from_histogram(histogram)
		marks = expected_distribution_test_for_digits_frequencies(bfd, frequencies,	self.treshold)
		return str(result(histogram, frequencies, marks))

class test_calculate_benfords_histogram_from_text(unittest.TestCase):
	def test_one_number(self):
		text = "12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_negative_number(self):
		text = "-12345"
		digits_histogram = {1:1, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_two_numbers_separated_by_space(self):
		text = "12345 234"
		digits_histogram = {1:1, 2:1, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_among_words(self):
		text = "abc 987 zdf"
		digits_histogram = {1:0, 2:0, 3:0, 4:0, 5:0, 6:0, 7:0, 8:0, 9:1}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_with_dot_colon(self):
		text = "2,34 5.34"
		digits_histogram = {1:0, 2:1, 3:0, 4:0, 5:1, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

	def test_number_starting_from_zeros(self):
		text = "00345"
		digits_histogram = {1:0, 2:0, 3:1, 4:0, 5:0, 6:0, 7:0, 8:0, 9:0}
		self.assertEqual(calculate_benfords_histogram_from_text(text), digits_histogram)

class test_result_str(unittest.TestCase):
	def test_print(self):
		digits_histogram = {x: x for x in range(1, 10)}
		digits_frequencies = {x: float(10) - float(x) for x in range(1, 10)}
		digits_passed = {x: (x % 2) == 0 for x in range(1, 10)}
		r = result(digits_histogram, digits_frequencies, digits_passed)
		bfd = benfords_frequncy_distribution()
		output = r.get_format()
		for digit in range(1, 10):
			output += str(digit) + ":\t" + str(digits_histogram[digit]) + "\t"
			output += str(digits_frequencies[digit]) + "\t" + str(bfd[digit]) + "\t" + str(digits_passed[digit]) + "\n"
		self.assertEqual(output, str(r))

class test_calculate_frequncies_from_histogram(unittest.TestCase):
	def test_two_positions(self):
		histogram = {1: 2, 2: 18}
		frequencies = {1: 0.1, 2: 0.9}
		self.assertEqual(frequencies, calculate_frequncies_from_histogram(histogram))

class test_expected_distribution_test_for_digits_frequencies(unittest.TestCase):
	def test_all_failed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 2 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies))

	def test_all_passed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		marks = {x: True for x in range(1, 10)}
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(frequencies, frequencies))

	def test_two_passed(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 2 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		for i in [2, 7]:
			frequencies[i] = expected[i]
			marks[i] = True
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies))

	def test_two_passed_with_treshold(self):
		frequencies = {x: 1 for x in range(1, 10)}
		expected = {x: 3 for x in range(1, 10)}
		marks = {x: False for x in range(1, 10)}
		frequencies[2] = expected[2] + 1
		marks[2] = True
		frequencies[7] = expected[7] - 1
		marks[7] = True
		self.assertEqual(marks, expected_distribution_test_for_digits_frequencies(expected, frequencies, 1))

initial_argv = copy.deepcopy(sys.argv)
test_program_option = "--run-tests"
parser = argparse.ArgumentParser(description="Check a text document against Benford's law.")
parser.add_argument(test_program_option, dest='test', action='store_true',
		help='run unit tests')

args = parser.parse_args()

if args.test:
	initial_argv.remove(test_program_option)
	unittest.main(argv = initial_argv)
else:
	bl = benfords_law(sys.stdin.read(), 0)
	print bl.get_result()
<|endoftext|>


<commit_before>import django_filters
from django_filters.fields import Lookup
from django_filters.filterset import STRICTNESS
from django.db.models import Q
from django import forms

from isisdata.models import *
from zotero.models import ImportAccession
from isisdata.helper_methods import strip_punctuation
import six, iso8601
from unidecode import unidecode
from curation.tracking import TrackingWorkflow
from django_filters import filters

filters.LOOKUP_TYPES = [
    ('', '---------'),
    ('exact', 'Is equal to'),
    ('not_exact', 'Is not equal to'),
    ('lt', 'Lesser than'),
    ('gt', 'Greater than'),
    ('gte', 'Greater than or equal to'),
    ('lte', 'Lesser than or equal to'),
    ('startswith', 'Starts with'),
    ('endswith', 'Ends with'),
    ('icontains', 'Contains'),
    ('not_contains', 'Does not contain'),
]

# class ChoiceMethodFilter(django_filters.MethodFilter, django_filters.ChoiceFilter):
#     pass

def filter_in_collections(queryset, value):
    if not value:
        return queryset
    q = Q()
    for collection in value:
        q |= Q(in_collections=collection)

    return queryset.filter(q)

class CitationFilter(django_filters.FilterSet):
    strict = STRICTNESS.RETURN_NO_RESULTS
    # strict = STRICTNESS.RAISE_VALIDATION_ERROR

    # id = django_filters.MethodFilter(name='id', lookup_type='exact')
    id = django_filters.CharFilter(method='filter_id')
    # title = django_filters.MethodFilter(name='title', lookup_type='icontains')
    title = django_filters.CharFilter(method='filter_title')
    type_controlled = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Citation.TYPE_CHOICES))
    # publication_date_from = django_filters.MethodFilter()
    publication_date_from = django_filters.CharFilter(method='filter_publication_date_from')
    # publication_date_to = django_filters.MethodFilter()
    publication_date_to = django_filters.CharFilter(method='filter_publication_date_to')
    # abstract = django_filters.MethodFilter(name='abstract', lookup_type='icontains')
    abstract = django_filters.CharFilter(method='filter_abstract')
    # description = django_filters.MethodFilter(name='description', lookup_type='icontains')
    description = django_filters.CharFilter(method='filter_description')

    # author_or_editor = django_filters.MethodFilter()
    author_or_editor = django_filters.CharFilter(method='filter_author_or_editor')
    # periodical = django_filters.MethodFilter()
    periodical = django_filters.CharFilter(method='filter_periodical')
    # publisher = django_filters.MethodFilter()
    publisher = django_filters.CharFilter(method='filter_publisher')
    # subject = django_filters.MethodFilter()
    subject = django_filters.CharFilter(method='filter_subject')

    record_status = django_filters.ChoiceFilter(name='record_status_value', choices=[('', 'All')] + list(CuratedMixin.STATUS_CHOICES))
    in_collections = django_filters.CharFilter(widget=forms.HiddenInput(), action=filter_in_collections)
    zotero_accession = django_filters.CharFilter(widget=forms.HiddenInput())
    belongs_to = django_filters.CharFilter(widget=forms.HiddenInput())

    tracking_state = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Tracking.TYPE_CHOICES), method='filter_tracking_state')

    # language = django_filters.ModelChoiceFilter(name='language', queryset=Language.objects.all())

    # order = ChoiceMethodFilter(name='order', choices=order_by)

    def __init__(self, *args, **kwargs):
        super(CitationFilter, self).__init__(*args, **kwargs)

        in_coll = self.data.get('in_collections', None)
        if in_coll:
            try:
                collection = CitationCollection.objects.get(pk=in_coll)
                if collection:
                    self.collection_name = collection.name
            except CitationCollection.DoesNotExist:
                self.collection_name = "Collection could not be found."

        zotero_acc = self.data.get('zotero_accession', None)

        if zotero_acc:
            try:
                accession = ImportAccession.objects.get(pk=zotero_acc)
                if accession:
                    self.zotero_accession_name = accession.name
                    self.zotero_accession_date = accession.imported_on
            except ImportAccession.DoesNotExist:
                self.zotero_accession_name = "Zotero accession could not be found."

        dataset = self.data.get('belongs_to', None)
        if dataset:
            try:
                ds = Dataset.objects.get(pk=dataset)
                if ds:
                    self.dataset_name = ds.name
            except Dataset.DoesNotExist:
                self.dataset_name = "Dataset could not be found."

    class Meta:
        model = Citation
        fields = [
            'id', 'title', 'abstract', 'description',
            'publication_date_from', 'publication_date_to',
            'author_or_editor', 'periodical', 'record_status',
            'belongs_to', 'zotero_accession', 'in_collections',
            'tracking_state'
        ]
        # order_by = [
        #     ('publication_date', 'Publication date (ascending)'),
        #     ('-publication_date', 'Publication date (descending)'),
        #     ('title_for_sort', 'Title (ascending)'),
        #     ('-title_for_sort', 'Title (descending)')
        # ]
    o = filters.OrderingFilter(
        # tuple-mapping retains order
        fields=(
            ('publication_date', 'publication_date'),
            ('title', 'title_for_sort'),
        ),

        # labels do not need to retain order
        field_labels={
            'publication_date': 'Publication date',
            'title': 'Title'
        }
    )

    # def get_ordering_field(self):
    #     if self._meta.order_by:
    #         if isinstance(self._meta.order_by, (list, tuple)):
    #             if isinstance(self._meta.order_by[0], (list, tuple)):
    #                 # e.g. (('field', 'Display name'), ...)
    #                 choices = [(f[0], f[1]) for f in self._meta.order_by]
    #             else:
    #                 choices = []
    #                 for f in self._meta.order_by:
    #                     if f[0] == '-':
    #                         label = _('%s (descending)' % capfirst(f[1:]))
    #                     else:
    #                         label = capfirst(f)
    #                     choices.append((f, label))
    #         else:
    #             # add asc and desc field names
    #             # use the filter's label if provided
    #             choices = []
    #             for f, fltr in self.filters.items():
    #                 choices.extend([
    #                     (f, fltr.label or capfirst(f)),
    #                     ("-%s" % (f), _('%s (descending)' % (fltr.label or capfirst(f))))
    #                 ])
    #         return forms.ChoiceField(widget=forms.HiddenInput(attrs={'value':"publication_date"}), choices=choices, initial="publication_date")

    def filter_id(self, queryset, value):
        if not value:
            return queryset

        ids = [i.strip() for i in value.split(',')]
        queryset = queryset.filter(pk__in=ids)
        return queryset

    def filter_title(self, queryset, value):
        value = unidecode(value)
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(title_for_sort__icontains=part)
        return queryset

    def filter_abstract(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(abstract__icontains=part)
        return queryset

    def filter_description(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(description__icontains=part)
        return queryset

    def filter_publication_date_from(self, queryset, value):
        try:
            date = iso8601.parse_date(value)
        except:
            return queryset
        return queryset.filter(publication_date__gte=date)

    def filter_publication_date_to(self, queryset, value):
        try:
            date = iso8601.parse_date(value)
        except:
            return queryset
        return queryset.filter(publication_date__lte=date)

    def filter_author_or_editor(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(
                        acrelation__authority__name__icontains=part,
                        acrelation__type_controlled__in=[
                                    ACRelation.AUTHOR,
                                    ACRelation.EDITOR])
        return queryset

    def filter_periodical(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled__in=[
                                    ACRelation.PERIODICAL,
                                    ACRelation.BOOK_SERIES])

    def filter_publisher(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled=ACRelation.PUBLISHER)

    def filter_subject(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled=ACRelation.SUBJECT)

    def filter_tracking_state(self, queryset, field, value):
        if not value:
            return queryset

        q = Q(tracking_records__type_controlled=value)

        next_state = TrackingWorkflow.stages.get(value)
        # if next_state:
        #     q &= ~Q(tracking_records__type_controlled=next_state)
        return queryset.filter(q)

class AuthorityFilter(django_filters.FilterSet):
    strict = STRICTNESS.RAISE_VALIDATION_ERROR # RETURN_NO_RESULTS

    id = django_filters.CharFilter(name='id', lookup_type='exact')
    # name = django_filters.MethodFilter()
    name = django_filters.CharFilter(method='filter_name')
    type_controlled = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Authority.TYPE_CHOICES))
    description = django_filters.CharFilter(name='description', lookup_type='icontains')
    classification_system = django_filters.ChoiceFilter(name='classification_system', choices=[('', 'All')] + list(Authority.CLASS_SYSTEM_CHOICES))
    classification_code = django_filters.AllValuesFilter(name='classification_code')
    classification_hierarchy = django_filters.AllValuesFilter(name='classification_hierarchy')
    # linked_data = django_filters.MethodFilter()
    linked_data = django_filters.CharFilter(method='filter_linked_data')

    record_status_value = django_filters.ChoiceFilter(name='record_status_value', choices=[('', 'All')] + list(CuratedMixin.STATUS_CHOICES))

    datasets = Dataset.objects.all()
    dataset_list = [(ds.pk, ds.name) for ds in datasets ]
    belongs_to = django_filters.ChoiceFilter(choices=[('', 'All')] + dataset_list)

    class Meta:
        model = Authority
        fields = [
            'id', 'name', 'type_controlled', 'description',
            'classification_system', 'classification_code',
            'classification_hierarchy', 'zotero_accession',
            'belongs_to']

        # order_by = [
        #     ('', 'None'),
        #     ('name_for_sort', 'Name (ascending)'),
        #     ('-name_for_sort', 'Name (descending)')
        # ]

    o = filters.OrderingFilter(
        # tuple-mapping retains order
        fields=(
            ('name', 'name_for_sort'),
        ),

        # labels do not need to retain order
        field_labels={
            'name': 'Name'
        }
    )

    def filter_name(self, queryset, value):
        value = unidecode(value)
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(name_for_sort__icontains=part)

        return queryset

    def filter_linked_data(self, queryset, value):
        if not value:
            return queryset
        authority_ids = LinkedData.objects\
                            .filter(universal_resource_name__contains=value)\
                            .values_list('authorities__id', flat=True)\
                            .distinct()

        if len(authority_ids) == 1 and authority_ids[0] is None:
            return queryset.none()
        return queryset.filter(pk__in=authority_ids)
<commit_msg>hotfix for filters<commit_after>import django_filters
from django_filters.fields import Lookup
from django_filters.filterset import STRICTNESS
from django.db.models import Q
from django import forms

from isisdata.models import *
from zotero.models import ImportAccession
from isisdata.helper_methods import strip_punctuation
import six, iso8601
from unidecode import unidecode
from curation.tracking import TrackingWorkflow
from django_filters import filters

filters.LOOKUP_TYPES = [
    ('', '---------'),
    ('exact', 'Is equal to'),
    ('not_exact', 'Is not equal to'),
    ('lt', 'Lesser than'),
    ('gt', 'Greater than'),
    ('gte', 'Greater than or equal to'),
    ('lte', 'Lesser than or equal to'),
    ('startswith', 'Starts with'),
    ('endswith', 'Ends with'),
    ('icontains', 'Contains'),
    ('not_contains', 'Does not contain'),
]

# class ChoiceMethodFilter(django_filters.MethodFilter, django_filters.ChoiceFilter):
#     pass

def filter_in_collections(queryset, value):
    if not value:
        return queryset
    q = Q()
    for collection in value:
        q |= Q(in_collections=collection)

    return queryset.filter(q)

class CitationFilter(django_filters.FilterSet):
    strict = STRICTNESS.RETURN_NO_RESULTS
    # strict = STRICTNESS.RAISE_VALIDATION_ERROR

    # id = django_filters.MethodFilter(name='id', lookup_type='exact')
    id = django_filters.CharFilter(method='filter_id')
    # title = django_filters.MethodFilter(name='title', lookup_type='icontains')
    title = django_filters.CharFilter(method='filter_title')
    type_controlled = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Citation.TYPE_CHOICES))
    # publication_date_from = django_filters.MethodFilter()
    publication_date_from = django_filters.CharFilter(method='filter_publication_date_from')
    # publication_date_to = django_filters.MethodFilter()
    publication_date_to = django_filters.CharFilter(method='filter_publication_date_to')
    # abstract = django_filters.MethodFilter(name='abstract', lookup_type='icontains')
    abstract = django_filters.CharFilter(method='filter_abstract')
    # description = django_filters.MethodFilter(name='description', lookup_type='icontains')
    description = django_filters.CharFilter(method='filter_description')

    # author_or_editor = django_filters.MethodFilter()
    author_or_editor = django_filters.CharFilter(method='filter_author_or_editor')
    # periodical = django_filters.MethodFilter()
    periodical = django_filters.CharFilter(method='filter_periodical')
    # publisher = django_filters.MethodFilter()
    publisher = django_filters.CharFilter(method='filter_publisher')
    # subject = django_filters.MethodFilter()
    subject = django_filters.CharFilter(method='filter_subject')

    record_status = django_filters.ChoiceFilter(name='record_status_value', choices=[('', 'All')] + list(CuratedMixin.STATUS_CHOICES))
    in_collections = django_filters.CharFilter(method='filter_in_collections', widget=forms.HiddenInput())
    zotero_accession = django_filters.CharFilter(widget=forms.HiddenInput())
    belongs_to = django_filters.CharFilter(widget=forms.HiddenInput())

    tracking_state = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Tracking.TYPE_CHOICES), method='filter_tracking_state')

    # language = django_filters.ModelChoiceFilter(name='language', queryset=Language.objects.all())

    # order = ChoiceMethodFilter(name='order', choices=order_by)

    def __init__(self, *args, **kwargs):
        super(CitationFilter, self).__init__(*args, **kwargs)

        in_coll = self.data.get('in_collections', None)
        if in_coll:
            try:
                collection = CitationCollection.objects.get(pk=in_coll)
                if collection:
                    self.collection_name = collection.name
            except CitationCollection.DoesNotExist:
                self.collection_name = "Collection could not be found."

        zotero_acc = self.data.get('zotero_accession', None)

        if zotero_acc:
            try:
                accession = ImportAccession.objects.get(pk=zotero_acc)
                if accession:
                    self.zotero_accession_name = accession.name
                    self.zotero_accession_date = accession.imported_on
            except ImportAccession.DoesNotExist:
                self.zotero_accession_name = "Zotero accession could not be found."

        dataset = self.data.get('belongs_to', None)
        if dataset:
            try:
                ds = Dataset.objects.get(pk=dataset)
                if ds:
                    self.dataset_name = ds.name
            except Dataset.DoesNotExist:
                self.dataset_name = "Dataset could not be found."

    class Meta:
        model = Citation
        fields = [
            'id', 'title', 'abstract', 'description',
            'publication_date_from', 'publication_date_to',
            'author_or_editor', 'periodical', 'record_status',
            'belongs_to', 'zotero_accession', 'in_collections',
            'tracking_state'
        ]
        # order_by = [
        #     ('publication_date', 'Publication date (ascending)'),
        #     ('-publication_date', 'Publication date (descending)'),
        #     ('title_for_sort', 'Title (ascending)'),
        #     ('-title_for_sort', 'Title (descending)')
        # ]
    o = filters.OrderingFilter(
        # tuple-mapping retains order
        fields=(
            ('publication_date', 'publication_date'),
            ('title', 'title_for_sort'),
        ),

        # labels do not need to retain order
        field_labels={
            'publication_date': 'Publication date',
            'title': 'Title'
        }
    )

    # def get_ordering_field(self):
    #     if self._meta.order_by:
    #         if isinstance(self._meta.order_by, (list, tuple)):
    #             if isinstance(self._meta.order_by[0], (list, tuple)):
    #                 # e.g. (('field', 'Display name'), ...)
    #                 choices = [(f[0], f[1]) for f in self._meta.order_by]
    #             else:
    #                 choices = []
    #                 for f in self._meta.order_by:
    #                     if f[0] == '-':
    #                         label = _('%s (descending)' % capfirst(f[1:]))
    #                     else:
    #                         label = capfirst(f)
    #                     choices.append((f, label))
    #         else:
    #             # add asc and desc field names
    #             # use the filter's label if provided
    #             choices = []
    #             for f, fltr in self.filters.items():
    #                 choices.extend([
    #                     (f, fltr.label or capfirst(f)),
    #                     ("-%s" % (f), _('%s (descending)' % (fltr.label or capfirst(f))))
    #                 ])
    #         return forms.ChoiceField(widget=forms.HiddenInput(attrs={'value':"publication_date"}), choices=choices, initial="publication_date")

    def filter_id(self, queryset, value):
        if not value:
            return queryset

        ids = [i.strip() for i in value.split(',')]
        queryset = queryset.filter(pk__in=ids)
        return queryset

    def filter_title(self, queryset, value):
        value = unidecode(value)
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(title_for_sort__icontains=part)
        return queryset

    def filter_abstract(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(abstract__icontains=part)
        return queryset

    def filter_description(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(description__icontains=part)
        return queryset

    def filter_publication_date_from(self, queryset, value):
        try:
            date = iso8601.parse_date(value)
        except:
            return queryset
        return queryset.filter(publication_date__gte=date)

    def filter_publication_date_to(self, queryset, value):
        try:
            date = iso8601.parse_date(value)
        except:
            return queryset
        return queryset.filter(publication_date__lte=date)

    def filter_author_or_editor(self, queryset, value):
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(
                        acrelation__authority__name__icontains=part,
                        acrelation__type_controlled__in=[
                                    ACRelation.AUTHOR,
                                    ACRelation.EDITOR])
        return queryset

    def filter_periodical(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled__in=[
                                    ACRelation.PERIODICAL,
                                    ACRelation.BOOK_SERIES])

    def filter_publisher(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled=ACRelation.PUBLISHER)

    def filter_subject(self, queryset, value):
        if not value:
            return queryset
        return queryset.filter(acrelation__authority__name__icontains=value,
                               acrelation__type_controlled=ACRelation.SUBJECT)

    def filter_tracking_state(self, queryset, field, value):
        if not value:
            return queryset

        q = Q(tracking_records__type_controlled=value)

        next_state = TrackingWorkflow.stages.get(value)
        # if next_state:
        #     q &= ~Q(tracking_records__type_controlled=next_state)
        return queryset.filter(q)

class AuthorityFilter(django_filters.FilterSet):
    strict = STRICTNESS.RAISE_VALIDATION_ERROR # RETURN_NO_RESULTS

    id = django_filters.CharFilter(name='id', lookup_type='exact')
    # name = django_filters.MethodFilter()
    name = django_filters.CharFilter(method='filter_name')
    type_controlled = django_filters.ChoiceFilter(choices=[('', 'All')] + list(Authority.TYPE_CHOICES))
    description = django_filters.CharFilter(name='description', lookup_type='icontains')
    classification_system = django_filters.ChoiceFilter(name='classification_system', choices=[('', 'All')] + list(Authority.CLASS_SYSTEM_CHOICES))
    classification_code = django_filters.AllValuesFilter(name='classification_code')
    classification_hierarchy = django_filters.AllValuesFilter(name='classification_hierarchy')
    # linked_data = django_filters.MethodFilter()
    linked_data = django_filters.CharFilter(method='filter_linked_data')

    record_status_value = django_filters.ChoiceFilter(name='record_status_value', choices=[('', 'All')] + list(CuratedMixin.STATUS_CHOICES))

    datasets = Dataset.objects.all()
    dataset_list = [(ds.pk, ds.name) for ds in datasets ]
    belongs_to = django_filters.ChoiceFilter(choices=[('', 'All')] + dataset_list)

    class Meta:
        model = Authority
        fields = [
            'id', 'name', 'type_controlled', 'description',
            'classification_system', 'classification_code',
            'classification_hierarchy', 'zotero_accession',
            'belongs_to']

        # order_by = [
        #     ('', 'None'),
        #     ('name_for_sort', 'Name (ascending)'),
        #     ('-name_for_sort', 'Name (descending)')
        # ]

    o = filters.OrderingFilter(
        # tuple-mapping retains order
        fields=(
            ('name', 'name_for_sort'),
        ),

        # labels do not need to retain order
        field_labels={
            'name': 'Name'
        }
    )

    def filter_name(self, queryset, value):
        value = unidecode(value)
        if not value:
            return queryset
        for part in value.split():
            queryset = queryset.filter(name_for_sort__icontains=part)

        return queryset

    def filter_linked_data(self, queryset, value):
        if not value:
            return queryset
        authority_ids = LinkedData.objects\
                            .filter(universal_resource_name__contains=value)\
                            .values_list('authorities__id', flat=True)\
                            .distinct()

        if len(authority_ids) == 1 and authority_ids[0] is None:
            return queryset.none()
        return queryset.filter(pk__in=authority_ids)
<|endoftext|>


<commit_before>r"""Spherical Harmonics as polynomials of x, y, z
"""
import itertools
import math
from functools import partial
from typing import Union

import jax
import jax.numpy as jnp

from e3nn_jax import Irreps, IrrepsData, clebsch_gordan

def spherical_harmonics(
    irreps_out: Union[Irreps, int],
    input: Union[IrrepsData, jnp.ndarray],
    normalize: bool,
    normalization: str = 'integral'
) -> IrrepsData:
    r"""Spherical harmonics

    .. image:: https://user-images.githubusercontent.com/333780/79220728-dbe82c00-7e54-11ea-82c7-b3acbd9b2246.gif

    | Polynomials defined on the 3d space :math:`Y^l: \mathbb{R}^3 \longrightarrow \mathbb{R}^{2l+1}`
    | Usually restricted on the sphere (with ``normalize=True``) :math:`Y^l: S^2 \longrightarrow \mathbb{R}^{2l+1}`
    | who satisfies the following properties:

    * are polynomials of the cartesian coordinates ``x, y, z``
    * is equivariant :math:`Y^l(R x) = D^l(R) Y^l(x)`
    * are orthogonal :math:`\int_{S^2} Y^l_m(x) Y^j_n(x) dx = \text{cste} \; \delta_{lj} \delta_{mn}`

    The value of the constant depends on the choice of normalization.

    It obeys the following property:

    .. math::

        Y^{l+1}_i(x) &= \text{cste}(l) \; & C_{ijk} Y^l_j(x) x_k

        \partial_k Y^{l+1}_i(x) &= \text{cste}(l) \; (l+1) & C_{ijk} Y^l_j(x)

    Where :math:`C` are the `clebsch_gordan`.

    .. note::

        This function match with this table of standard real spherical harmonics from Wikipedia_
        when ``normalize=True``, ``normalization='integral'`` and is called with the argument in the order ``y,z,x`` (instead of ``x,y,z``).

    .. _Wikipedia: https://en.wikipedia.org/wiki/Table_of_spherical_harmonics#Real_spherical_harmonics

    Args:
        irreps_out (`Irreps` or int): output irreps
        input (`IrrepsData` or `jnp.ndarray`): cartesian coordinates
        normalize (bool): if True, the polynomials are restricted to the sphere
        normalization (str): normalization of the constant :math:`\text{cste}`. Default is 'integral'

    Returns:
        `jnp.ndarray`: polynomials of the spherical harmonics
    """
    assert normalization in ['integral', 'component', 'norm']

    if isinstance(irreps_out, int):
        l = irreps_out
        assert isinstance(input, IrrepsData)
        [(mul, ir)] = input.irreps
        irreps_out = Irreps([(1, (l, ir.p**l))])

    irreps_out = Irreps(irreps_out)

    assert all([l % 2 == 1 or p == 1 for _, (l, p) in irreps_out])
    assert len(set([p for _, (l, p) in irreps_out if l % 2 == 1])) <= 1

    if isinstance(input, IrrepsData):
        [(mul, ir)] = input.irreps
        assert mul == 1
        assert ir.l == 1
        assert all([ir.p == p for _, (l, p) in irreps_out if l % 2 == 1])
        x = input.contiguous
    else:
        x = input

    _lmax = 12
    if irreps_out.lmax > _lmax:
        raise NotImplementedError(f'spherical_harmonics maximum l implemented is {_lmax}, send us an email to ask for more')

    return _jited_spherical_harmonics(irreps_out, x, normalize, normalization)

@partial(jax.jit, static_argnums=(0, 2, 3), inline=True)
def _jited_spherical_harmonics(irreps_out, x, normalize, normalization):
    if normalize:
        r = jnp.linalg.norm(x, ord=2, axis=-1, keepdims=True)
        x = x / jnp.where(r == 0.0, 1.0, r)

    sh = _spherical_harmonics(x[..., 0], x[..., 1], x[..., 2])
    sh = [jnp.stack(next(sh), axis=-1) for _ in range(irreps_out.lmax + 1)]
    sh = [jnp.repeat(sh[ir.l][..., None, :], mul, -2) for mul, ir in irreps_out]

    if normalization == 'integral':
        sh = [
            (math.sqrt(ir.dim) / math.sqrt(4 * math.pi)) * y
            for (_, ir), y in zip(irreps_out, sh)
        ]
    elif normalization == 'component':
        sh = [
            math.sqrt(ir.dim) * y
            for (_, ir), y in zip(irreps_out, sh)
        ]

    return IrrepsData.from_list(irreps_out, sh, x.shape[:-1])

def _spherical_harmonics(x, y, z):
    import sympy

    from e3nn_jax.util.sympy import sqrtQarray_to_sympy

    jax_context = dict(x=x, y=y, z=z, sqrt=jnp.sqrt)
    jax_context["sh0_0"] = jnp.ones_like(x)
    yield [jax_context["sh0_0"]]

    xyz = sympy.symbols("x, y, z")

    sph_x = {
        0: sympy.Array([1]),
    }
    sph_1 = {
        0: sympy.Array([1]),
    }

    for l in itertools.count(0):
        d = 2 * l + 1
        sh_var = [sympy.symbols(f"sh{l}_{m}") for m in range(d)]
        w = sqrtQarray_to_sympy(clebsch_gordan(1, l, l + 1))
        yx = sympy.Array([sum(xyz[i] * sh_var[n] * w[i, n, m] for i in range(3) for n in range(d)) for m in range(d + 2)])

        if l <= 1:
            yx = yx.subs(zip(sh_var, sph_x[l]))

        y1 = yx.subs(zip(xyz, (1, 0, 0))).subs(zip(sh_var, sph_1[l]))
        norm = sympy.sqrt(sum(y1.applyfunc(lambda x: x**2)))
        y1 = y1 / norm
        yx = yx / norm
        yx = sympy.simplify(yx)

        l = l + 1
        d = d + 2

        sph_x[l] = yx
        sph_1[l] = y1

        # print code
        values = [eval(f"{p}", jax_context) for m, p in enumerate(yx)]
        yield values

        for m in range(d):
            jax_context[f"sh{l}_{m}"] = values[m]

def print_spherical_harmonics(lmax):  # pragma: no cover
    import sympy

    from e3nn_jax.util.sympy import sqrtQarray_to_sympy

    xyz = sympy.symbols("x, y, z")

    print("sh0_0 = 1")
    print("yield [sh0_0]\n")

    sph_x = {
        0: sympy.Array([1]),
    }
    sph_1 = {
        0: sympy.Array([1]),
    }

    for l in range(lmax):
        d = 2 * l + 1
        sh_var = [sympy.symbols(f"sh{l}_{m}") for m in range(d)]
        w = sqrtQarray_to_sympy(clebsch_gordan(1, l, l + 1))
        yx = sympy.Array([sum(xyz[i] * sh_var[n] * w[i, n, m] for i in range(3) for n in range(d)) for m in range(d + 2)])

        if l <= 1:
            yx = yx.subs(zip(sh_var, sph_x[l]))

        y1 = yx.subs(zip(xyz, (1, 0, 0))).subs(zip(sh_var, sph_1[l]))
        norm = sympy.sqrt(sum(y1.applyfunc(lambda x: x**2)))
        y1 = y1 / norm
        yx = yx / norm
        yx = sympy.simplify(yx)

        sph_x[l + 1] = yx
        sph_1[l + 1] = y1

        # print code
        for m, p in enumerate(yx):
            print(f"sh{l+1}_{m} = {p}")

        print(f"yield [{', '.join([f'sh{l+1}_{m}' for m in range(d + 2)])}]\n")
<commit_msg>remove useless raise<commit_after>r"""Spherical Harmonics as polynomials of x, y, z
"""
import itertools
import math
from functools import partial
from typing import Union

import jax
import jax.numpy as jnp

from e3nn_jax import Irreps, IrrepsData, clebsch_gordan

def spherical_harmonics(
    irreps_out: Union[Irreps, int],
    input: Union[IrrepsData, jnp.ndarray],
    normalize: bool,
    normalization: str = 'integral'
) -> IrrepsData:
    r"""Spherical harmonics

    .. image:: https://user-images.githubusercontent.com/333780/79220728-dbe82c00-7e54-11ea-82c7-b3acbd9b2246.gif

    | Polynomials defined on the 3d space :math:`Y^l: \mathbb{R}^3 \longrightarrow \mathbb{R}^{2l+1}`
    | Usually restricted on the sphere (with ``normalize=True``) :math:`Y^l: S^2 \longrightarrow \mathbb{R}^{2l+1}`
    | who satisfies the following properties:

    * are polynomials of the cartesian coordinates ``x, y, z``
    * is equivariant :math:`Y^l(R x) = D^l(R) Y^l(x)`
    * are orthogonal :math:`\int_{S^2} Y^l_m(x) Y^j_n(x) dx = \text{cste} \; \delta_{lj} \delta_{mn}`

    The value of the constant depends on the choice of normalization.

    It obeys the following property:

    .. math::

        Y^{l+1}_i(x) &= \text{cste}(l) \; & C_{ijk} Y^l_j(x) x_k

        \partial_k Y^{l+1}_i(x) &= \text{cste}(l) \; (l+1) & C_{ijk} Y^l_j(x)

    Where :math:`C` are the `clebsch_gordan`.

    .. note::

        This function match with this table of standard real spherical harmonics from Wikipedia_
        when ``normalize=True``, ``normalization='integral'`` and is called with the argument in the order ``y,z,x`` (instead of ``x,y,z``).

    .. _Wikipedia: https://en.wikipedia.org/wiki/Table_of_spherical_harmonics#Real_spherical_harmonics

    Args:
        irreps_out (`Irreps` or int): output irreps
        input (`IrrepsData` or `jnp.ndarray`): cartesian coordinates
        normalize (bool): if True, the polynomials are restricted to the sphere
        normalization (str): normalization of the constant :math:`\text{cste}`. Default is 'integral'

    Returns:
        `jnp.ndarray`: polynomials of the spherical harmonics
    """
    assert normalization in ['integral', 'component', 'norm']

    if isinstance(irreps_out, int):
        l = irreps_out
        assert isinstance(input, IrrepsData)
        [(mul, ir)] = input.irreps
        irreps_out = Irreps([(1, (l, ir.p**l))])

    irreps_out = Irreps(irreps_out)

    assert all([l % 2 == 1 or p == 1 for _, (l, p) in irreps_out])
    assert len(set([p for _, (l, p) in irreps_out if l % 2 == 1])) <= 1

    if isinstance(input, IrrepsData):
        [(mul, ir)] = input.irreps
        assert mul == 1
        assert ir.l == 1
        assert all([ir.p == p for _, (l, p) in irreps_out if l % 2 == 1])
        x = input.contiguous
    else:
        x = input

    return _jited_spherical_harmonics(irreps_out, x, normalize, normalization)

@partial(jax.jit, static_argnums=(0, 2, 3), inline=True)
def _jited_spherical_harmonics(irreps_out, x, normalize, normalization):
    if normalize:
        r = jnp.linalg.norm(x, ord=2, axis=-1, keepdims=True)
        x = x / jnp.where(r == 0.0, 1.0, r)

    sh = _spherical_harmonics(x[..., 0], x[..., 1], x[..., 2])
    sh = [jnp.stack(next(sh), axis=-1) for _ in range(irreps_out.lmax + 1)]
    sh = [jnp.repeat(sh[ir.l][..., None, :], mul, -2) for mul, ir in irreps_out]

    if normalization == 'integral':
        sh = [
            (math.sqrt(ir.dim) / math.sqrt(4 * math.pi)) * y
            for (_, ir), y in zip(irreps_out, sh)
        ]
    elif normalization == 'component':
        sh = [
            math.sqrt(ir.dim) * y
            for (_, ir), y in zip(irreps_out, sh)
        ]

    return IrrepsData.from_list(irreps_out, sh, x.shape[:-1])

def _spherical_harmonics(x, y, z):
    import sympy

    from e3nn_jax.util.sympy import sqrtQarray_to_sympy

    jax_context = dict(x=x, y=y, z=z, sqrt=jnp.sqrt)
    jax_context["sh0_0"] = jnp.ones_like(x)
    yield [jax_context["sh0_0"]]

    xyz = sympy.symbols("x, y, z")

    sph_x = {
        0: sympy.Array([1]),
    }
    sph_1 = {
        0: sympy.Array([1]),
    }

    for l in itertools.count(0):
        d = 2 * l + 1
        sh_var = [sympy.symbols(f"sh{l}_{m}") for m in range(d)]
        w = sqrtQarray_to_sympy(clebsch_gordan(1, l, l + 1))
        yx = sympy.Array([sum(xyz[i] * sh_var[n] * w[i, n, m] for i in range(3) for n in range(d)) for m in range(d + 2)])

        if l <= 1:
            yx = yx.subs(zip(sh_var, sph_x[l]))

        y1 = yx.subs(zip(xyz, (1, 0, 0))).subs(zip(sh_var, sph_1[l]))
        norm = sympy.sqrt(sum(y1.applyfunc(lambda x: x**2)))
        y1 = y1 / norm
        yx = yx / norm
        yx = sympy.simplify(yx)

        l = l + 1
        d = d + 2

        sph_x[l] = yx
        sph_1[l] = y1

        # print code
        values = [eval(f"{p}", jax_context) for m, p in enumerate(yx)]
        yield values

        for m in range(d):
            jax_context[f"sh{l}_{m}"] = values[m]

def print_spherical_harmonics(lmax):  # pragma: no cover
    import sympy

    from e3nn_jax.util.sympy import sqrtQarray_to_sympy

    xyz = sympy.symbols("x, y, z")

    print("sh0_0 = 1")
    print("yield [sh0_0]\n")

    sph_x = {
        0: sympy.Array([1]),
    }
    sph_1 = {
        0: sympy.Array([1]),
    }

    for l in range(lmax):
        d = 2 * l + 1
        sh_var = [sympy.symbols(f"sh{l}_{m}") for m in range(d)]
        w = sqrtQarray_to_sympy(clebsch_gordan(1, l, l + 1))
        yx = sympy.Array([sum(xyz[i] * sh_var[n] * w[i, n, m] for i in range(3) for n in range(d)) for m in range(d + 2)])

        if l <= 1:
            yx = yx.subs(zip(sh_var, sph_x[l]))

        y1 = yx.subs(zip(xyz, (1, 0, 0))).subs(zip(sh_var, sph_1[l]))
        norm = sympy.sqrt(sum(y1.applyfunc(lambda x: x**2)))
        y1 = y1 / norm
        yx = yx / norm
        yx = sympy.simplify(yx)

        sph_x[l + 1] = yx
        sph_1[l + 1] = y1

        # print code
        for m, p in enumerate(yx):
            print(f"sh{l+1}_{m} = {p}")

        print(f"yield [{', '.join([f'sh{l+1}_{m}' for m in range(d + 2)])}]\n")
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
__actions = ("acf", "slk", "fdr", "peaks", "rpsim", "hist")

def main():
    if len(sys.argv) == 1 or sys.argv[1] in ("-h", "--help"):
        print >>sys.stderr,"""
Tools for viewing and adjusting p-values in BED files.

   Contact: Brent Pedersen - bpederse@gmail.com
   License: BSD

To run, indicate one of:

   acf   - calculate autocorrelation within BED file
   slk   - Stouffer-Liptak-Kechris correction of spatially correlated p-values
   fdr   - Benjamini-Hochberg correction of p-values
   peaks - find peaks in a BED file.
   rpsim - generate p-values for a region (of p-values) by simulation.**
   hist  - plot a histogram of a column and check for uniformity.

NOTE: all of these assume *sorted* BED files.
    """
        sys.exit()
    if not sys.argv[1] in __actions:
        _pipeline()
    else:
        module = __import__(sys.argv[1])
        # remove the action
        sys.argv.pop(1)
        module.main()

def _pipeline():
    # TODO
    pass

if __name__ == "__main__":
    main()
<commit_msg>hist doesnt requires sorted<commit_after>#!/usr/bin/env python
import sys
__actions = ("acf", "slk", "fdr", "peaks", "rpsim", "hist")

def main():
    if len(sys.argv) == 1 or sys.argv[1] in ("-h", "--help"):
        print >>sys.stderr,"""
Tools for viewing and adjusting p-values in BED files.

   Contact: Brent Pedersen - bpederse@gmail.com
   License: BSD

To run, indicate one of:

   acf   - calculate autocorrelation within BED file
   slk   - Stouffer-Liptak-Kechris correction of spatially correlated p-values
   fdr   - Benjamini-Hochberg correction of p-values
   peaks - find peaks in a BED file.
   rpsim - generate p-values for a region (of p-values) by simulation.**
   hist  - plot a histogram of a column and check for uniformity.

NOTE: most of these assume *sorted* BED files.
    """
        sys.exit()
    if not sys.argv[1] in __actions:
        _pipeline()
    else:
        module = __import__(sys.argv[1])
        # remove the action
        sys.argv.pop(1)
        module.main()

def _pipeline():
    # TODO
    pass

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#pangramGenerator
import pickle,random

#prompt for a lexicon first
corpus=raw_input('enter a corpus: ')
corpus_name = 'lexicons/lexicon-' + corpus

#load our trained lexicon
a=open(corpus_name,'rb')
successorlist=pickle.load(a)
a.close()

#get the next word in the sentence
def nextword(a,r):
    if a in successorlist:
        #look through list and update probability based on ratio of new letters
        # for i in successorlist:
        #     print i
        #count number of new letters in word
        #count number of total letters in word
        #place this ratio in an array assigned with this index
        #normalize the array values

        #alternate: choose randomly with the weightings assigned by new letter ratio
        #to do this, the easiest way was to create a new list where a word
        #appears in the list as many times as it has new and useful letters
        weightedList = []
        for word in successorlist[a]:
            # count = 0
            for i in range(0,len(r)):
                if r[i] in word:
                    # count += 1
                    weightedList.append(word)
                    # print str(count) + word
        if len(weightedList) > 0:
            return random.choice(weightedList)
        else:
            return random.choice(successorlist[a])
    else:
        return 'the'

speech=''

#main loop (only quit when user types quit)
while speech!='quit':
    #start us off with a word, if we don't have it, we'll start with 'the'
    speech=raw_input('>')
    s=random.choice(speech.split())
    response=''

    #keep track of letters used in the pangram so far
    remaining = 'abcdefghijklmnopqrstuvwxyz'

    while True:
        if s[-1] in ',?!.':
            # print 'yup there is punctuation in ' + s + ' and it is ' + s[-1]
            s = s[:-1]
        neword = nextword(s, remaining)
        response += ' ' + neword
        # print response
        s = neword

        #remove letters used in the latest word from the remaining string
        count = 0
        for i in range(0,len(remaining)):
            if remaining[count] in neword:
                # print remaining[count] + ' is in ' + neword
                remaining = remaining[:count] + remaining[count+1:]
            else:
                count -= 1

        # only finish when we have a sentence with all letters
        if len(remaining) < 1:
            print str(len(remaining)) + ' letters remaining: ' + remaining
            break

    # show the magic
    print response<commit_msg>load corpus for pangram<commit_after>#pangramGenerator
import pickle,random

#prompt for a lexicon first
corpus = raw_input('enter a corpus: ')
corpus_name = 'lexicons/lexicon-' + corpus

#load our trained lexicon
a = open(corpus_name,'rb')
successorlist = pickle.load(a)
a.close()

#get the next word in the sentence
def nextword(a,r):
    if a in successorlist:
        #look through list and update probability based on ratio of new letters
        # for i in successorlist:
        #     print i
        #count number of new letters in word
        #count number of total letters in word
        #place this ratio in an array assigned with this index
        #normalize the array values

        #alternate: choose randomly with the weightings assigned by new letter ratio
        #to do this, the easiest way was to create a new list where a word
        #appears in the list as many times as it has new and useful letters
        weightedList = []
        for word in successorlist[a]:
            # count = 0
            for i in range(0,len(r)):
                if r[i] in word:
                    # count += 1
                    weightedList.append(word)
                    # print str(count) + word
        if len(weightedList) > 0:
            return random.choice(weightedList)
        else:
            return random.choice(successorlist[a])
    else:
        return 'the'

speech = ''

#main loop (only quit when user types quit)
while speech != 'quit':
    #start us off with a word, if we don't have it, we'll start with 'the'
    speech = raw_input('>')
    s = random.choice(speech.split())
    response = ''

    #keep track of letters used in the pangram so far
    remaining = 'abcdefghijklmnopqrstuvwxyz'

    while True:
        if s[-1] in ',?!.':
            # print 'yup there is punctuation in ' + s + ' and it is ' + s[-1]
            s = s[:-1]
        neword = nextword(s, remaining)
        response += ' ' + neword
        # print response
        s = neword

        #remove letters used in the latest word from the remaining string
        count = 0
        for i in range(0,len(remaining)):
            if remaining[count] in neword:
                # print remaining[count] + ' is in ' + neword
                remaining = remaining[:count] + remaining[count+1:]
            else:
                count -= 1

        # only finish when we have a sentence with all letters
        if len(remaining) < 1:
            print str(len(remaining)) + ' letters remaining: ' + remaining
            break

    # show the magic
    print response<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function, division
import numpy as np
import argparse
from astropy import log
from os import path, environ
from glob import glob
import astropy.io.fits as pyfits
from astropy.time import Time
import astropy.units as u

from nicer.values import *

def read_kpfiles_potsdam():
    kpfiles = glob(path.join(datadir,'KP-potsdam','kp*.tab'))
    kpfiles.sort()
    hours = np.array([0.0, 3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0])*u.hour

    kpdict = {            '0o': 0.0, '0+': 0.33,
              '1-': 0.66, '1o': 1.0, '1+': 1.33,
              '2-': 1.66, '2o': 2.0, '2+': 2.33,
              '3-': 2.66, '3o': 3.0, '3+': 3.33,
              '4-': 3.66, '4o': 4.0, '4+': 4.33,
              '5-': 4.66, '5o': 5.0, '5+': 5.33,
              '6-': 5.66, '6o': 6.0, '6+': 6.33,
              '7-': 6.66, '7o': 7.0, '7+': 7.33,
              '8-': 7.66, '8o': 8.0, '8+': 8.33,
              '9-': 8.66, '9o': 9.0, '9+': 9.33
    }

    kpmets = []
    kpvals = []
    for kpname in kpfiles:
        with open(kpname,'r') as file:
            for line in file:
                if len(line.strip()) <= 0:
                    continue
                cols = line.split()
                if len(cols[0]) == 6:
                    t0 = Time("20{0}-{1}-{2}T00:00:00".format(cols[0][0:2],cols[0][2:4],cols[0][4:6]),format='isot',scale='utc')
                    for kpstring,h in zip(cols[1:9],hours):
                        met = ((t0+h)-MET0).to(u.s).value
                        kpval = kpdict[kpstring]
                        kpmets.append(met)
                        kpvals.append(kpval)
    kpmets = np.array(kpmets)
    kpvals = np.array(kpvals)
    log.info('KP table goes from {0} to {1}'.format(kpmets[0],kpmets[-1]))

    return (kpmets,kpvals)

def read_kp_GOF(fname):
    dat = pyfits.getdata(fname,extname='KP')
    return (dat['TIME'], dat['KP'])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Add Kp values as a column to an existing MKF file')
    parser.add_argument("mkf", help="Input MKF file process (will be modified)", nargs='+')
    parser.add_argument("--potsdam",help="Use Potsdam Kp values from NICERsoft data (default is NOA data from NICER GOF)",action="store_true",default=False)
    parser.add_argument("--nearest",help="Use nearest Kp value, instead of the default linear interpolation",action="store_true",default=False)
    parser.add_argument("--kp", help="Path to NICER GOF kp.fits file (default $CALDB/data/gen/pcf/kp.fits). Only used if not --potsdam.", default=None)
    args = parser.parse_args()

    # Read the Pottsdam Kp index data files
    if args.potsdam:
        log.info('Reading Potsdam Kp files...')
        kpmets, kpvals = read_kpfiles_potsdam()
    else:
        log.info('Reading NICER GOF KP files')
        if args.kp is None:
            args.kp = path.join(environ['CALDB'],"data/gen/pcf/kp.fits")
            if not path.isfile(args.kp):
                log.error("KP file {0} does not exist.  Please download https://heasarc.gsfc.nasa.gov/FTP/caldb/data/gen/pcf/kp.fits",args.kp)
                raise RuntimeError
        kpmets, kpvals = read_kp_GOF(args.kp)

    for mkf in args.mkf:
        # Read the METs from the MKF file
        with pyfits.open(mkf) as hdulist:
            log.info('Processing {0}'.format(mkf))
            #mkfdat = pyfits.getdata(mkf,extname='PREFILTER')
            mkfmets = hdulist[1].data['TIME']

            # Look up, or interpolate, all the Kp values
            if args.nearest:
                kp = np.array([kpvals[kpmets.searchsorted(m)] for m in mkfmets])
            else:
                kp = np.interp(mkfmets, kpmets, kpvals)

            names     = hdulist[1].data.names

            if 'KP' in names:
                log.info('Found KP column, overwriting...')
                hdulist[1].data.field('KP')[:] = kp
            else:
                log.info('Adding KP column to MKF file...')
                col   = pyfits.Column(name='KP',format='E',array=kp)
                tbhdu = pyfits.BinTableHDU.from_columns(hdulist[1].data.columns+col,
                        header=hdulist[1].header)
                hdulist[1] = tbhdu

            hdulist.writeto(mkf,overwrite=True)
            hdulist.close()
<commit_msg>Fix name of KP extension in GOF file since they changed it<commit_after>#!/usr/bin/env python
from __future__ import print_function, division
import numpy as np
import argparse
from astropy import log
from os import path, environ
from glob import glob
import astropy.io.fits as pyfits
from astropy.time import Time
import astropy.units as u

from nicer.values import *

def read_kpfiles_potsdam():
    kpfiles = glob(path.join(datadir,'KP-potsdam','kp*.tab'))
    kpfiles.sort()
    hours = np.array([0.0, 3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0])*u.hour

    kpdict = {            '0o': 0.0, '0+': 0.33,
              '1-': 0.66, '1o': 1.0, '1+': 1.33,
              '2-': 1.66, '2o': 2.0, '2+': 2.33,
              '3-': 2.66, '3o': 3.0, '3+': 3.33,
              '4-': 3.66, '4o': 4.0, '4+': 4.33,
              '5-': 4.66, '5o': 5.0, '5+': 5.33,
              '6-': 5.66, '6o': 6.0, '6+': 6.33,
              '7-': 6.66, '7o': 7.0, '7+': 7.33,
              '8-': 7.66, '8o': 8.0, '8+': 8.33,
              '9-': 8.66, '9o': 9.0, '9+': 9.33
    }

    kpmets = []
    kpvals = []
    for kpname in kpfiles:
        with open(kpname,'r') as file:
            for line in file:
                if len(line.strip()) <= 0:
                    continue
                cols = line.split()
                if len(cols[0]) == 6:
                    t0 = Time("20{0}-{1}-{2}T00:00:00".format(cols[0][0:2],cols[0][2:4],cols[0][4:6]),format='isot',scale='utc')
                    for kpstring,h in zip(cols[1:9],hours):
                        met = ((t0+h)-MET0).to(u.s).value
                        kpval = kpdict[kpstring]
                        kpmets.append(met)
                        kpvals.append(kpval)
    kpmets = np.array(kpmets)
    kpvals = np.array(kpvals)
    log.info('KP table goes from {0} to {1}'.format(kpmets[0],kpmets[-1]))

    return (kpmets,kpvals)

def read_kp_GOF(fname):
    dat = pyfits.getdata(fname,extname='NOAO_KP')
    return (dat['TIME'], dat['KP'])

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description = 'Add Kp values as a column to an existing MKF file')
    parser.add_argument("mkf", help="Input MKF file process (will be modified)", nargs='+')
    parser.add_argument("--potsdam",help="Use Potsdam Kp values from NICERsoft data (default is NOA data from NICER GOF)",action="store_true",default=False)
    parser.add_argument("--nearest",help="Use nearest Kp value, instead of the default linear interpolation",action="store_true",default=False)
    parser.add_argument("--kp", help="Path to NICER GOF kp.fits file (default $CALDB/data/gen/pcf/kp.fits). Only used if not --potsdam.", default=None)
    args = parser.parse_args()

    # Read the Pottsdam Kp index data files
    if args.potsdam:
        log.info('Reading Potsdam Kp files...')
        kpmets, kpvals = read_kpfiles_potsdam()
    else:
        log.info('Reading NICER GOF KP files')
        if args.kp is None:
            args.kp = path.join(environ['CALDB'],"data/gen/pcf/kp.fits")
            if not path.isfile(args.kp):
                log.error("KP file {0} does not exist.  Please download https://heasarc.gsfc.nasa.gov/FTP/caldb/data/gen/pcf/kp.fits",args.kp)
                raise RuntimeError
        kpmets, kpvals = read_kp_GOF(args.kp)

    for mkf in args.mkf:
        # Read the METs from the MKF file
        with pyfits.open(mkf) as hdulist:
            log.info('Processing {0}'.format(mkf))
            #mkfdat = pyfits.getdata(mkf,extname='PREFILTER')
            mkfmets = hdulist[1].data['TIME']

            # Look up, or interpolate, all the Kp values
            if args.nearest:
                kp = np.array([kpvals[kpmets.searchsorted(m)] for m in mkfmets])
            else:
                kp = np.interp(mkfmets, kpmets, kpvals)

            names     = hdulist[1].data.names

            if 'KP' in names:
                log.info('Found KP column, overwriting...')
                hdulist[1].data.field('KP')[:] = kp
            else:
                log.info('Adding KP column to MKF file...')
                col   = pyfits.Column(name='KP',format='E',array=kp)
                tbhdu = pyfits.BinTableHDU.from_columns(hdulist[1].data.columns+col,
                        header=hdulist[1].header)
                hdulist[1] = tbhdu

            hdulist.writeto(mkf,overwrite=True)
            hdulist.close()
<|endoftext|>


<commit_before>from django import http
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from django.views.generic import TemplateView
from django.views.generic import FormView
from django.views.generic import View
from history import forms

def get_form_data_from_session(session):
    form_data = session['forms']
    for form in form_data:
        form.pop("csrfmiddlewaretoken", None)
    return form_data

def format_history_entry(entry):
    description = ""
    if entry["description"][0]:
        description = "({0})".format(entry["description"][0])
    circumstance_data = entry["circumstances"]
    if 'other' in circumstance_data:
        circumstance_data.remove('other')
    if entry["other_more"][0]:
        circumstance_data += entry["other_more"]
    formatted_circumstances = list(map(format_circumstance, circumstance_data))
    circumstances = ", ".join(formatted_circumstances)
    duration_dict = dict(forms.HistoryDetailsForm.DURATION_CHOICES)
    duration = duration_dict[entry["duration"][0]]
    return "For {0}: {1} {2}".format(duration, circumstances, description)

def format_circumstance(circumstance):
    circumstance_dict = dict(forms.HistoryDetailsForm.CIRCUMSTANCE_CHOICES)
    return circumstance_dict.get(circumstance, circumstance)

class HistoryDetailsView(FormView):
    """
    Render HistoryDetailsView and redirect to HistoryReportView when
    the form has been completed 3 times
    """

    template_name = "history/details.html"
    form_class = forms.HistoryDetailsForm

    def get(self, request, *args, **kwargs):
        if len(self.request.session.get('forms', [])) >= 3:
            url = reverse('history:report')
            return http.HttpResponseRedirect(url)
        else:
            response = super().get(request, *args, **kwargs)
            return response

    def form_valid(self, form):
        if 'forms' not in self.request.session:
            self.request.session['forms'] = []
        self.request.session['forms'] += [dict(form.data.lists())]
        if len(self.request.session['forms']) < 3:
            url = reverse('history:details')
        else:
            url = reverse('history:report')
        return http.HttpResponseRedirect(url)

    def get_context_data(self, **kwargs):
        context = kwargs
        history_data = get_form_data_from_session(self.request.session)
        if history_data:
            context['history'] = map(format_history_entry, history_data)
            context['circumstance_title'] = "Your circumstances previously"
        else:
            context['circumstance_title'] = "Your current circumstances"
        return context

class HistoryReportView(TemplateView):
    def get(self, request, *args, **kwargs):
        session = self.request.session
        if 'forms' not in session or len(session['forms']) < 3:
            url = reverse('history:details')
            return http.HttpResponseRedirect(url)
        else:
            self.template_name = "history/report.html"
            response = super().get(request, *args, **kwargs)
            return response

    def get_context_data(self, **kwargs):
        context = kwargs
        history_data = get_form_data_from_session(self.request.session)
        context['report'] = map(format_history_entry, history_data)
        return context
<commit_msg>Don't crash in absence of forms data in session<commit_after>from django import http
from django.core.urlresolvers import reverse
from django.shortcuts import get_object_or_404
from django.views.generic import TemplateView
from django.views.generic import FormView
from django.views.generic import View
from history import forms

def get_form_data_from_session(session):
    form_data = session.get('forms', [])
    for form in form_data:
        form.pop("csrfmiddlewaretoken", None)
    return form_data

def format_history_entry(entry):
    description = ""
    if entry["description"][0]:
        description = "({0})".format(entry["description"][0])
    circumstance_data = entry["circumstances"]
    if 'other' in circumstance_data:
        circumstance_data.remove('other')
    if entry["other_more"][0]:
        circumstance_data += entry["other_more"]
    formatted_circumstances = list(map(format_circumstance, circumstance_data))
    circumstances = ", ".join(formatted_circumstances)
    duration_dict = dict(forms.HistoryDetailsForm.DURATION_CHOICES)
    duration = duration_dict[entry["duration"][0]]
    return "For {0}: {1} {2}".format(duration, circumstances, description)

def format_circumstance(circumstance):
    circumstance_dict = dict(forms.HistoryDetailsForm.CIRCUMSTANCE_CHOICES)
    return circumstance_dict.get(circumstance, circumstance)

class HistoryDetailsView(FormView):
    """
    Render HistoryDetailsView and redirect to HistoryReportView when
    the form has been completed 3 times
    """

    template_name = "history/details.html"
    form_class = forms.HistoryDetailsForm

    def get(self, request, *args, **kwargs):
        if len(self.request.session.get('forms', [])) >= 3:
            url = reverse('history:report')
            return http.HttpResponseRedirect(url)
        else:
            response = super().get(request, *args, **kwargs)
            return response

    def form_valid(self, form):
        if 'forms' not in self.request.session:
            self.request.session['forms'] = []
        self.request.session['forms'] += [dict(form.data.lists())]
        if len(self.request.session['forms']) < 3:
            url = reverse('history:details')
        else:
            url = reverse('history:report')
        return http.HttpResponseRedirect(url)

    def get_context_data(self, **kwargs):
        context = kwargs
        history_data = get_form_data_from_session(self.request.session)
        if history_data:
            context['history'] = map(format_history_entry, history_data)
            context['circumstance_title'] = "Your circumstances previously"
        else:
            context['circumstance_title'] = "Your current circumstances"
        return context

class HistoryReportView(TemplateView):
    def get(self, request, *args, **kwargs):
        session = self.request.session
        if 'forms' not in session or len(session['forms']) < 3:
            url = reverse('history:details')
            return http.HttpResponseRedirect(url)
        else:
            self.template_name = "history/report.html"
            response = super().get(request, *args, **kwargs)
            return response

    def get_context_data(self, **kwargs):
        context = kwargs
        history_data = get_form_data_from_session(self.request.session)
        context['report'] = map(format_history_entry, history_data)
        return context
<|endoftext|>


<commit_before>from threading import *
import time
import random
class ImageManager:
    def __init__(self):
        self.gamma = bytearray(256)
        for i in range(256):
            self.gamma[i] = 0x80 | int(pow(float(i)/255.0, 2.5) * 127.0 + 0.5)

        self.STRIPLEN = 32 # configurable to however many pixels you have on your led strip
        self.COLORLEN = self.STRIPLEN * 3 #Colorlen is the size of the part of the array containing actual
                                          #color information.
        self.ARRAYLEN = self.COLORLEN + 1 #Arraylen is the size of the whole array.
        self.lights = bytearray([0x80 for x in range(self.ARRAYLEN)])
        self.lights[self.ARRAYLEN - 1] = 0

        self.DEV = '/dev/spidev0.1'
        self.spidev = open(self.DEV, 'wb')

    def write(self, red,green,blue):
        for x in range(self.STRIPLEN):
            if x < 3:
                self.setpixel((0,0,0),x)
            else:
                self.setpixel((red,green,blue), x)
        self.output()

    def setpixel(self, pixel, position):
        """
        Sets a certain pixel of the strip (at 'position') to the color tuple 'pixel'
        """
        index = position * 3
        red, green, blue = pixel
        self.lights[index] = self.gamma[green]
        self.lights[index + 1] = self.gamma[red]
        self.lights[index + 2] = self.gamma[blue]

    def blackout(self):
        for x in range(self.STRIPLEN):
            self.setpixel((0,0,0), x)
        self.output()

    def output(self):
        """
        Actually writes the array to the physical strip
        """
        self.spidev.write(self.lights)
        self.spidev.flush()

class WorkerThread(Thread):
    """Aims to make the network server very responsive"""
    def __init__(self):
        """Initialize thread"""
        Thread.__init__(self)
        self.message = ""
        self.cond = Condition()
        self.keepGoing = True
        self.lights = ImageManager()
        self.start()

    def run(self):
        while self.keepGoing:
            if self.message != "":
                self.processMessage()
            #print("waiting...")

            with(self.cond):
                self.cond.wait()

    """
    Posts a message and tells the run method to process it.

    The message will be in the format 'qualifier : data'
    For now, the only qualifier is ALT, meaning altitude. (above body's surface or MSL)
    """
    def sendMessage(self, data):
        with(self.cond):
            self.message = data.decode('utf-8').strip()
            self.cond.notifyAll()

    def stop(self):
        with(self.cond):
            self.keepGoing = False
            self.cond.notifyAll()
            self.lights.blackout()

    def processMessage(self):
        try:
            info = self.message.split(':')
            red=int(info[0])
            green=int(info[1])
            blue=int(info[2])
            if red>=0 and red<=255 and blue>=0 and blue<=255 and green>=0 and green<=255:
                self.lights.write(red,green,blue)
        except:
            pass

<commit_msg>Stabilized strip in between writes, blacked out first two LEDs for ease on eyes.<commit_after>from threading import *
import time
import random
class ImageManager:
    def __init__(self):
        self.gamma = bytearray(256)
        for i in range(256):
            self.gamma[i] = 0x80 | int(pow(float(i)/255.0, 2.5) * 127.0 + 0.5)

        self.STRIPLEN = 32 # configurable to however many pixels you have on your led strip
        self.COLORLEN = self.STRIPLEN * 3 #Colorlen is the size of the part of the array containing actual
                                          #color information.
        self.ARRAYLEN = self.COLORLEN + 2 #Arraylen is the size of the whole array.
        self.lights = bytearray([0x80 for x in range(self.ARRAYLEN)])
        self.lights[0] = 0
        self.lights[self.ARRAYLEN-1] = 0x80
        self.DEV = '/dev/spidev0.1'
        self.spidev = open(self.DEV, 'wb')

    def write(self, red,green,blue):
        for x in range(self.STRIPLEN):
            if x < 2:
                self.setpixel((0,0,0),x)
            else:
                self.setpixel((red,green,blue), x)
        self.output()

    def setpixel(self, pixel, position):
        """
        Sets a certain pixel of the strip (at 'position') to the color tuple 'pixel'
        """
        index = position * 3
        red, green, blue = pixel
        self.lights[index + 1] = self.gamma[green]
        self.lights[index + 2] = self.gamma[red]
        self.lights[index + 3] = self.gamma[blue]

    def blackout(self):
        for x in range(self.STRIPLEN):
            self.setpixel((0,0,0), x)
        self.output()

    def output(self):
        """
        Actually writes the array to the physical strip
        """
        self.spidev.write(self.lights)
        self.spidev.flush()

class WorkerThread(Thread):
    """Aims to make the network server very responsive"""
    def __init__(self):
        """Initialize thread"""
        Thread.__init__(self)
        self.message = ""
        self.cond = Condition()
        self.keepGoing = True
        self.lights = ImageManager()
        self.start()

    def run(self):
        while self.keepGoing:
            if self.message != "":
                self.processMessage()
            #print("waiting...")

            with(self.cond):
                self.cond.wait()

    """
    Posts a message and tells the run method to process it.

    The message will be in the format 'qualifier : data'
    For now, the only qualifier is ALT, meaning altitude. (above body's surface or MSL)
    """
    def sendMessage(self, data):
        with(self.cond):
            self.message = data.decode('utf-8').strip()
            self.cond.notifyAll()

    def stop(self):
        with(self.cond):
            self.keepGoing = False
            self.cond.notifyAll()
            self.lights.blackout()

    def processMessage(self):
        try:
            info = self.message.split(':')
            red=int(info[0])
            green=int(info[1])
            blue=int(info[2])
            if red>=0 and red<=255 and blue>=0 and blue<=255 and green>=0 and green<=255:
                self.lights.write(red,green,blue)
        except:
            pass

<|endoftext|>


<commit_before>from settings import *
import sys
from binascii import hexlify
import time
from client import Client
from threading import Thread

def access(host, port):
    con = Client(host, port, False, True)
    con.notifyHTTP2()
    time.sleep(0.2)
    con.send(TYPE.SETTINGS, ident=SETTINGS.NO, value = "")
    time.sleep(0.2)
    con.send(TYPE.HEADERS, FLAG.NO, 1, headers = [[":method", "GET"],
                                                  [":scheme", "http"],
                                                  [":authority", "127.0.0.1"],
                                                  [":path", "/"]])

    time.sleep(0.2)
    con.send(TYPE.PUSH_PROMISE, FLAG.NO, 1, pushId = 3, headers = [[":method", "GET"],
                                                                   [":scheme", "http"],
                                                                   [":authority", "127.0.0.1"],
                                                                   [":path", "/"]])
    #con.send(TYPE.PRIORITY, FLAG.NO, 1, E = 1, depend = 1, weight = 1)
    time.sleep(0.2)
    con.send(TYPE.PING, ping = "hello!!")
    time.sleep(0.2)
    con.send(TYPE.WINDOW_UPDATE, streamId = 0, windowSizeIncrement = 10, R = 1)
    time.sleep(0.2)
    con.send(TYPE.WINDOW_UPDATE, streamId = 1, windowSizeIncrement = 10, R = 1)
    time.sleep(0.2)
    con.send(TYPE.RST_STREAM, streamId = 1, err = ERR_CODE.NO_ERROR)
    time.sleep(0.2)
    con.send(TYPE.GOAWAY, err = ERR_CODE.NO_ERROR, debug = None)

if __name__ == "__main__":
    host = "127.0.0.1"
    port = 8888

    if len(sys.argv) == 3:
        host, port = sys.argv[1], int(sys.argv[2])

    access(host, port)
<commit_msg>replace time.sleep to raw_input for easy debugging<commit_after>from settings import *
import sys
from binascii import hexlify
import time
from client import Client
from threading import Thread

def access(host, port):
    con = Client(host, port, False, True)
    con.notifyHTTP2()
    time.sleep(0.2)
    con.send(TYPE.SETTINGS, ident=SETTINGS.NO, value = "")
    time.sleep(0.2)
    raw_input("next")
    con.send(TYPE.HEADERS, FLAG.NO, 1, headers = [[":method", "GET"],
                                                  [":scheme", "http"],
                                                  [":authority", "127.0.0.1"],
                                                  [":path", "/"]])

    raw_input("next")
    con.send(TYPE.PUSH_PROMISE, FLAG.NO, 1, pushId = 3, headers = [[":method", "GET"],
                                                                   [":scheme", "http"],
                                                                   [":authority", "127.0.0.1"],
                                                                   [":path", "/"]])
    #con.send(TYPE.PRIORITY, FLAG.NO, 1, E = 1, depend = 1, weight = 1)
    raw_input("next")
    con.send(TYPE.PING, ping = "hello!!")
    raw_input("next")
    con.send(TYPE.WINDOW_UPDATE, streamId = 0, windowSizeIncrement = 10, R = 1)
    raw_input("next")
    con.send(TYPE.WINDOW_UPDATE, streamId = 1, windowSizeIncrement = 10, R = 1)
    raw_input("next")
    con.send(TYPE.RST_STREAM, streamId = 1, err = ERR_CODE.NO_ERROR)
    raw_input("next")
    con.send(TYPE.GOAWAY, err = ERR_CODE.NO_ERROR, debug = None)

if __name__ == "__main__":
    host = "127.0.0.1"
    port = 8888

    if len(sys.argv) == 3:
        host, port = sys.argv[1], int(sys.argv[2])

    access(host, port)
<|endoftext|>