<commit_before>from zipfile import ZipFile
from StringIO import StringIO
import datetime
from model_mommy import mommy
from django import test

from devilry.devilry_comment.models import Comment
from devilry.devilry_group import models as groupmodels
from devilry.devilry_group.views import feedbackfeed_bulkfiledownload
from django.core.files.base import ContentFile

class BulkDownloadTestClass(feedbackfeed_bulkfiledownload.BulkFileDownloadBaseView):
    def get_queryset(self, request):
        return groupmodels.FeedbackSet.objects.all()

    def get_zipfilename(self, request):
        return 'testfile.zip'

tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)

class TestBulkFileDownloadBase(test.TestCase):

    def test_get_zipfile(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_students_in_assignmentgroup(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        candidate2 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser2")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1.testuser2/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_attempts(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=False,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))
        feedbackset2 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_NEW_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs2_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset2,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs2_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs2_1, filename='testfile2.txt')
        commentfile_fbs2_1.file.save('testfile2.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt2/testfile2.txt')
        self.assertEquals(filecontents, "test2")

    def test_multiple_files_same_attempt_same_name(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        comment_fbs1_2 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_2 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2.file.save('testfile1.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1-1.txt')
        self.assertEquals(filecontents, "test2")

    def test_multiple_files_same_comment_same_name(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)

        comment_fbs1_2 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_2 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2.file.save('testfile1.txt', ContentFile('test2'))
        commentfile_fbs1_2_2 = mommy.make('devilry_comment.CommentFile',
                                          comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2_2.file.save('testfile1.txt', ContentFile('test3'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test2")
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1-1.txt')
        self.assertEquals(filecontents, "test3")

    def test_file_from_examiner(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_EXAMINER)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/from_examiner/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_file_after_deadline(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=yesterday)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/not_part_of_delivery/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_assignmentgroups(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        candidate1 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup1,
                                relatedstudent__user__shortname="testuser1")
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1, filename='testfile1.txt')
        commentfile_fbs1.file.save('testfile1.txt', ContentFile('test'))

        assignmentgroup2 = mommy.make('core.AssignmentGroup',
                                      parentnode=assignmentgroup1.parentnode)
        candidate2 = mommy.make('core.Candidate',
                                assignment_group=assignmentgroup2,
                                relatedstudent__user__shortname="testuser2")
        feedbackset2 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup2,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs2 = mommy.make('devilry_group.GroupComment',
                                  feedback_set=feedbackset2,
                                  user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs2 = mommy.make('devilry_comment.CommentFile',
                                      comment=comment_fbs2, filename='testfile2.txt')
        commentfile_fbs2.file.save('testfile1.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        file = ZipFile(StringIO(response.content))
        filecontents = file.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = file.read('test2100.spring2015.oblig1.testuser2/attempt1/testfile2.txt')
        self.assertEquals(filecontents, "test2")<commit_msg>devilry_group...feedbackfeed_bulkfiledownload: Cleanup tests for pep8.<commit_after>from zipfile import ZipFile
from StringIO import StringIO
import datetime
from model_mommy import mommy
from django import test

from devilry.devilry_comment.models import Comment
from devilry.devilry_group import models as groupmodels
from devilry.devilry_group.views import feedbackfeed_bulkfiledownload
from django.core.files.base import ContentFile

class BulkDownloadTestClass(feedbackfeed_bulkfiledownload.BulkFileDownloadBaseView):
    def get_queryset(self, request):
        return groupmodels.FeedbackSet.objects.all()

    def get_zipfilename(self, request):
        return 'testfile.zip'

class TestBulkFileDownloadBase(test.TestCase):

    def test_get_zipfile(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_students_in_assignmentgroup(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser2")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1.testuser2/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_attempts(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=False,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))
        feedbackset2 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_NEW_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs2_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset2,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs2_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs2_1, filename='testfile2.txt')
        commentfile_fbs2_1.file.save('testfile2.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt2/testfile2.txt')
        self.assertEquals(filecontents, "test2")

    def test_multiple_files_same_attempt_same_name(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        comment_fbs1_2 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_2 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2.file.save('testfile1.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1-1.txt')
        self.assertEquals(filecontents, "test2")

    def test_multiple_files_same_comment_same_name(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)

        comment_fbs1_2 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_2 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2.file.save('testfile1.txt', ContentFile('test2'))
        commentfile_fbs1_2_2 = mommy.make('devilry_comment.CommentFile',
                                          comment=comment_fbs1_2, filename='testfile1.txt')
        commentfile_fbs1_2_2.file.save('testfile1.txt', ContentFile('test3'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test2")
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1-1.txt')
        self.assertEquals(filecontents, "test3")

    def test_file_from_examiner(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_EXAMINER)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/from_examiner/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_file_after_deadline(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=yesterday)
        comment_fbs1_1 = mommy.make('devilry_group.GroupComment',
                                    feedback_set=feedbackset1,
                                    user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1_1 = mommy.make('devilry_comment.CommentFile',
                                        comment=comment_fbs1_1, filename='testfile1.txt')
        commentfile_fbs1_1.file.save('testfile1.txt', ContentFile('test'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read(
            'test2100.spring2015.oblig1.testuser1/attempt1/not_part_of_delivery/testfile1.txt')
        self.assertEquals(filecontents, "test")

    def test_multiple_assignmentgroups(self):
        assignmentgroup1 = mommy.make('core.AssignmentGroup',
                                      parentnode__parentnode__parentnode__short_name="test2100",
                                      parentnode__parentnode__short_name="spring2015",
                                      parentnode__short_name="oblig1")
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup1,
                   relatedstudent__user__shortname="testuser1")
        tomorrow = datetime.datetime.now() + datetime.timedelta(days=1)
        feedbackset1 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup1,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs1 = mommy.make('devilry_group.GroupComment',
                                  feedback_set=feedbackset1,
                                  user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs1 = mommy.make('devilry_comment.CommentFile',
                                      comment=comment_fbs1, filename='testfile1.txt')
        commentfile_fbs1.file.save('testfile1.txt', ContentFile('test'))

        assignmentgroup2 = mommy.make('core.AssignmentGroup',
                                      parentnode=assignmentgroup1.parentnode)
        mommy.make('core.Candidate',
                   assignment_group=assignmentgroup2,
                   relatedstudent__user__shortname="testuser2")
        feedbackset2 = mommy.make('devilry_group.FeedbackSet',
                                  group=assignmentgroup2,
                                  is_last_in_group=True,
                                  feedbackset_type=groupmodels.FeedbackSet.FEEDBACKSET_TYPE_FIRST_TRY,
                                  deadline_datetime=tomorrow)
        comment_fbs2 = mommy.make('devilry_group.GroupComment',
                                  feedback_set=feedbackset2,
                                  user_role=Comment.USER_ROLE_STUDENT)
        commentfile_fbs2 = mommy.make('devilry_comment.CommentFile',
                                      comment=comment_fbs2, filename='testfile2.txt')
        commentfile_fbs2.file.save('testfile1.txt', ContentFile('test2'))

        testclass = BulkDownloadTestClass()
        response = testclass.get(None)
        zipfileobject = ZipFile(StringIO(response.content))
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser1/attempt1/testfile1.txt')
        self.assertEquals(filecontents, "test")
        filecontents = zipfileobject.read('test2100.spring2015.oblig1.testuser2/attempt1/testfile2.txt')
        self.assertEquals(filecontents, "test2")
<|endoftext|>


<commit_before># Based on Drake's drake.bzl file,
# https://github.com/RobotLocomotion/drake/blob/master/tools/drake.bzl.

DREAL_VERSION = "4.19.06.2"

DREAL_PREFIX = "opt/dreal/%s" % DREAL_VERSION

PYTHON_VERSION_STRING = select({
    "@dreal//tools:PY3": "3",
    "@dreal//tools:PY2": "2.7",
})

PYTHON_PACKAGE_DIR = "lib/python" + PYTHON_VERSION_STRING + "/site-packages"

# The CXX_FLAGS will be enabled for all C++ rules in the project
# building with any compiler.
CXX_FLAGS = [
    "-Werror=all",
    "-Werror=attributes",
    "-Werror=deprecated",
    "-Werror=deprecated-declarations",
    "-Werror=ignored-qualifiers",
    "-Werror=old-style-cast",
    "-Werror=overloaded-virtual",
    "-Werror=shadow",
]

# The CLANG_FLAGS will be enabled for all C++ rules in the project when
# building with clang.
CLANG_FLAGS = CXX_FLAGS + [
    "-Werror=absolute-value",
    "-Werror=inconsistent-missing-override",
    "-Werror=non-virtual-dtor",
    "-Werror=return-stack-address",
    "-Werror=sign-compare",
]

# The GCC_FLAGS will be enabled for all C++ rules in the project when
# building with gcc.
GCC_FLAGS = CXX_FLAGS + [
    "-Werror=extra",
    "-Werror=logical-op",
    "-Werror=non-virtual-dtor",
    "-Werror=return-local-addr",
    "-Werror=unused-but-set-parameter",
]

# The GCC_CC_TEST_FLAGS will be enabled for all cc_test rules in the project
# when building with gcc.
GCC_CC_TEST_FLAGS = [
    "-Wno-unused-parameter",
]

def _platform_copts(rule_copts, cc_test = 0):
    """Returns both the rule_copts, and platform-specific copts.

    When cc_test=1, the GCC_CC_TEST_FLAGS will be added.  It should only be set
    to 1 from cc_test rules or rules that are boil down to cc_test rules.
    """
    extra_gcc_flags = []
    if cc_test:
        extra_gcc_flags = GCC_CC_TEST_FLAGS
    return select({
        "//tools:gcc": GCC_FLAGS + extra_gcc_flags + rule_copts,
        "//tools:clang": CLANG_FLAGS + rule_copts,
        "//tools:apple": CLANG_FLAGS + rule_copts,
        "//conditions:default": rule_copts,
    })

def _check_library_deps_blacklist(name, deps):
    """Report an error if a library should not use something from deps."""
    if not deps:
        return
    if type(deps) != "list":
        # We can't handle select() yet.
        return
    for dep in deps:
        if dep.endswith(":main"):
            fail("The cc_library '" + name + "' must not depend on a :main " +
                 "function from a cc_library; only cc_binary program should " +
                 "have a main function")

def dreal_cc_library(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ library.
    """
    _check_library_deps_blacklist(name, deps)
    native.cc_library(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        **kwargs
    )

def _make_search_paths(prefix, levels_to_root):
    return ",".join(
        [
            "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level))
            for search_level in range(levels_to_root + 1)
        ],
    )

def dreal_pybind_library(
        name,
        py_srcs = [],
        py_deps = [],
        cc_srcs = [],
        cc_deps = []):
    """Creates a rule to declare a pybind library.

    Note that `cc_deps` should includes header-only dependencies.
    """
    cc_so_name = "_" + name + ".so"

    # The last +3 is for "lib/python*/site-packages".
    levels_to_root = native.package_name().count("/") + name.count("/") + 3
    dreal_cc_binary(
        name = cc_so_name,
        srcs = cc_srcs,
        linkshared = 1,
        linkstatic = 1,
        deps = cc_deps + [
            "//:dreal_shared_library",
            "@pybind11",
        ],
        copts = [
            "-fvisibility=hidden",
        ],
        linkopts = select({
            "@dreal//tools:linux": [
                "-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
            ],
            "@//conditions:default": [],
        }),
    )
    native.py_library(
        name = name,
        srcs = py_srcs,
        data = [
            cc_so_name,
        ],
        deps = py_deps,
        srcs_version = "PY2AND3",
        visibility = ["//dreal:__subpackages__"],
    )

def dreal_cc_binary(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        testonly = 0,
        add_test_rule = 0,
        test_rule_args = [],
        test_rule_size = None,
        **kwargs):
    """Creates a rule to declare a C++ binary.

    If you wish to create a smoke-test demonstrating that your binary runs
    without crashing, supply add_test_rule=1. Note that if you wish to do
    this, you should consider suppressing that urge, and instead writing real
    tests. The smoke-test will be named <name>_test. You may override cc_test
    defaults using test_rule_args=["-f", "--bar=42"] or test_rule_size="baz".
    """
    native.cc_binary(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        testonly = testonly,
        **kwargs
    )

    if add_test_rule:
        dreal_cc_test(
            name = name + "_test",
            hdrs = hdrs,
            srcs = srcs,
            deps = deps,
            copts = copts,
            size = test_rule_size,
            testonly = testonly,
            args = test_rule_args,
            **kwargs
        )

def dreal_cc_test(
        name,
        size = None,
        srcs = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ unit test.

    Note that for almost all cases, dreal_cc_googletest should be
    used, instead of this rule.

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename
    convention.

    """
    if size == None:
        size = "small"
    if srcs == None:
        srcs = ["test/%s.cc" % name]
    native.cc_test(
        name = name,
        size = size,
        srcs = srcs,
        copts = _platform_copts(copts, cc_test = 1),
        **kwargs
    )

def dreal_cc_googletest(
        name,
        deps = None,
        use_default_main = True,
        **kwargs):
    """Creates a rule to declare a C++ unit test using googletest.

    Always adds a deps= entry for googletest main
    (@com_google_googletest//:gtest_main).

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename convention.
    By default, sets use_default_main=True to use GTest's main, via
    @com_google_googletest//:gtest_main. Otherwise, it will depend on
    @com_google_googlegtest//:gtest.

    """
    if deps == None:
        deps = []
    if use_default_main:
        deps.append("@com_google_googletest//:gtest_main")
    else:
        deps.append("@com_google_googletest//:gtest")
    dreal_cc_test(
        name = name,
        deps = deps,
        **kwargs
    )

def smt2_test(
        name,
        smt2 = None,
        options = [],
        tags = [],
        **kwargs):
    """Create smt2 test."""
    if not smt2:
        smt2 = name + ".smt2"
    expected = smt2 + ".expected"
    data_files = native.glob([
        smt2 + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % smt2,
            "$(location %s)" % expected,
        ] + options,
        tags = tags + ["smt2"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

def dr_test(
        name,
        args = [],
        **kwargs):
    """Create dr test."""
    dr = name + ".dr"
    expected = dr + ".expected"
    data_files = native.glob([
        dr + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % dr,
            "$(location %s)" % expected,
        ] + args,
        tags = ["dr"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

# Generate a file with specified content
def _generate_file_impl(ctx):
    ctx.actions.write(output = ctx.outputs.out, content = ctx.attr.content)

dreal_generate_file = rule(
    attrs = {
        "content": attr.string(),
        "out": attr.output(mandatory = True),
    },
    output_to_genfiles = True,
    implementation = _generate_file_impl,
)
<commit_msg>bazel(tools/dreal.bzl): Add dreal_py_test<commit_after># Based on Drake's drake.bzl file,
# https://github.com/RobotLocomotion/drake/blob/master/tools/drake.bzl.

DREAL_VERSION = "4.19.06.2"

DREAL_PREFIX = "opt/dreal/%s" % DREAL_VERSION

PYTHON_VERSION_STRING = select({
    "@dreal//tools:PY3": "3",
    "@dreal//tools:PY2": "2.7",
})

PYTHON_PACKAGE_DIR = "lib/python" + PYTHON_VERSION_STRING + "/site-packages"

# The CXX_FLAGS will be enabled for all C++ rules in the project
# building with any compiler.
CXX_FLAGS = [
    "-Werror=all",
    "-Werror=attributes",
    "-Werror=deprecated",
    "-Werror=deprecated-declarations",
    "-Werror=ignored-qualifiers",
    "-Werror=old-style-cast",
    "-Werror=overloaded-virtual",
    "-Werror=shadow",
]

# The CLANG_FLAGS will be enabled for all C++ rules in the project when
# building with clang.
CLANG_FLAGS = CXX_FLAGS + [
    "-Werror=absolute-value",
    "-Werror=inconsistent-missing-override",
    "-Werror=non-virtual-dtor",
    "-Werror=return-stack-address",
    "-Werror=sign-compare",
]

# The GCC_FLAGS will be enabled for all C++ rules in the project when
# building with gcc.
GCC_FLAGS = CXX_FLAGS + [
    "-Werror=extra",
    "-Werror=logical-op",
    "-Werror=non-virtual-dtor",
    "-Werror=return-local-addr",
    "-Werror=unused-but-set-parameter",
]

# The GCC_CC_TEST_FLAGS will be enabled for all cc_test rules in the project
# when building with gcc.
GCC_CC_TEST_FLAGS = [
    "-Wno-unused-parameter",
]

def _platform_copts(rule_copts, cc_test = 0):
    """Returns both the rule_copts, and platform-specific copts.

    When cc_test=1, the GCC_CC_TEST_FLAGS will be added.  It should only be set
    to 1 from cc_test rules or rules that are boil down to cc_test rules.
    """
    extra_gcc_flags = []
    if cc_test:
        extra_gcc_flags = GCC_CC_TEST_FLAGS
    return select({
        "//tools:gcc": GCC_FLAGS + extra_gcc_flags + rule_copts,
        "//tools:clang": CLANG_FLAGS + rule_copts,
        "//tools:apple": CLANG_FLAGS + rule_copts,
        "//conditions:default": rule_copts,
    })

def _check_library_deps_blacklist(name, deps):
    """Report an error if a library should not use something from deps."""
    if not deps:
        return
    if type(deps) != "list":
        # We can't handle select() yet.
        return
    for dep in deps:
        if dep.endswith(":main"):
            fail("The cc_library '" + name + "' must not depend on a :main " +
                 "function from a cc_library; only cc_binary program should " +
                 "have a main function")

def dreal_cc_library(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ library.
    """
    _check_library_deps_blacklist(name, deps)
    native.cc_library(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        **kwargs
    )

def _make_search_paths(prefix, levels_to_root):
    return ",".join(
        [
            "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level))
            for search_level in range(levels_to_root + 1)
        ],
    )

def dreal_pybind_library(
        name,
        py_srcs = [],
        py_deps = [],
        cc_srcs = [],
        cc_deps = []):
    """Creates a rule to declare a pybind library.

    Note that `cc_deps` should includes header-only dependencies.
    """
    cc_so_name = "_" + name + ".so"

    # The last +3 is for "lib/python*/site-packages".
    levels_to_root = native.package_name().count("/") + name.count("/") + 3
    dreal_cc_binary(
        name = cc_so_name,
        srcs = cc_srcs,
        linkshared = 1,
        linkstatic = 1,
        deps = cc_deps + [
            "//:dreal_shared_library",
            "@pybind11",
        ],
        copts = [
            "-fvisibility=hidden",
        ],
        linkopts = select({
            "@dreal//tools:linux": [
                "-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),),
            ],
            "@//conditions:default": [],
        }),
    )
    native.py_library(
        name = name,
        srcs = py_srcs,
        data = [
            cc_so_name,
        ],
        deps = py_deps,
        srcs_version = "PY2AND3",
        visibility = ["//dreal:__subpackages__"],
    )

def dreal_cc_binary(
        name,
        hdrs = None,
        srcs = None,
        deps = None,
        copts = [],
        testonly = 0,
        add_test_rule = 0,
        test_rule_args = [],
        test_rule_size = None,
        **kwargs):
    """Creates a rule to declare a C++ binary.

    If you wish to create a smoke-test demonstrating that your binary runs
    without crashing, supply add_test_rule=1. Note that if you wish to do
    this, you should consider suppressing that urge, and instead writing real
    tests. The smoke-test will be named <name>_test. You may override cc_test
    defaults using test_rule_args=["-f", "--bar=42"] or test_rule_size="baz".
    """
    native.cc_binary(
        name = name,
        hdrs = hdrs,
        srcs = srcs,
        deps = deps,
        copts = _platform_copts(copts),
        testonly = testonly,
        **kwargs
    )

    if add_test_rule:
        dreal_cc_test(
            name = name + "_test",
            hdrs = hdrs,
            srcs = srcs,
            deps = deps,
            copts = copts,
            size = test_rule_size,
            testonly = testonly,
            args = test_rule_args,
            **kwargs
        )

def dreal_cc_test(
        name,
        size = None,
        srcs = None,
        copts = [],
        **kwargs):
    """Creates a rule to declare a C++ unit test.

    Note that for almost all cases, dreal_cc_googletest should be
    used, instead of this rule.

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename
    convention.

    """
    if size == None:
        size = "small"
    if srcs == None:
        srcs = ["test/%s.cc" % name]
    native.cc_test(
        name = name,
        size = size,
        srcs = srcs,
        copts = _platform_copts(copts, cc_test = 1),
        **kwargs
    )

def dreal_py_test(
        name,
        srcs,
        main,
        tags,
        deps,
        **kwargs):
    """Creates a rule to declare a Python unit test."""
    native.py_test(
        name = name,
        srcs = srcs,
        main = main,
        tags = tags,
        deps = deps,
        srcs_version = "PY2AND3",
        python_version = "PY2",
        **kwargs
    )

def dreal_cc_googletest(
        name,
        deps = None,
        use_default_main = True,
        **kwargs):
    """Creates a rule to declare a C++ unit test using googletest.

    Always adds a deps= entry for googletest main
    (@com_google_googletest//:gtest_main).

    By default, sets size="small" because that indicates a unit test.
    By default, sets name="test/${name}.cc" per Dreal's filename convention.
    By default, sets use_default_main=True to use GTest's main, via
    @com_google_googletest//:gtest_main. Otherwise, it will depend on
    @com_google_googlegtest//:gtest.

    """
    if deps == None:
        deps = []
    if use_default_main:
        deps.append("@com_google_googletest//:gtest_main")
    else:
        deps.append("@com_google_googletest//:gtest")
    dreal_cc_test(
        name = name,
        deps = deps,
        **kwargs
    )

def smt2_test(
        name,
        smt2 = None,
        options = [],
        tags = [],
        **kwargs):
    """Create smt2 test."""
    if not smt2:
        smt2 = name + ".smt2"
    expected = smt2 + ".expected"
    data_files = native.glob([
        smt2 + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % smt2,
            "$(location %s)" % expected,
        ] + options,
        tags = tags + ["smt2"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

def dr_test(
        name,
        args = [],
        **kwargs):
    """Create dr test."""
    dr = name + ".dr"
    expected = dr + ".expected"
    data_files = native.glob([
        dr + "*",
    ])
    native.py_test(
        name = name,
        args = [
            "$(location //dreal:dreal)",
            "$(location %s)" % dr,
            "$(location %s)" % expected,
        ] + args,
        tags = ["dr"],
        srcs = ["test.py"],
        data = [
            "//dreal:dreal",
        ] + data_files,
        main = "test.py",
        srcs_version = "PY2AND3",
        **kwargs
    )

# Generate a file with specified content
def _generate_file_impl(ctx):
    ctx.actions.write(output = ctx.outputs.out, content = ctx.attr.content)

dreal_generate_file = rule(
    attrs = {
        "content": attr.string(),
        "out": attr.output(mandatory = True),
    },
    output_to_genfiles = True,
    implementation = _generate_file_impl,
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import sys

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand
from django_js_reverse.core import generate_js
from django_js_reverse.js_reverse_settings import JS_OUTPUT_PATH

try:
    from django.urls import get_resolver
except ImportError:
    from django.core.urlresolvers import get_resolver

class Command(BaseCommand):
    help = 'Creates a static urls-js file for django-js-reverse'

    def get_location(self):
        output_path = getattr(settings, 'JS_REVERSE_OUTPUT_PATH', JS_OUTPUT_PATH)
        if output_path:
            return output_path

        if not hasattr(settings, 'STATIC_ROOT') or not settings.STATIC_ROOT:
            raise ImproperlyConfigured(
                'The collectstatic_js_reverse command needs settings.JS_REVERSE_OUTPUT_PATH or settings.STATIC_ROOT to be set.')

        return os.path.join(settings.STATIC_ROOT, 'django_js_reverse', 'js')

    def handle(self, *args, **options):
        location = self.get_location()
        file = 'reverse.js'
        fs = FileSystemStorage(location=location)
        if fs.exists(file):
            fs.delete(file)

        default_urlresolver = get_resolver(None)
        content = generate_js(default_urlresolver)
        fs.save(file, ContentFile(content))
        if len(sys.argv) > 1 and sys.argv[1] in ['collectstatic_js_reverse']:
            self.stdout.write('js-reverse file written to %s' % (location))  # pragma: no cover
<commit_msg>Database presence should not be required for generating js file<commit_after># -*- coding: utf-8 -*-
import os
import sys

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage
from django.core.management.base import BaseCommand
from django_js_reverse.core import generate_js
from django_js_reverse.js_reverse_settings import JS_OUTPUT_PATH

try:
    from django.urls import get_resolver
except ImportError:
    from django.core.urlresolvers import get_resolver

class Command(BaseCommand):
    help = 'Creates a static urls-js file for django-js-reverse'
    requires_system_checks = False
    def get_location(self):
        output_path = getattr(settings, 'JS_REVERSE_OUTPUT_PATH', JS_OUTPUT_PATH)
        if output_path:
            return output_path

        if not hasattr(settings, 'STATIC_ROOT') or not settings.STATIC_ROOT:
            raise ImproperlyConfigured(
                'The collectstatic_js_reverse command needs settings.JS_REVERSE_OUTPUT_PATH or settings.STATIC_ROOT to be set.')

        return os.path.join(settings.STATIC_ROOT, 'django_js_reverse', 'js')

    def handle(self, *args, **options):
        location = self.get_location()
        file = 'reverse.js'
        fs = FileSystemStorage(location=location)
        if fs.exists(file):
            fs.delete(file)

        default_urlresolver = get_resolver(None)
        content = generate_js(default_urlresolver)
        fs.save(file, ContentFile(content))
        if len(sys.argv) > 1 and sys.argv[1] in ['collectstatic_js_reverse']:
            self.stdout.write('js-reverse file written to %s' % (location))  # pragma: no cover
<|endoftext|>


<commit_before>import os
import sys
curr_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(curr_path, "../python"))
import mxnet as mx
import random
import numpy as np
import argparse
import threading
import cv, cv2
import time
def list_image(root, recursive, exts):
    image_list = []
    if recursive:
        cat = {}
        for path, subdirs, files in os.walk(root):
            print(len(cat), path)
            for fname in files:
                fpath = os.path.join(path, fname)
                suffix = os.path.splitext(fname)[1].lower()
                if os.path.isfile(fpath) and (suffix in exts):
                    if path not in cat:
                        cat[path] = len(cat)
                    image_list.append((len(image_list), os.path.relpath(fpath, root), cat[path]))
    else:
        for fname in os.listdir(root):
            fpath = os.path.join(root, fname)
            suffix = os.path.splitext(fname)[1].lower()
            if os.path.isfile(fpath) and (suffix in exts):
                image_list.append((len(image_list), os.path.relpath(fpath, root), 0))
    return image_list

def write_list(path_out, image_list):
    with open(path_out, 'w') as fout:
        for i in xrange(len(image_list)):
            line = '%d\t'%image_list[i][0]
            for j in image_list[i][2:]:
                line += '%f\t'%j
            line += '%s\n'%image_list[i][1]
            fout.write(line)

def make_list(prefix_out, root, recursive, exts, num_chunks, train_ratio):
    image_list = list_image(root, recursive, exts)
    random.shuffle(image_list)
    N = len(image_list)
    chunk_size = (N+num_chunks-1)/num_chunks
    for i in xrange(num_chunks):
        chunk = image_list[i*chunk_size:(i+1)*chunk_size]
        if num_chunks > 1:
            str_chunk = '_%d'%i
        else:
            str_chunk = ''
        if train_ratio < 1:
            sep = int(chunk_size*train_ratio)
            write_list(prefix_out+str_chunk+'_train.lst', chunk[:sep])
            write_list(prefix_out+str_chunk+'_val.lst', chunk[sep:])
        else:
            write_list(prefix_out+str_chunk+'.lst', chunk)

def read_list(path_in):
    image_list = []
    with open(path_in) as fin:
        for line in fin.readlines():
            line = [i.strip() for i in line.strip().split('\t')]
            item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]]
            image_list.append(item)
    return image_list

def write_record(args, image_list):
    source = image_list
    tic = [time.time()]
    color_modes = {-1: cv2.IMREAD_UNCHANGED,
                    0: cv2.IMREAD_GRAYSCALE,
                    1: cv2.IMREAD_COLOR}
    total = len(source)

    def image_encode(item, q_out):
        try:
            img = cv2.imread(os.path.join(args.root, item[1]), color_modes[args.color])
        except:
            print 'imread error:', item[1]
            return
        if img == None:
            print 'read none error:', item[1]
            return
        if args.center_crop:
            if img.shape[0] > img.shape[1]:
                margin = (img.shape[0] - img.shape[1])/2;
                img = img[margin:margin+img.shape[1], :]
            else:
                margin = (img.shape[1] - img.shape[0])/2;
                img = img[:, margin:margin+img.shape[0]]
        if args.resize:
            if img.shape[0] > img.shape[1]:
                newsize = (img.shape[0]*args.resize/img.shape[1], args.resize)
            else:
                newsize = (args.resize, img.shape[1]*args.resize/img.shape[0])
            img = cv2.resize(img, newsize)
        header = mx.recordio.IRHeader(0, item[2], item[0], 0)

        try:
            s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding)
            q_out.put(('data', s))
        except:
            print 'pack_img error:',item[1]
            return

    def read_worker(q_in, q_out):
        while not q_in.empty():
            item = q_in.get()
            image_encode(item, q_out)

    def write_worker(q_out, prefix):
        pre_time = time.time()
        cnt = 0
        record = mx.recordio.MXRecordIO(prefix+'.rec', 'w')
        while True:
            stat, s = q_out.get()
            if stat == 'finish':
                break
            record.write(s)
            cnt += 1
            if cnt % 1000 == 0:
                cur_time = time.time()
                print 'time:', cur_time - pre_time, ' count:', cnt
                pre_time = cur_time

    try:
        import multiprocessing
        q_in = [multiprocessing.Queue() for i in range(args.num_thread)]
        q_out = multiprocessing.Queue(1024)
        for i in range(len(image_list)):
            q_in[i % len(q_in)].put(image_list[i])
        read_process = [multiprocessing.Process(target=read_worker, args=(q_in[i], q_out)) \
                for i in range(args.num_thread)]
        for p in read_process:
            p.start()
        write_process = multiprocessing.Process(target=write_worker, args=(q_out,args.prefix))
        write_process.start()
        for p in read_process:
            p.join()
        q_out.put(('finish', ''))
        write_process.join()
    except ImportError:
        print('multiprocessing not available, fall back to single threaded encoding')
        import Queue
        q_out = Queue.Queue()
        record = mx.recordio.MXRecordIO(args.prefix+'.rec', 'w')
        cnt = 0
        pre_time = time.time()
        for item in image_list:
            image_encode(item, q_out)
            if q_out.empty():
                continue
            _, s = q_out.get()
            record.write(s)
            cnt += 1
            if cnt % 1000 == 0:
                cur_time = time.time()
                print 'time:', cur_time - pre_time, ' count:', cnt
                pre_time = cur_time

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description='Make an image record database by reading from\
        an image list or creating one')
    parser.add_argument('prefix', help='prefix of input/output files.')
    parser.add_argument('root', help='path to folder containing images.')

    cgroup = parser.add_argument_group('Options for creating image lists')
    cgroup.add_argument('--list', type=bool, default=False,
        help='If this is set im2rec will create image list(s) by traversing root folder\
        and output to <prefix>.lst.\
        Otherwise im2rec will read <prefix>.lst and create a database at <prefix>.rec')
    cgroup.add_argument('--exts', type=list, default=['.jpeg','.jpg'],
        help='list of acceptable image extensions.')
    cgroup.add_argument('--chunks', type=int, default=1, help='number of chunks.')
    cgroup.add_argument('--train_ratio', type=float, default=1.0,
        help='Ratio of images to use for training.')
    cgroup.add_argument('--recursive', type=bool, default=False,
        help='If true recursively walk through subdirs and assign an unique label\
        to images in each folder. Otherwise only include images in the root folder\
        and give them label 0.')

    rgroup = parser.add_argument_group('Options for creating database')
    rgroup.add_argument('--resize', type=int, default=0,
        help='resize the shorter edge of image to the newsize, original images will\
        be packed by default.')
    rgroup.add_argument('--center_crop', type=bool, default=False,
        help='specify whether to crop the center image to make it rectangular.')
    rgroup.add_argument('--quality', type=int, default=80,
        help='JPEG quality for encoding, 1-100; or PNG compression for encoding, 1-9')
    rgroup.add_argument('--num_thread', type=int, default=1,
        help='number of thread to use for encoding. order of images will be different\
        from the input list if >1. the input list will be modified to match the\
        resulting order.')
    rgroup.add_argument('--color', type=int, default=1, choices=[-1, 0, 1],
        help='specify the color mode of the loaded image.\
        1: Loads a color image. Any transparency of image will be neglected. It is the default flag.\
        0: Loads image in grayscale mode.\
        -1:Loads image as such including alpha channel.')
    rgroup.add_argument('--encoding', type=str, default='.jpg', choices=['.jpg', '.png'],
        help='specify the encoding of the images.')

    args = parser.parse_args()

    if args.list:
        make_list(args.prefix, args.root, args.recursive,
                  args.exts, args.chunks, args.train_ratio)
    else:
        image_list = read_list(args.prefix+'.lst')
        write_record(args, image_list)

if __name__ == '__main__':
    main()
<commit_msg>presever image order for multiprocessing<commit_after>import os
import sys
curr_path = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.join(curr_path, "../python"))
import mxnet as mx
import random
import numpy as np
import argparse
import threading
import cv, cv2
import time
def list_image(root, recursive, exts):
    image_list = []
    if recursive:
        cat = {}
        for path, subdirs, files in os.walk(root):
            print(len(cat), path)
            for fname in files:
                fpath = os.path.join(path, fname)
                suffix = os.path.splitext(fname)[1].lower()
                if os.path.isfile(fpath) and (suffix in exts):
                    if path not in cat:
                        cat[path] = len(cat)
                    image_list.append((len(image_list), os.path.relpath(fpath, root), cat[path]))
    else:
        for fname in os.listdir(root):
            fpath = os.path.join(root, fname)
            suffix = os.path.splitext(fname)[1].lower()
            if os.path.isfile(fpath) and (suffix in exts):
                image_list.append((len(image_list), os.path.relpath(fpath, root), 0))
    return image_list

def write_list(path_out, image_list):
    with open(path_out, 'w') as fout:
        for i in xrange(len(image_list)):
            line = '%d\t'%image_list[i][0]
            for j in image_list[i][2:]:
                line += '%f\t'%j
            line += '%s\n'%image_list[i][1]
            fout.write(line)

def make_list(prefix_out, root, recursive, exts, num_chunks, train_ratio):
    image_list = list_image(root, recursive, exts)
    random.shuffle(image_list)
    N = len(image_list)
    chunk_size = (N+num_chunks-1)/num_chunks
    for i in xrange(num_chunks):
        chunk = image_list[i*chunk_size:(i+1)*chunk_size]
        if num_chunks > 1:
            str_chunk = '_%d'%i
        else:
            str_chunk = ''
        if train_ratio < 1:
            sep = int(chunk_size*train_ratio)
            write_list(prefix_out+str_chunk+'_train.lst', chunk[:sep])
            write_list(prefix_out+str_chunk+'_val.lst', chunk[sep:])
        else:
            write_list(prefix_out+str_chunk+'.lst', chunk)

def read_list(path_in):
    image_list = []
    with open(path_in) as fin:
        for line in fin.readlines():
            line = [i.strip() for i in line.strip().split('\t')]
            item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]]
            image_list.append(item)
    return image_list

def write_record(args, image_list):
    source = image_list
    tic = [time.time()]
    color_modes = {-1: cv2.IMREAD_UNCHANGED,
                    0: cv2.IMREAD_GRAYSCALE,
                    1: cv2.IMREAD_COLOR}
    total = len(source)

    def image_encode(item, q_out):
        try:
            img = cv2.imread(os.path.join(args.root, item[1]), color_modes[args.color])
        except:
            print 'imread error:', item[1]
            return
        if img == None:
            print 'read none error:', item[1]
            return
        if args.center_crop:
            if img.shape[0] > img.shape[1]:
                margin = (img.shape[0] - img.shape[1])/2;
                img = img[margin:margin+img.shape[1], :]
            else:
                margin = (img.shape[1] - img.shape[0])/2;
                img = img[:, margin:margin+img.shape[0]]
        if args.resize:
            if img.shape[0] > img.shape[1]:
                newsize = (img.shape[0]*args.resize/img.shape[1], args.resize)
            else:
                newsize = (args.resize, img.shape[1]*args.resize/img.shape[0])
            img = cv2.resize(img, newsize)
        header = mx.recordio.IRHeader(0, item[2], item[0], 0)

        try:
            s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding)
            q_out.put(('data', s, item))
        except:
            print 'pack_img error:',item[1]
            return

    def read_worker(q_in, q_out):
        while not q_in.empty():
            item = q_in.get()
            image_encode(item, q_out)

    def write_worker(q_out, prefix):
        pre_time = time.time()
        sink = []
        record = mx.recordio.MXRecordIO(prefix+'.rec', 'w')
        while True:
            stat, s, item = q_out.get()
            if stat == 'finish':
                write_list(prefix+'.lst', sink)
                break
            record.write(s)
            sink.append(item)
            if len(sink) % 1000 == 0:
                cur_time = time.time()
                print 'time:', cur_time - pre_time, ' count:', len(sink)
                pre_time = cur_time

    try:
        import multiprocessing
        q_in = [multiprocessing.Queue() for i in range(args.num_thread)]
        q_out = multiprocessing.Queue(1024)
        for i in range(len(image_list)):
            q_in[i % len(q_in)].put(image_list[i])
        read_process = [multiprocessing.Process(target=read_worker, args=(q_in[i], q_out)) \
                for i in range(args.num_thread)]
        for p in read_process:
            p.start()
        write_process = multiprocessing.Process(target=write_worker, args=(q_out,args.prefix))
        write_process.start()
        for p in read_process:
            p.join()
        q_out.put(('finish', '', []))
        write_process.join()
    except ImportError:
        print('multiprocessing not available, fall back to single threaded encoding')
        import Queue
        q_out = Queue.Queue()
        record = mx.recordio.MXRecordIO(args.prefix+'.rec', 'w')
        cnt = 0
        pre_time = time.time()
        for item in image_list:
            image_encode(item, q_out)
            if q_out.empty():
                continue
            _, s, _ = q_out.get()
            record.write(s)
            cnt += 1
            if cnt % 1000 == 0:
                cur_time = time.time()
                print 'time:', cur_time - pre_time, ' count:', cnt
                pre_time = cur_time

def main():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description='Make an image record database by reading from\
        an image list or creating one')
    parser.add_argument('prefix', help='prefix of input/output files.')
    parser.add_argument('root', help='path to folder containing images.')

    cgroup = parser.add_argument_group('Options for creating image lists')
    cgroup.add_argument('--list', type=bool, default=False,
        help='If this is set im2rec will create image list(s) by traversing root folder\
        and output to <prefix>.lst.\
        Otherwise im2rec will read <prefix>.lst and create a database at <prefix>.rec')
    cgroup.add_argument('--exts', type=list, default=['.jpeg','.jpg'],
        help='list of acceptable image extensions.')
    cgroup.add_argument('--chunks', type=int, default=1, help='number of chunks.')
    cgroup.add_argument('--train_ratio', type=float, default=1.0,
        help='Ratio of images to use for training.')
    cgroup.add_argument('--recursive', type=bool, default=False,
        help='If true recursively walk through subdirs and assign an unique label\
        to images in each folder. Otherwise only include images in the root folder\
        and give them label 0.')

    rgroup = parser.add_argument_group('Options for creating database')
    rgroup.add_argument('--resize', type=int, default=0,
        help='resize the shorter edge of image to the newsize, original images will\
        be packed by default.')
    rgroup.add_argument('--center_crop', type=bool, default=False,
        help='specify whether to crop the center image to make it rectangular.')
    rgroup.add_argument('--quality', type=int, default=80,
        help='JPEG quality for encoding, 1-100; or PNG compression for encoding, 1-9')
    rgroup.add_argument('--num_thread', type=int, default=1,
        help='number of thread to use for encoding. order of images will be different\
        from the input list if >1. the input list will be modified to match the\
        resulting order.')
    rgroup.add_argument('--color', type=int, default=1, choices=[-1, 0, 1],
        help='specify the color mode of the loaded image.\
        1: Loads a color image. Any transparency of image will be neglected. It is the default flag.\
        0: Loads image in grayscale mode.\
        -1:Loads image as such including alpha channel.')
    rgroup.add_argument('--encoding', type=str, default='.jpg', choices=['.jpg', '.png'],
        help='specify the encoding of the images.')

    args = parser.parse_args()

    if args.list:
        make_list(args.prefix, args.root, args.recursive,
                  args.exts, args.chunks, args.train_ratio)
    else:
        image_list = read_list(args.prefix+'.lst')
        write_record(args, image_list)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import os
import json
import logging
from datetime import datetime

from tweepy import Stream, StreamListener
from tweepy.models import Status

from yubari.lib.twitter import ttapi
from yubari.lib.qq import qqbot

logger = logging.getLogger(__name__)

kancolle_PROFILE_IMAGE_TMP = '/tmp/twitter_kancolle_staff_profile_image'

TWITTERS = {
    "KanColle_STAFF": "294025417",
    "maesanpicture": "2381595966",
    "komatan": "96604067",
    "Strangestone": "93332575"
}

def get_previous_profile_img():
    if not os.path.exists(kancolle_PROFILE_IMAGE_TMP):
        logger.warning("kancolle profile image not exists")
        return ""
    with open(kancolle_PROFILE_IMAGE_TMP, 'r') as f:
        url = f.read()
        if url and url.startswith("https://"):
            return url
    return ""

def update_profile_img(img):
    previous_image = get_previous_profile_img()
    if previous_image and previous_image == img:
        return
    logger.info("profile image changed from [%s] to [%s]", previous_image, img)
    send_profile_image(img)
    with open(kancolle_PROFILE_IMAGE_TMP, 'w') as f:
        f.write(img)

# TODO
def send_profile_image(img):
    qqbot.sendGroupMsg(img.replace("_normal", ""))

class MyStreamListener(StreamListener):
    def on_data(self, raw_data):
        data = json.loads(raw_data)
        if "event" in data:
            logger.debug("ignore event: %s", data["event"])
            return
        status = Status.parse(self.api, data)
        user = getattr(status, "user", None)
        if not user:
            logger.debug("empty user")
            return
        if user.id_str not in TWITTERS.values():
            logger.debug("other user")
            return
        if getattr(status, "retweeted_status", None):
            logger.debug("ignore retweet from: %s", user.name)
            return
        if user.id_str == TWITTERS["KanColle_STAFF"]:
            self.proceed_kancolle(status)
        elif user.id_str == TWITTERS["maesanpicture"]:
            self.proceed_samidare(status)
        elif user.id_str == TWITTERS["komatan"]:
            self.proceed_komatan(status)
        elif user.id_str == TWITTERS["Strangestone"]:
            self.proceed_tawawa(status)
        else:
            logger.debug("ignore user: %s", user.name)

    def proceed_kancolle(self, status):
        _tags = status.entities.get("hashtags", [])
        tags = [t["text"] for t in _tags]
        if "艦これ" not in tags:
            return
        logger.info("kancolle: %s", status.text.replace("\n", " "))
        user = status.user
        update_profile_img(user.profile_image_url_https)
        msg = [user.name]
        timestamp = int(int(status.timestamp_ms) / 1000)
        msg.append(datetime.fromtimestamp(timestamp).ctime())
        msg.append(status.text)
        qqbot.sendGroupMsg('\n'.join(msg))

    def proceed_samidare(self, status):
        _tags = status.entities.get("hashtags", [])
        tags = [t["text"] for t in _tags]
        if "毎日五月雨" not in tags:
            return
        logger.info("maesan: %s", status.text.replace("\n", " "))
        medias = status.entities.get("media", [])
        for media in medias:
            logger.info("samidare: %s", media["media_url_https"])
            qqbot.sendSelfMsg(media["media_url_https"])

    def proceed_komatan(self, status):
        medias = status.entities.get("media", [])
        if not medias:
            return
        logger.info("komatan: %s", status.text.replace("\n", " "))
        for media in medias:
            logger.info("komatan: %s", media["media_url_https"])
            qqbot.sendSelfMsg(media["media_url_https"])

    def proceed_tawawa(self, status):
        if not status.text.startswith("月曜日のたわわ"):
            return
        logger.info("tawawa: %s", status.text.replace("\n", " "))
        medias = status.entities.get("media", [])
        for media in medias:
            logger.info("tawawa: %s", media["media_url_https"])
            qqbot.sendSelfMsg(media["media_url_https"])

    def on_error(self, code):
        logger.error("error: %s", code)

def run():
    msl = Stream(auth=ttapi.auth, listener=MyStreamListener())
    msl.filter(follow=TWITTERS.values())

if __name__ == '__main__':
    run()
<commit_msg>return if not img<commit_after>#!/usr/bin/env python
# coding: utf-8

import os
import json
import logging
from datetime import datetime

from tweepy import Stream, StreamListener
from tweepy.models import Status

from yubari.lib.twitter import ttapi
from yubari.lib.qq import qqbot

logger = logging.getLogger(__name__)

kancolle_PROFILE_IMAGE_TMP = '/tmp/twitter_kancolle_staff_profile_image'

TWITTERS = {
    "KanColle_STAFF": "294025417",
    "maesanpicture": "2381595966",
    "komatan": "96604067",
    "Strangestone": "93332575"
}

def get_previous_profile_img():
    if not os.path.exists(kancolle_PROFILE_IMAGE_TMP):
        logger.warning("kancolle profile image not exists")
        return ""
    with open(kancolle_PROFILE_IMAGE_TMP, 'r') as f:
        url = f.read()
        if url and url.startswith("https://"):
            return url
    return ""

def update_profile_img(img):
    if not img:
        return
    previous_image = get_previous_profile_img()
    if previous_image and previous_image == img:
        return
    logger.info("profile image changed from [%s] to [%s]", previous_image, img)
    send_profile_image(img)
    with open(kancolle_PROFILE_IMAGE_TMP, 'w') as f:
        f.write(img)

# TODO
def send_profile_image(img):
    qqbot.sendGroupMsg(img.replace("_normal", ""))

class MyStreamListener(StreamListener):
    def on_data(self, raw_data):
        data = json.loads(raw_data)
        if "event" in data:
            logger.debug("ignore event: %s", data["event"])
            return
        status = Status.parse(self.api, data)
        user = getattr(status, "user", None)
        if not user:
            logger.debug("empty user")
            return
        if user.id_str not in TWITTERS.values():
            logger.debug("other user")
            return
        if getattr(status, "retweeted_status", None):
            logger.debug("ignore retweet from: %s", user.name)
            return
        if user.id_str == TWITTERS["KanColle_STAFF"]:
            self.proceed_kancolle(status)
        elif user.id_str == TWITTERS["maesanpicture"]:
            self.proceed_samidare(status)
        elif user.id_str == TWITTERS["komatan"]:
            self.proceed_komatan(status)
        elif user.id_str == TWITTERS["Strangestone"]:
            self.proceed_tawawa(status)
        else:
            logger.debug("ignore user: %s", user.name)

    def proceed_kancolle(self, status):
        _tags = status.entities.get("hashtags", [])
        tags = [t["text"] for t in _tags]
        if "艦これ" not in tags:
            return
        logger.info("kancolle: %s", status.text.replace("\n", " "))
        user = status.user
        update_profile_img(user.profile_image_url_https)
        msg = [user.name]
        timestamp = int(int(status.timestamp_ms) / 1000)
        msg.append(datetime.fromtimestamp(timestamp).ctime())
        msg.append(status.text)
        qqbot.sendGroupMsg('\n'.join(msg))

    def proceed_samidare(self, status):
        _tags = status.entities.get("hashtags", [])
        tags = [t["text"] for t in _tags]
        if "毎日五月雨" not in tags:
            return
        logger.info("maesan: %s", status.text.replace("\n", " "))
        medias = status.entities.get("media", [])
        for media in medias:
            logger.info("samidare: %s", media["media_url_https"])
            qqbot.sendGroupMsg(media["media_url_https"])

    def proceed_komatan(self, status):
        medias = status.entities.get("media", [])
        if not medias:
            return
        logger.info("komatan: %s", status.text.replace("\n", " "))
        for media in medias:
            logger.info("komatan: %s", media["media_url_https"])
            qqbot.sendSelfMsg(media["media_url_https"])

    def proceed_tawawa(self, status):
        if not status.text.startswith("月曜日のたわわ"):
            return
        logger.info("tawawa: %s", status.text.replace("\n", " "))
        medias = status.entities.get("media", [])
        for media in medias:
            logger.info("tawawa: %s", media["media_url_https"])
            qqbot.sendGroupMsg(media["media_url_https"])

    def on_error(self, code):
        logger.error("error: %s", code)

def run():
    msl = Stream(auth=ttapi.auth, listener=MyStreamListener())
    msl.filter(follow=TWITTERS.values())

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import

import glob
import os
import copy

from distutils.core import Extension
from zeeko._build_helpers import get_utils_extension_args, get_zmq_extension_args, _generate_cython_extensions

HERE = os.path.dirname(__file__)

pjoin = os.path.join
pxd = lambda *path : os.path.relpath(pjoin(HERE, *path) + ".pxd")
h = lambda *path : os.path.relpath(pjoin(HERE, *path) + ".h")

dependencies = {
    'condition' : [pxd("condition"), pxd("pthread"), pxd("clock"), h("mclock")]
}

def get_extensions(**kwargs):
    """Get the Cython extensions"""

    extension_args = setup_helpers.DistutilsExtensionArgs()
    extension_args.update(get_utils_extension_args())
    extension_args.update(get_zmq_extension_args())
    extension_args['include_dirs'].append('numpy')

    package_name = __name__.split(".")[:-1]
    extensions = [e for e in _generate_cython_extensions(extension_args, os.path.dirname(__file__), package_name)]
    for extension in extensions:
        name = extension.name.split(".")[-1]
        if name in dependencies:
            extension.depends.extend(dependencies[name])
    return extensions
    <commit_msg>Fix a bug in setup_package<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import

import glob
import os
import copy

from zeeko._build_helpers import get_utils_extension_args, get_zmq_extension_args, _generate_cython_extensions
from astropy_helpers import setup_helpers

HERE = os.path.dirname(__file__)

pjoin = os.path.join
pxd = lambda *path : os.path.relpath(pjoin(HERE, *path) + ".pxd")
h = lambda *path : os.path.relpath(pjoin(HERE, *path) + ".h")

dependencies = {
    'condition' : [pxd("condition"), pxd("pthread"), pxd("clock"), h("mclock")]
}

def get_extensions(**kwargs):
    """Get the Cython extensions"""

    extension_args = setup_helpers.DistutilsExtensionArgs()
    extension_args.update(get_utils_extension_args())
    extension_args.update(get_zmq_extension_args())
    extension_args['include_dirs'].append('numpy')

    package_name = __name__.split(".")[:-1]
    extensions = [e for e in _generate_cython_extensions(extension_args, os.path.dirname(__file__), package_name)]
    for extension in extensions:
        name = extension.name.split(".")[-1]
        if name in dependencies:
            extension.depends.extend(dependencies[name])
    return extensions
    <|endoftext|>


<commit_before>#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from numbers import Number
from collections import defaultdict
from math import sqrt

from zipline import ndict
from zipline.transforms.utils import EventWindow, TransformMeta

class MovingStandardDev(object):
    """
    Class that maintains a dictionary from sids to
    MovingStandardDevWindows.  For each sid, we maintain standard
    deviations over any number of distinct fields. (For example, we can
    maintain a sid's moving standard deviation of returns as well as its
    moving standard deviation of prices.
    """
    __metaclass__ = TransformMeta

    def __init__(self, fields,
                 market_aware=True, window_length=None, delta=None):

        self.market_aware = market_aware

        self.delta = delta
        self.window_length = window_length

        # Market-aware mode only works with full-day windows.
        if self.market_aware:
            # Window length must be 1 or greater
            assert self.window_length >= 1

            assert self.window_length and not self.delta,\
                "Market-aware mode only works with full-day windows."

        # Non-market-aware mode requires a timedelta.
        else:
            assert self.delta and not self.window_length, \
                "Non-market-aware mode requires a timedelta."

        # No way to pass arguments to the defaultdict factory, so we
        # need to define a method to generate the correct EventWindows.
        self.sid_windows = defaultdict(self.create_window)

    def create_window(self):
        """
        Factory method for self.sid_windows.
        """
        return MovingStandardDevWindow(
            self.fields,
            self.market_aware,
            self.window_length,
            self.delta
        )

    def update(self, event):
        """
        Update the event window for this event's sid.  Return an ndict
        from tracked fields to moving standard deviations.
        """
        # This will create a new EventWindow if this is the first
        # message for this sid.
        window = self.sid_windows[event.sid]
        window.update(event)
        return window.get_stddevs()

class MovingStandardDevWindow(EventWindow):
    """
    Iteratively calculates moving standard deviations for a particular sid
    over a given time window. We can maintain standard deviations for
    arbitrarily many fields on a single sid. (For example, we might track
    moving standard deviation of returns as well as its moving standard
    deviation of prices.) The expected functionality of this class is to be
    instantiated inside a MovingStandardDev.
    """

    def __init__(self, fields, market_aware, window_length, delta):
        # Call the superclass constructor to set up base EventWindow
        # infrastructure.
        EventWindow.__init__(self, market_aware, window_length, delta)

        self.fields = fields
        self.sum = defaultdict(float)
        self.sum_sqr = defaultdict(float)

    def handle_add(self, event):
        # Sanity check on the event.
        self.assert_required_fields(event)

        # Increment our running totals with data from the event.
        for field in self.fields:
            self.sum[field] += event[field]
            self.sum_sqr[field] += event[field] ** 2

    def handle_remove(self, event):
        # Sanity check on the event.
        self.assert_required_fields(event)

        # Decrement our running totals with data from the event.
        for field in self.fields:
            self.sum[field] -= event[field]
            self.sum_sqr[field] -= event[field] ** 2

    def stdev(self, field):
        """
        Calculate the standard deviation of our ticks over a single field
        using a naive algorithm (see http://goo.gl/wPFtf).
        """
        # Sanity check.
        assert field in self.fields
        # Standard deviation is undefined for no event and 0 for one event
        if len(self.ticks) <= 1:
            return None

        # Calculate and return the standard deviation.
        else:
            _mean = self.sum[field] / len(self.ticks)
            _var = (self.sum_sqr[field] -
                    self.sum[field] * _mean) / (len(self.ticks) - 1)
            return sqrt(_var)

    def get_stddevs(self):
        """
        Return an ndict of all our tracked standard deviations.
        """
        out = ndict()
        for field in self.fields:
            out[field] = self.stdev(field)
        return out

    def assert_required_fields(self, event):
        """
        We only allow events with all of our tracked fields.
        """
        for field in self.fields:
            assert field in event, \
                "Event missing [%s] in MovingStandardDevEventWindow" % field
            assert isinstance(event[field], Number), \
                "Got %s for %s in MovingStandardDevEventWindow" \
                % (event[field], field)
<commit_msg>Fix for stddev during merging.<commit_after>#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from numbers import Number
from collections import defaultdict
from math import sqrt

from zipline import ndict
from zipline.transforms.utils import EventWindow, TransformMeta

class MovingStandardDev(object):
    """
    Class that maintains a dictionary from sids to
    MovingStandardDevWindows.  For each sid, we maintain standard
    deviations over any number of distinct fields. (For example, we can
    maintain a sid's moving standard deviation of returns as well as its
    moving standard deviation of prices.
    """
    __metaclass__ = TransformMeta

    def __init__(self, fields,
                 market_aware=True, window_length=None, delta=None):

        self.market_aware = market_aware
        self.fields = fields

        self.delta = delta
        self.window_length = window_length

        # Market-aware mode only works with full-day windows.
        if self.market_aware:
            # Window length must be 1 or greater
            assert self.window_length >= 1

            assert self.window_length and not self.delta,\
                "Market-aware mode only works with full-day windows."

        # Non-market-aware mode requires a timedelta.
        else:
            assert self.delta and not self.window_length, \
                "Non-market-aware mode requires a timedelta."

        # No way to pass arguments to the defaultdict factory, so we
        # need to define a method to generate the correct EventWindows.
        self.sid_windows = defaultdict(self.create_window)

    def create_window(self):
        """
        Factory method for self.sid_windows.
        """
        return MovingStandardDevWindow(
            self.fields,
            self.market_aware,
            self.window_length,
            self.delta
        )

    def update(self, event):
        """
        Update the event window for this event's sid.  Return an ndict
        from tracked fields to moving standard deviations.
        """
        # This will create a new EventWindow if this is the first
        # message for this sid.
        window = self.sid_windows[event.sid]
        window.update(event)
        return window.get_stddevs()

class MovingStandardDevWindow(EventWindow):
    """
    Iteratively calculates moving standard deviations for a particular sid
    over a given time window. We can maintain standard deviations for
    arbitrarily many fields on a single sid. (For example, we might track
    moving standard deviation of returns as well as its moving standard
    deviation of prices.) The expected functionality of this class is to be
    instantiated inside a MovingStandardDev.
    """

    def __init__(self, fields, market_aware, window_length, delta):
        # Call the superclass constructor to set up base EventWindow
        # infrastructure.
        EventWindow.__init__(self, market_aware, window_length, delta)

        self.fields = fields
        self.sum = defaultdict(float)
        self.sum_sqr = defaultdict(float)

    def handle_add(self, event):
        # Sanity check on the event.
        self.assert_required_fields(event)

        # Increment our running totals with data from the event.
        for field in self.fields:
            self.sum[field] += event[field]
            self.sum_sqr[field] += event[field] ** 2

    def handle_remove(self, event):
        # Sanity check on the event.
        self.assert_required_fields(event)

        # Decrement our running totals with data from the event.
        for field in self.fields:
            self.sum[field] -= event[field]
            self.sum_sqr[field] -= event[field] ** 2

    def stdev(self, field):
        """
        Calculate the standard deviation of our ticks over a single field
        using a naive algorithm (see http://goo.gl/wPFtf).
        """
        # Sanity check.
        assert field in self.fields
        # Standard deviation is undefined for no event and 0 for one event
        if len(self.ticks) <= 1:
            return None

        # Calculate and return the standard deviation.
        else:
            _mean = self.sum[field] / len(self.ticks)
            _var = (self.sum_sqr[field] -
                    self.sum[field] * _mean) / (len(self.ticks) - 1)
            return sqrt(_var)

    def get_stddevs(self):
        """
        Return an ndict of all our tracked standard deviations.
        """
        out = ndict()
        for field in self.fields:
            out[field] = self.stdev(field)
        return out

    def assert_required_fields(self, event):
        """
        We only allow events with all of our tracked fields.
        """
        for field in self.fields:
            assert field in event, \
                "Event missing [%s] in MovingStandardDevEventWindow" % field
            assert isinstance(event[field], Number), \
                "Got %s for %s in MovingStandardDevEventWindow" \
                % (event[field], field)
<|endoftext|>