<commit_before># -*- coding: utf-8 -*-
# Amara, universalsubtitles.org
#
# Copyright (C) 2013 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import os
import json

from django.core.urlresolvers import reverse

from apps.auth.models import CustomUser as User
from apps.subtitles import pipeline
from apps.subtitles.models import ORIGIN_UPLOAD
from apps.videos import metadata_manager
from apps.videos.models import Video, SubtitleLanguage, Subtitle
from apps.videos.tasks import video_changed_tasks
from apps.videos.tests.data import (
    get_video, get_user, make_subtitle_language
)
from apps.videos.tests.videotestutils import (
    WebUseTest, refresh_obj, _create_trans
)
from apps.widget.rpc import Rpc
from apps.widget.tests import (
    create_two_sub_dependent_session, create_two_sub_session, RequestMockup,
    NotAuthenticatedUser
)

up = os.path.dirname

def refresh(m):
    return m.__class__._default_manager.get(pk=m.pk)

class UploadRequiresLoginTest(WebUseTest):
    def test_upload_requires_login(self):
        # When not logged in trying to upload should redirect to the login page.
        self._simple_test('videos:upload_subtitles', status=302)

class UploadSubtitlesTest(WebUseTest):
    def _srt(self, filename):
        return os.path.join(up(up(__file__)), 'fixtures/%s' % filename)

    def _data(self, video, language_code, primary_audio_language_code,
              from_language_code, complete, draft):
        return {
            'video': video.pk,
            'language_code': language_code,
            'primary_audio_language_code': primary_audio_language_code,
            'from_language_code': from_language_code or '',
            'complete': '1' if complete else '0',
            'draft': draft,
        }

    def _upload(self, video, language_code, primary_audio_language_code,
                from_language_code, complete, filename):
        with open(self._srt(filename)) as draft:
            return self.client.post(
                reverse('videos:upload_subtitles'),
                self._data(video, language_code, primary_audio_language_code,
                           from_language_code, complete, draft))

    def setUp(self):
        self._login()

    def _assertVersionCount(self, video, n):
        self.assertEqual(video.newsubtitleversion_set.full().count(), n)

    def _assertCounts(self, video, counts):
        self.assertEqual(
            counts, dict([(sl.language_code, sl.subtitleversion_set.full().count())
                            for sl in video.newsubtitlelanguage_set.all()]))

    # Basic uploading tests.
    def test_upload_subtitles_primary_language(self):
        # Start with a fresh video.
        video = get_video()
        self.assertEqual(video.primary_audio_language_code, '')
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.complete_date)

        # Upload subtitles in the primary audio language.
        response = self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)

        # The writelock should be gone once the subtitles have been uploaded.
        self.assertFalse(video.is_writelocked)

        # The video should now have a primary audio language, since it was set
        # as part of the upload process.
        self.assertEqual(video.primary_audio_language_code, 'en')
        self.assertTrue(video.has_original_language())

        # Ensure that the subtitles actually got uploaded too.
        sl_en = video.subtitle_language()
        self.assertIsNotNone(sl_en)
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        en1 = sl_en.get_tip()
        subtitles = en1.get_subtitles()
        self.assertEqual(en1.subtitle_count, 32)
        self.assertEqual(len(subtitles), 32)

        # Now that we've uploaded a complete set of subtitles, the video and
        # language should be marked as completed.
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_en.subtitles_complete)

        # Let's make sure they didn't get mistakenly marked as a translation.
        self.assertIsNone(sl_en.get_translation_source_language())

        # Upload another version just to be sure.
        response = self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)
        sl_en = refresh(sl_en)

        self.assertEqual(sl_en.subtitleversion_set.full().count(), 2)

    def test_upload_subtitles_non_primary_language(self):
        # Start with a fresh video.
        video = get_video()
        self.assertEqual(video.primary_audio_language_code, '')
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.complete_date)

        # Upload subtitles in a language other than the primary.
        response = self._upload(video, 'fr', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)

        # The writelock should be gone once the subtitles have been uploaded.
        self.assertFalse(video.is_writelocked)

        # The video should now have a primary audio language, since it was set
        # as part of the upload process.
        self.assertEqual(video.primary_audio_language_code, 'en')

        # But it doesn't have a SubtitleLanguage for it.
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.subtitle_language())
        self.assertIsNone(video.get_primary_audio_subtitle_language())

        # Ensure that the subtitles actually got uploaded too.
        sl_fr = video.subtitle_language('fr')
        self.assertIsNotNone(sl_fr)
        self.assertEqual(sl_fr.subtitleversion_set.full().count(), 1)

        fr1 = sl_fr.get_tip()
        subtitles = fr1.get_subtitles()
        self.assertEqual(fr1.subtitle_count, 32)
        self.assertEqual(len(subtitles), 32)

        # Now that we've uploaded a complete set of subtitles, the video and
        # language should be marked as completed.
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_fr.subtitles_complete)

        # Let's make sure they didn't get mistakenly marked as a translation.
        # They're not in the primary language but are still a transcription.
        self.assertIsNone(sl_fr.get_translation_source_language())

        # Upload another version just to be sure.
        response = self._upload(video, 'fr', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)
        sl_fr = refresh(sl_fr)

        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.subtitle_language())
        self.assertIsNone(video.get_primary_audio_subtitle_language())
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_fr.subtitles_complete)
        self.assertEqual(sl_fr.subtitleversion_set.full().count(), 2)

    def test_upload_respects_lock(self):
        user1 = get_user(1)
        user2 = get_user(2)
        video = get_video()
        sl_en = make_subtitle_language(video, 'en')
        request1 = RequestMockup(user1)

        # Lock the language for user 1.
        sl_en.writelock(user1, request1.browser_id)
        self.assertTrue(sl_en.is_writelocked)

        # Now try to upload subtitles as user 2.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')

        # The subtitles should not have been created.
        #
        # We can't really tests the response here because the upload_subtitles
        # view is terrible and always returns a 200 with a horrible mix of HTML
        # and JSON as a response.  So instead we'll just make sure that the
        # subtitles were not actually created.
        self.assertFalse(sl_en.subtitleversion_set.full().exists())

        # User 1 has the writelock, so uploading subtitles should work for them.
        self._login(user1)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        # User 1 still has the writelock, so user 2 still can't upload.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        # Release the writelock.
        sl_en.release_writelock()

        # Now user 2 can finally upload their subtitles.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 2)

    def test_upload_unsynced_subs(self):
        video = get_video()
        self._upload(video, 'en', 'en', None, True, 'subs-with-unsynced.srt')
        self._assertCounts(video, {'en': 1})

        subs = video.subtitle_language('en').get_tip().get_subtitles()

        fully_synced = len([1 for start, end, _, _ in subs if start and end])
        synced_begin = len([1 for start, _, _, _ in subs if start])
        synced_end = len([1 for _, end, _, _ in subs if end])
        fully_unsynced = len([1 for start, end, _, _ in subs
                              if (not start) and (not end)])

        self.assertEqual(fully_synced, 56)
        self.assertEqual(synced_begin, 57)
        self.assertEqual(synced_end, 56)
        self.assertEqual(fully_unsynced, 5)

    # Translation-related tests.
    def test_upload_translation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        # Try uploading a translation of language that doesn't exist.
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')

        # This should fail.  Translations need to be based on an existing
        # langauge.
        self._assertVersionCount(video, 0)

        # Let's upload the first language.
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 1)

        # And now uploading a translation should work.
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

        # Translations of translations are okay too I guess.
        self._upload(video, 'ru', 'en', 'fr', True, 'test.srt')
        self._assertVersionCount(video, 3)

    def test_upload_translation_over_nontranslation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'de', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 2)

        # You can't upload a translation over an existing non-translation
        # language.
        self._upload(video, 'de', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

    def test_upload_translation_over_other_translation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'de', 'en', None, True, 'test.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 3)

        # Now fr is  translated from de, this should fail
        self._upload(video, 'fr', 'en', 'de', True, 'test.srt')
        self._assertVersionCount(video, 3)

    def test_upload_to_language_with_dependents(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        # You cannot upload subtitles to a language that already has dependents.
        # So if you create English and then create a French translation of it
        # English is now off limits to uploads.
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 2)

    def test_upload_translation_with_fewer_subs(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 1)

        # Uploading a translation with a different number of subs than the
        # original language is not allowed.
        self._upload(video, 'is', 'en', 'en', True, 'test_fewer_subs.srt')
        self.assertEqual(video.newsubtitlelanguage_set.count(), 2)

        self._upload(video, 'is', 'en', 'en', True, 'test.srt')
        self.assertEqual(video.newsubtitlelanguage_set.count(), 2)

    def test_upload_and_rollbacks(self):
        video = get_video()

        self._assertCounts(video, {})

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 2, 'fr': 1})

        # We now have:
        #
        # en fr
        #    1
        #   /
        #  /
        # 2
        # |
        # 1

        # Let's sanity check that we can't upload to English now that it has
        # a dependent language (French).
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 2, 'fr': 1})

        # Now let's roll English back to v1.
        pipeline.rollback_to(video, 'en', 1)
        self._assertCounts(video, {'en': 3, 'fr': 1})

        # The translation should now be forked.
        self.assertTrue(video.subtitle_language('fr').is_forked)

        # Now that the translation is forked, we should be able to upload to
        # English again.  Whee!
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 4, 'fr': 1})

    def test_upload_origin_on_version(self):
        # Start with a fresh video.
        video = get_video()

        self._upload(video, 'fr', 'en', None, True, 'test.srt')
        sv = video.newsubtitlelanguage_set.get(language_code='fr').get_tip()
        self.assertEqual(sv.origin, ORIGIN_UPLOAD)
<commit_msg>2374: fix tests to allow uploads to parent languages<commit_after># -*- coding: utf-8 -*-
# Amara, universalsubtitles.org
#
# Copyright (C) 2013 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

import os
import json

from django.core.urlresolvers import reverse

from apps.auth.models import CustomUser as User
from apps.subtitles import pipeline
from apps.subtitles.models import ORIGIN_UPLOAD
from apps.videos import metadata_manager
from apps.videos.models import Video, SubtitleLanguage, Subtitle
from apps.videos.tasks import video_changed_tasks
from apps.videos.tests.data import (
    get_video, get_user, make_subtitle_language
)
from apps.videos.tests.videotestutils import (
    WebUseTest, refresh_obj, _create_trans
)
from apps.widget.rpc import Rpc
from apps.widget.tests import (
    create_two_sub_dependent_session, create_two_sub_session, RequestMockup,
    NotAuthenticatedUser
)

up = os.path.dirname

def refresh(m):
    return m.__class__._default_manager.get(pk=m.pk)

class UploadRequiresLoginTest(WebUseTest):
    def test_upload_requires_login(self):
        # When not logged in trying to upload should redirect to the login page.
        self._simple_test('videos:upload_subtitles', status=302)

class UploadSubtitlesTest(WebUseTest):
    def _srt(self, filename):
        return os.path.join(up(up(__file__)), 'fixtures/%s' % filename)

    def _data(self, video, language_code, primary_audio_language_code,
              from_language_code, complete, draft):
        return {
            'video': video.pk,
            'language_code': language_code,
            'primary_audio_language_code': primary_audio_language_code,
            'from_language_code': from_language_code or '',
            'complete': '1' if complete else '0',
            'draft': draft,
        }

    def _upload(self, video, language_code, primary_audio_language_code,
                from_language_code, complete, filename):
        with open(self._srt(filename)) as draft:
            return self.client.post(
                reverse('videos:upload_subtitles'),
                self._data(video, language_code, primary_audio_language_code,
                           from_language_code, complete, draft))

    def setUp(self):
        self._login()

    def _assertVersionCount(self, video, n):
        self.assertEqual(video.newsubtitleversion_set.full().count(), n)

    def _assertCounts(self, video, counts):
        self.assertEqual(
            counts, dict([(sl.language_code, sl.subtitleversion_set.full().count())
                            for sl in video.newsubtitlelanguage_set.all()]))

    # Basic uploading tests.
    def test_upload_subtitles_primary_language(self):
        # Start with a fresh video.
        video = get_video()
        self.assertEqual(video.primary_audio_language_code, '')
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.complete_date)

        # Upload subtitles in the primary audio language.
        response = self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)

        # The writelock should be gone once the subtitles have been uploaded.
        self.assertFalse(video.is_writelocked)

        # The video should now have a primary audio language, since it was set
        # as part of the upload process.
        self.assertEqual(video.primary_audio_language_code, 'en')
        self.assertTrue(video.has_original_language())

        # Ensure that the subtitles actually got uploaded too.
        sl_en = video.subtitle_language()
        self.assertIsNotNone(sl_en)
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        en1 = sl_en.get_tip()
        subtitles = en1.get_subtitles()
        self.assertEqual(en1.subtitle_count, 32)
        self.assertEqual(len(subtitles), 32)

        # Now that we've uploaded a complete set of subtitles, the video and
        # language should be marked as completed.
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_en.subtitles_complete)

        # Let's make sure they didn't get mistakenly marked as a translation.
        self.assertIsNone(sl_en.get_translation_source_language())

        # Upload another version just to be sure.
        response = self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)
        sl_en = refresh(sl_en)

        self.assertEqual(sl_en.subtitleversion_set.full().count(), 2)

    def test_upload_subtitles_non_primary_language(self):
        # Start with a fresh video.
        video = get_video()
        self.assertEqual(video.primary_audio_language_code, '')
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.complete_date)

        # Upload subtitles in a language other than the primary.
        response = self._upload(video, 'fr', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)

        # The writelock should be gone once the subtitles have been uploaded.
        self.assertFalse(video.is_writelocked)

        # The video should now have a primary audio language, since it was set
        # as part of the upload process.
        self.assertEqual(video.primary_audio_language_code, 'en')

        # But it doesn't have a SubtitleLanguage for it.
        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.subtitle_language())
        self.assertIsNone(video.get_primary_audio_subtitle_language())

        # Ensure that the subtitles actually got uploaded too.
        sl_fr = video.subtitle_language('fr')
        self.assertIsNotNone(sl_fr)
        self.assertEqual(sl_fr.subtitleversion_set.full().count(), 1)

        fr1 = sl_fr.get_tip()
        subtitles = fr1.get_subtitles()
        self.assertEqual(fr1.subtitle_count, 32)
        self.assertEqual(len(subtitles), 32)

        # Now that we've uploaded a complete set of subtitles, the video and
        # language should be marked as completed.
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_fr.subtitles_complete)

        # Let's make sure they didn't get mistakenly marked as a translation.
        # They're not in the primary language but are still a transcription.
        self.assertIsNone(sl_fr.get_translation_source_language())

        # Upload another version just to be sure.
        response = self._upload(video, 'fr', 'en', None, True, 'test.srt')
        self.assertEqual(response.status_code, 200)

        video = refresh(video)
        sl_fr = refresh(sl_fr)

        self.assertFalse(video.has_original_language())
        self.assertIsNone(video.subtitle_language())
        self.assertIsNone(video.get_primary_audio_subtitle_language())
        self.assertIsNotNone(video.complete_date)
        self.assertTrue(sl_fr.subtitles_complete)
        self.assertEqual(sl_fr.subtitleversion_set.full().count(), 2)

    def test_upload_respects_lock(self):
        user1 = get_user(1)
        user2 = get_user(2)
        video = get_video()
        sl_en = make_subtitle_language(video, 'en')
        request1 = RequestMockup(user1)

        # Lock the language for user 1.
        sl_en.writelock(user1, request1.browser_id)
        self.assertTrue(sl_en.is_writelocked)

        # Now try to upload subtitles as user 2.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')

        # The subtitles should not have been created.
        #
        # We can't really tests the response here because the upload_subtitles
        # view is terrible and always returns a 200 with a horrible mix of HTML
        # and JSON as a response.  So instead we'll just make sure that the
        # subtitles were not actually created.
        self.assertFalse(sl_en.subtitleversion_set.full().exists())

        # User 1 has the writelock, so uploading subtitles should work for them.
        self._login(user1)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        # User 1 still has the writelock, so user 2 still can't upload.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 1)

        # Release the writelock.
        sl_en.release_writelock()

        # Now user 2 can finally upload their subtitles.
        self._login(user2)
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self.assertEqual(sl_en.subtitleversion_set.full().count(), 2)

    def test_upload_unsynced_subs(self):
        video = get_video()
        self._upload(video, 'en', 'en', None, True, 'subs-with-unsynced.srt')
        self._assertCounts(video, {'en': 1})

        subs = video.subtitle_language('en').get_tip().get_subtitles()

        fully_synced = len([1 for start, end, _, _ in subs if start and end])
        synced_begin = len([1 for start, _, _, _ in subs if start])
        synced_end = len([1 for _, end, _, _ in subs if end])
        fully_unsynced = len([1 for start, end, _, _ in subs
                              if (not start) and (not end)])

        self.assertEqual(fully_synced, 56)
        self.assertEqual(synced_begin, 57)
        self.assertEqual(synced_end, 56)
        self.assertEqual(fully_unsynced, 5)

    # Translation-related tests.
    def test_upload_translation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        # Try uploading a translation of language that doesn't exist.
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')

        # This should fail.  Translations need to be based on an existing
        # langauge.
        self._assertVersionCount(video, 0)

        # Let's upload the first language.
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 1)

        # And now uploading a translation should work.
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

        # Translations of translations are okay too I guess.
        self._upload(video, 'ru', 'en', 'fr', True, 'test.srt')
        self._assertVersionCount(video, 3)

    def test_upload_translation_over_nontranslation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'de', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 2)

        # You can't upload a translation over an existing non-translation
        # language.
        self._upload(video, 'de', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

    def test_upload_translation_over_other_translation(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'de', 'en', None, True, 'test.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 3)

        # Now fr is  translated from de, this should fail
        self._upload(video, 'fr', 'en', 'de', True, 'test.srt')
        self._assertVersionCount(video, 3)

    def test_upload_to_language_with_dependents(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        # You CAN upload subtitles to a language that already has dependents (it
        # will just fork the dependents).
        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test.srt')
        self._assertVersionCount(video, 2)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 3)

    def test_upload_translation_with_fewer_subs(self):
        video = get_video()
        self._assertVersionCount(video, 0)

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._assertVersionCount(video, 1)

        # Uploading a translation with a different number of subs than the
        # original language is not allowed.
        self._upload(video, 'is', 'en', 'en', True, 'test_fewer_subs.srt')
        self.assertEqual(video.newsubtitlelanguage_set.count(), 2)

        self._upload(video, 'is', 'en', 'en', True, 'test.srt')
        self.assertEqual(video.newsubtitlelanguage_set.count(), 2)

    def test_upload_and_rollbacks(self):
        video = get_video()

        self._assertCounts(video, {})

        self._upload(video, 'en', 'en', None, True, 'test.srt')
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._upload(video, 'fr', 'en', 'en', True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 2, 'fr': 1})

        # We now have:
        #
        # en fr
        #    1
        #   /
        #  /
        # 2
        # |
        # 1

        # Let's sanity check that we can still upload to English now that it has
        # a dependent language (French).
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 3, 'fr': 1})

        # The translation should now be forked.
        self.assertTrue(video.subtitle_language('fr').is_forked)

        # Now let's roll English back to v1.
        pipeline.rollback_to(video, 'en', 1)
        self._assertCounts(video, {'en': 4, 'fr': 1})

        # And try uploading something on top of the rollback.
        self._upload(video, 'en', 'en', None, True, 'test_fewer_subs.srt')
        self._assertCounts(video, {'en': 5, 'fr': 1})

    def test_upload_origin_on_version(self):
        # Start with a fresh video.
        video = get_video()

        self._upload(video, 'fr', 'en', None, True, 'test.srt')
        sv = video.newsubtitlelanguage_set.get(language_code='fr').get_tip()
        self.assertEqual(sv.origin, ORIGIN_UPLOAD)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2011-2012  Romain Bignon, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import datetime
import re

from weboob.tools.capabilities.thumbnail import Thumbnail
from weboob.tools.browser import BasePage, BrokenPageError

from .video import TricTracTVVideo

__all__ = ['IndexPage', 'VideoPage']

class IndexPage(BasePage):
    def iter_videos(self):
        for div in self.parser.select(self.document.getroot(), 'li#contentsearch'):
            title = self.parser.select(div, '#titlesearch span', 1)

            a = self.parser.select(div, 'a', 1)
            url = a.attrib['href']
            m = re.match('/video-(.*)', url)
            if not m:
                self.logger.debug('url %s does not match' % url)
                continue
            _id = m.group(1)
            video = TricTracTVVideo(_id)
            video.title = unicode(title.text)

            url = self.parser.select(div, 'img', 1).attrib['src']
            stars = self.parser.select(div, '.etoile_on')
            video.rating = len(stars)
            video.rating_max = 5

            video.thumbnail = Thumbnail ( unicode ( 'http://www.trictrac.tv/%s' % url ) )

            yield video

class VideoPage(BasePage):
    def on_loaded(self):
        p = self.parser.select(self.document.getroot(), 'p.alert')
        if len(p) > 0:
            raise Exception(p[0].text)

    def get_info_url(self):
        try:
            div = self.parser.select(self.document.getroot(), '#Content_Video object', 1)
        except BrokenPageError:
            return None
        else:
            for param in self.parser.select(div, 'param', None):
                if param.get('name') == 'flashvars':
                    m = re.match('varplaymedia=([0-9]*)', param.attrib['value'])
                    if m:
                        return r'http://www.trictrac.tv/swf/listelement.php?idfile=%s' % m.group(1)

    def get_title(self):
        try:
            title = self.parser.select(self.document.getroot(), 'title', 1)
        except BrokenPageError:
            return None
        else:
            return title.text

    def get_descriptif(self):
        try:
            descriptif = self.parser.select(self.document.getroot(), '.video_descriptif p', 1)
        except BrokenPageError:
            return None
        else:
            return descriptif.text

    def get_duration(self):
        try:
            details = self.parser.select(self.document.getroot(), 'div#video_detail div')
        except BrokenPageError:
            return None
        else:
            duration = details[2]
            duration_string = duration.text [ duration.text.rfind ( ' ' ) + 1 : ]
            tokens = duration_string.split(':')
            if len(tokens) > 2:
                return datetime.timedelta(hours=int(tokens[0]), minutes=int(tokens[1]), seconds=int(tokens[2]))
            else:
                return datetime.timedelta(minutes=int(tokens[0]), seconds=int(tokens[1]))

    def get_date(self):
        try:
            date = self.parser.select(self.document.getroot(), 'div#video_detail div.date', 1)
        except BrokenPageError:
            return None
        else:
            string = date.text
            string = string [ string.rfind('le ') + 3 : ]
            months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ]
            words = string.split ( ' ' )
            month_no = months.index ( words [ 1 ] ) + 1
            return datetime.datetime.strptime ( ( '%s %s %s %s' %
                                                  ( words [ 0 ], month_no, words [ 2 ], words [ 3 ] ) ),
                                                '%d %m %Y, %H:%M:%S')

    def get_rating(self):
        try:
            stars = self.parser.select(self.document.getroot(), '#video_info .etoile_on')
        except BrokenPageError:
            return None
        else:
            return len(stars)

    def get_id(self):
        return self.groups[0]
<commit_msg>modify trictractv module to use the new BaseVideo/ICapVideo classes<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2011-2012  Romain Bignon, Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import datetime
import re

from weboob.capabilities.image import BaseImage
from weboob.tools.browser import BasePage, BrokenPageError

from .video import TricTracTVVideo

__all__ = ['IndexPage', 'VideoPage']

class IndexPage(BasePage):
    def iter_videos(self):
        for div in self.parser.select(self.document.getroot(), 'li#contentsearch'):
            title = self.parser.select(div, '#titlesearch span', 1)

            a = self.parser.select(div, 'a', 1)
            url = a.attrib['href']
            m = re.match('/video-(.*)', url)
            if not m:
                self.logger.debug('url %s does not match' % url)
                continue
            _id = m.group(1)
            video = TricTracTVVideo(_id)
            video.title = unicode(title.text)

            url = self.parser.select(div, 'img', 1).attrib['src']
            stars = self.parser.select(div, '.etoile_on')
            video.rating = len(stars)
            video.rating_max = 5

            video.thumbnail = BaseImage('http://www.trictrac.tv/%s' % url)
            video.thumbnail.url = video.thumbnail.id

            yield video

class VideoPage(BasePage):
    def on_loaded(self):
        p = self.parser.select(self.document.getroot(), 'p.alert')
        if len(p) > 0:
            raise Exception(p[0].text)

    def get_info_url(self):
        try:
            div = self.parser.select(self.document.getroot(), '#Content_Video object', 1)
        except BrokenPageError:
            return None
        else:
            for param in self.parser.select(div, 'param', None):
                if param.get('name') == 'flashvars':
                    m = re.match('varplaymedia=([0-9]*)', param.attrib['value'])
                    if m:
                        return r'http://www.trictrac.tv/swf/listelement.php?idfile=%s' % m.group(1)

    def get_title(self):
        try:
            title = self.parser.select(self.document.getroot(), 'title', 1)
        except BrokenPageError:
            return None
        else:
            return title.text

    def get_descriptif(self):
        try:
            descriptif = self.parser.select(self.document.getroot(), '.video_descriptif p', 1)
        except BrokenPageError:
            return None
        else:
            return descriptif.text

    def get_duration(self):
        try:
            details = self.parser.select(self.document.getroot(), 'div#video_detail div')
        except BrokenPageError:
            return None
        else:
            duration = details[2]
            duration_string = duration.text [ duration.text.rfind ( ' ' ) + 1 : ]
            tokens = duration_string.split(':')
            if len(tokens) > 2:
                return datetime.timedelta(hours=int(tokens[0]), minutes=int(tokens[1]), seconds=int(tokens[2]))
            else:
                return datetime.timedelta(minutes=int(tokens[0]), seconds=int(tokens[1]))

    def get_date(self):
        try:
            date = self.parser.select(self.document.getroot(), 'div#video_detail div.date', 1)
        except BrokenPageError:
            return None
        else:
            string = date.text
            string = string [ string.rfind('le ') + 3 : ]
            months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ]
            words = string.split ( ' ' )
            month_no = months.index ( words [ 1 ] ) + 1
            return datetime.datetime.strptime ( ( '%s %s %s %s' %
                                                  ( words [ 0 ], month_no, words [ 2 ], words [ 3 ] ) ),
                                                '%d %m %Y, %H:%M:%S')

    def get_rating(self):
        try:
            stars = self.parser.select(self.document.getroot(), '#video_info .etoile_on')
        except BrokenPageError:
            return None
        else:
            return len(stars)

    def get_id(self):
        return self.groups[0]
<|endoftext|>


<commit_before># The current Orchestra version.
__version__ = '0.2.58'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<commit_msg>Version bump.<commit_after># The current Orchestra version.
__version__ = '0.2.59'

default_app_config = 'orchestra.apps.OrchestraAppConfig'
<|endoftext|>


<commit_before># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Dataclass implementation of AQT configuration."""

import dataclasses
from typing import Any, Dict, List, Optional, Type, TypeVar, Union

import dacite

class ConfigError(Exception):
  pass

T = TypeVar('T', bound='_BaseConfig')

@dataclasses.dataclass
class _BaseConfig:
  """Adds serialization and primitive type checking to dataclasses."""

  def to_dict(self) -> Dict[str, Any]:
    """Converts a dataclass to dict while preserving dataclass-specific logic."""
    dataclass_dict = {}
    for field in dataclasses.fields(self):
      attr = getattr(self, field.name)
      if isinstance(attr, _BaseConfig):
        dataclass_dict[field.name] = attr.to_dict()
      elif isinstance(attr, list):
        dataclass_dict[field.name] = [
            a.to_dict() if isinstance(a, _BaseConfig) else a for a in attr
        ]
      else:
        dataclass_dict[field.name] = attr

    return dataclass_dict

  @classmethod
  def from_dict(cls: Type[T], data: Dict[str, Any]) -> T:
    return dacite.from_dict(cls, data)

  def validate(self):
    pass

def _validate_intervals(configs: List['AqtTensorConfig']):
  """Ensures that the provided schedule specifies increasing time segments.

  We require configs to be in sorted order with respect to
  `(start_at_event, end_at_event)` intervals. This makes disjointness checking
  easier.

  Args:
    configs: a list of tensor quantization configs.

  Raises:
    ConfigError: if any of the described conditions are not met.
  """
  previous_end = None
  for i, config in enumerate(configs):
    current_begin = config.begin_at_event
    current_end = config.end_at_event

    neither_none = current_begin is not None and current_end is not None
    if neither_none and current_end < current_begin:
      raise ConfigError(f'config[{i}].end_at_event ({current_end}) < '
                        f'config[{i}].begin_at_event ({current_begin})')

    if i > 0:
      if current_begin is None:
        raise ConfigError('only the first config may omit begin_at_event, but '
                          f'config[{i}] does too')

      if previous_end is None:
        raise ConfigError('only the last config may omit end_at_event, but '
                          f'config[{i-1}] does too')

      if previous_end > current_begin:
        raise ConfigError(f'config[{i-1}].end_at_event ({previous_end}) > '
                          f'config[{i}].begin_at_event ({current_begin})')

    if previous_end is not None and previous_end != current_begin:
      raise ConfigError(
          f'There is a gap between config[{i-1}].end_at_event ({previous_end}) '
          f'and config[{i}].begin_at_event ({current_begin}). There must be '
          f'always one active config.')

    previous_end = current_end

def _validate_alignment(
    lhs_path: str,  #
    lhs_configs: List['AqtTensorConfig'],
    rhs_path: str,
    rhs_configs: List['AqtTensorConfig']):
  """Ensures that quantization schedules for both arguments are aligned.

  For binary quantized operations, a quantization schedule for each argument
  specifies the type of quantization that should be applied at a given time.

  The configuration is assumed to specify equal-length lists of quantization
  configurations for LHS and RHS configs, for which parallel entries must
  have identical endpoints for quantization activation.

  Args:
    lhs_path: what to call the `lhs` in error strings.
    lhs_configs: the tensor quantization schedule for the left argument.
    rhs_path: what to call the `rhs` in error strings.
    rhs_configs: the tensor quantization schedule for the right argument.

  Raises:
    ConfigError: if any of the described conditions are not met.
  """
  if len(lhs_configs) != len(rhs_configs):
    raise ConfigError(f'lhs config len {len(lhs_configs)} must equal '
                      f'rhs config len {len(rhs_configs)}')

  for i, (lhs_config, rhs_config) in enumerate(zip(lhs_configs, rhs_configs)):

    def _interval(config):
      return config.begin_at_event, config.end_at_event

    if _interval(lhs_config) != _interval(rhs_config):
      raise ConfigError(
          '(begin_at_event, end_at_event) intervals do not match: '
          f'{lhs_path}[{i}] ({_interval(lhs_config)}) != '
          f'{rhs_path}[{i}] ({_interval(rhs_config)})')

@dataclasses.dataclass
class IntQuantConfig(_BaseConfig):
  # pyformat: disable
  """Config for integer quantization in aqt_ops.

  Attributes:
    bits:
      Number of bits to quantize to (e.g 4 for int4). Must be positive.
    preserve_zero:
      Whether or not zeros should be representable.
  """
  # pyformat: enable
  bits: int
  preserve_zero: bool = True

  def validate(self):
    if self.bits < 1:
      raise ConfigError(f'expected bits={self.bits} > 0')

  def compatible_with_int8(self) -> bool:
    """Checks if the config is compatible with int8."""
    return self.bits <= 8

@dataclasses.dataclass
class FloatConfig(_BaseConfig):
  """Config for non quantization in aqt_ops.

  Attributes:
    use_bf16: Whether or not to use bfloat16.
  """
  use_bf16: bool = True

@dataclasses.dataclass
class StatsConfig(_BaseConfig):
  # pyformat: disable
  """Config for aqt_ops.Stats.

  Attributes:
    ema_update_count:
      Scale calibration sample size.
      The weight of the newest batch will be 1 / ema_update_count.
      E.g. 1 means that only the current batch is used in calibration.
      Weights of examples in a batch are also correctly taken into account.
    share_stats_axes:
      Bound sharing. Required for all contraction axes. Each axis in this list
      must be in [0, rank) where rank is the rank of the tensor to quantize.
      This list must also be strictly sorted.
    filter_zeros:
      Ignores zeros when computing statistics. Useful for distributions with a
      zero spike like the output of Relu.
    lp_order:
      Defines moment used in lp_dev, needs to be even and positive.

    update_count_prior:
      Emulates a situation where some statistics have already been collected.
      Default of `1.0` prevents division by zero in prod, while setting it to
      `0` allows for precise logic in unit testing.
    mean_prior:
      Emulates a situation where some statistics have already been collected.
    l1_dev_prior:
      Emulates a situation where some statistics have already been collected.
    lp_dev_prior:
      Emulates a situation where some statistics have already been collected.
    max_dev_prior:
      Encode prior knowledge of the maximum absolute value observed; note that
      the `update_count_prior` does not affect the prior weight for the max so
      in practice this may be usually left at zero.

    tpu_cross_replica_sum:
      Setting it to false might improve performance, but note that checkpoint
      might contain wrong value.
  """
  # pyformat: enable
  ema_update_count: int
  share_stats_axes: List[int]
  filter_zeros: bool = True
  lp_order: int = 2

  update_count_prior: float = 1.0
  mean_prior: float = 0.0
  l1_dev_prior: float = 0.0
  lp_dev_prior: float = 0.0
  max_dev_prior: float = 0.0

  tpu_cross_replica_sum: bool = True

  def validate(self, data_shape: List[Optional[int]]):
    """Validates this StatsConfig for the provided data shape.

    Args:
      data_shape: the shape of the input tensor which will be quantized with
        self as the statistics configuration. If an entry is None, this
        indicates a dimension whose size is unknown at graph compilation time.

    Raises:
      ConfigError: if any of the specified share_stats_axes are not between
        [0, rank) where rank is len(data_shape), the share_stats_axes are
        not sorted, don't contain all unknown dimensions (None in data_shape),
        or if ema_update_count is not positive.
    """
    rank = len(data_shape)
    if any(ax < 0 or ax >= rank for ax in self.share_stats_axes):  # pylint: disable=not-an-iterable
      raise ConfigError(f'share_stats_axes ({self.share_stats_axes}) must be '
                        f'between 0 and rank ({rank}), not including rank')
    if not all(ax1 < ax2 for ax1, ax2 in zip(self.share_stats_axes[:-1],
                                             self.share_stats_axes[1:])):
      raise ConfigError(
          f'share_stats_axes ({self.share_stats_axes}) must be strictly sorted')

    unknown_axes = {i for i, dim in enumerate(data_shape) if dim is None}
    shared_axes = set(self.share_stats_axes)
    if not unknown_axes.issubset(shared_axes):
      raise ConfigError(f'expected share_stats_axes ({self.share_stats_axes}) '
                        'to contain unknown axes for given data shape '
                        f'({data_shape})')

    if self.ema_update_count < 1:
      raise ConfigError(
          f'expected ema_update_count={self.ema_update_count} >= 1')

    if self.lp_order < 1:
      raise ConfigError(f'expected lp_order={self.lp_order} >= 1')

@dataclasses.dataclass
class CalibrationConfig(_BaseConfig):
  """Config for aqt_ops.Stats.

  Attributes:
    const_bound_coeff: Bias used in bound calibration.
    l1_dev_coeff: Weight of the l1_dev in bound calibration.
    lp_dev_coeff: Weight of the lp_dev in bound calibration.
    max_dev_coeff: Weight of the max_dev in bound calibration.
  """
  const_bound_coeff: float = 0.0
  l1_dev_coeff: float = 0.0
  lp_dev_coeff: float = 0.0
  max_dev_coeff: float = 0.0

@dataclasses.dataclass
class AqtTensorConfig(_BaseConfig):
  """Config for aqt_ops.AqtTensor.

  Attributes:
    quant_config: Either IntQuantConfig or FloatConfig.
    calibration_config: Calibration config.
    freeze_scale_at_begin: Freezes the calibration when begin_at_event is
      reached. This breaks the feedback look between calibration scale and
      distribution observed by stats.
    begin_at_event: Start of this quantization interval. You need to make sure
      that there are enough updates before quantization is enabled with
      begin_at_event or set a proper prior or use const_bound. This is very
      important if freeze_scale_at_begin == true.
    end_at_event: End of this quantization interval.
  """
  quant_config: Union[IntQuantConfig, FloatConfig]
  calibration_config: CalibrationConfig
  freeze_scale_at_begin: bool
  begin_at_event: Optional[int] = None
  end_at_event: Optional[int] = None

  def validate(self):
    """Validates this tensor config."""
    if (self.begin_at_event is not None and self.end_at_event is not None and
        self.begin_at_event > self.end_at_event):
      raise ConfigError(f'expected begin_at_event={self.begin_at_event} <= '
                        f'end_at_event={self.end_at_event}')

    if not isinstance(self.quant_config, IntQuantConfig) and not isinstance(
        self.quant_config, FloatConfig):
      raise ConfigError(
          'quant_config must be one of {int_quant_config, float_config}.'
      )

  def to_dict(self) -> Dict[str, Any]:
    # AqtTensorConfig dataclass does not have `int_quant_config` and
    # `float_config` which are present in proto. Instead, it comes with
    # `quant_config`, but it is not a valid field in the corresponding proto.
    # So, they are manually added/removed below for proper conversion.
    dataclass_dict = dataclasses.asdict(self)
    cfg_type = 'int_quant_config' if isinstance(
        self.quant_config, IntQuantConfig) else 'float_config'
    dataclass_dict[cfg_type] = dataclass_dict.pop('quant_config')
    return dataclass_dict

@dataclasses.dataclass
class AqtScheduleConfig(_BaseConfig):
  """Config for an `aqt_tensor.TensorQuantizer` configuration schedule.

  Attributes:
    stats_config: Statistics configs describing what statistics to track.
    tensor_configs: Tensor configs to apply to the input tensor. *_at_event must
      describe sorted and disjoint intervals. *_at_event must be identical in
      the filter configs.
    use_quantized_variable:
      If true and in training mode, saves intermediate quantizations to
      user-provided variables. During inference, quantized variables
      are read from but not written to with new input values.
    inference_config_index:
      If not None, then the index into `tensor_configs` which indicates which
      tensor configuration to use during inference. This allows for static
      switching logic at inference time, which improves throughput. If None,
      then the most recently trained-on config is the one that's used.
  """
  stats_config: StatsConfig
  tensor_configs: List[AqtTensorConfig]
  use_quantized_variable: bool = False
  inference_config_index: Optional[int] = None

  def validate(self, data_shape: List[Optional[int]]):
    """Validates this AqtScheduleConfig for the provided data shape."""
    _validate_intervals(self.tensor_configs)
    if any(ax is None for ax in data_shape) and self.use_quantized_variable:
      raise ConfigError(
          'use of quantized variable with unknown dimensions is disallowed')
    if self.inference_config_index is not None and not (
        0 <= self.inference_config_index < len(self.tensor_configs)):
      raise ConfigError(
          f'inference_config_index ({self.inference_config_index}) must be '
          'at least 0 and less than len(tensor_configs) '
          f'({len(self.tensor_configs)})')

  def fill_gaps_with_float_config(self):
    """Fills gaps with FloatConfig to always have one active config."""
    def create_float_config(begin, end):
      tc = AqtTensorConfig(
          quant_config=FloatConfig(),
          calibration_config=CalibrationConfig(),
          freeze_scale_at_begin=True,
          begin_at_event=begin,
          end_at_event=end)
      return tc

    previous_end = None
    filled_configs = []
    inference_config_index = None
    for i, config in enumerate(self.tensor_configs):
      current_begin = config.begin_at_event

      if previous_end != current_begin and current_begin > 0:
        filled_configs.append(create_float_config(previous_end, current_begin))

      filled_configs.append(config)
      previous_end = config.end_at_event

      if self.inference_config_index == i:
        inference_config_index = len(filled_configs) - 1

    if previous_end is not None or not filled_configs:
      filled_configs.append(create_float_config(previous_end, None))

    self.tensor_configs = filled_configs
    # Note that self.inference_config_index is either None or in [0, len)
    # where len is the length of the previous self.tensor_configs. So the
    # update below either keeps it set to None or updates the index to the
    # reindexed value for the new self.tensor_configs.
    self.inference_config_index = inference_config_index

<commit_msg>Add AqtMatmulConfig<commit_after># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Dataclass implementation of AQT configuration."""

import dataclasses
from typing import Any, Dict, List, Optional, Type, TypeVar, Union

import dacite

class ConfigError(Exception):
  pass

T = TypeVar('T', bound='_BaseConfig')

@dataclasses.dataclass
class _BaseConfig:
  """Adds serialization and primitive type checking to dataclasses."""

  def to_dict(self) -> Dict[str, Any]:
    """Converts a dataclass to dict while preserving dataclass-specific logic."""
    dataclass_dict = {}
    for field in dataclasses.fields(self):
      attr = getattr(self, field.name)
      if isinstance(attr, _BaseConfig):
        dataclass_dict[field.name] = attr.to_dict()
      elif isinstance(attr, list):
        dataclass_dict[field.name] = [
            a.to_dict() if isinstance(a, _BaseConfig) else a for a in attr
        ]
      else:
        dataclass_dict[field.name] = attr

    return dataclass_dict

  @classmethod
  def from_dict(cls: Type[T], data: Dict[str, Any]) -> T:
    return dacite.from_dict(cls, data)

  def validate(self):
    pass

def _validate_intervals(configs: List['AqtTensorConfig']):
  """Ensures that the provided schedule specifies increasing time segments.

  We require configs to be in sorted order with respect to
  `(start_at_event, end_at_event)` intervals. This makes disjointness checking
  easier.

  Args:
    configs: a list of tensor quantization configs.

  Raises:
    ConfigError: if any of the described conditions are not met.
  """
  previous_end = None
  for i, config in enumerate(configs):
    current_begin = config.begin_at_event
    current_end = config.end_at_event

    neither_none = current_begin is not None and current_end is not None
    if neither_none and current_end < current_begin:
      raise ConfigError(f'config[{i}].end_at_event ({current_end}) < '
                        f'config[{i}].begin_at_event ({current_begin})')

    if i > 0:
      if current_begin is None:
        raise ConfigError('only the first config may omit begin_at_event, but '
                          f'config[{i}] does too')

      if previous_end is None:
        raise ConfigError('only the last config may omit end_at_event, but '
                          f'config[{i-1}] does too')

      if previous_end > current_begin:
        raise ConfigError(f'config[{i-1}].end_at_event ({previous_end}) > '
                          f'config[{i}].begin_at_event ({current_begin})')

    if previous_end is not None and previous_end != current_begin:
      raise ConfigError(
          f'There is a gap between config[{i-1}].end_at_event ({previous_end}) '
          f'and config[{i}].begin_at_event ({current_begin}). There must be '
          f'always one active config.')

    previous_end = current_end

def _validate_alignment(
    lhs_path: str,  #
    lhs_configs: List['AqtTensorConfig'],
    rhs_path: str,
    rhs_configs: List['AqtTensorConfig']):
  """Ensures that quantization schedules for both arguments are aligned.

  For binary quantized operations, a quantization schedule for each argument
  specifies the type of quantization that should be applied at a given time.

  The configuration is assumed to specify equal-length lists of quantization
  configurations for LHS and RHS configs, for which parallel entries must
  have identical endpoints for quantization activation.

  Args:
    lhs_path: what to call the `lhs` in error strings.
    lhs_configs: the tensor quantization schedule for the left argument.
    rhs_path: what to call the `rhs` in error strings.
    rhs_configs: the tensor quantization schedule for the right argument.

  Raises:
    ConfigError: if any of the described conditions are not met.
  """
  if len(lhs_configs) != len(rhs_configs):
    raise ConfigError(f'lhs config len {len(lhs_configs)} must equal '
                      f'rhs config len {len(rhs_configs)}')

  for i, (lhs_config, rhs_config) in enumerate(zip(lhs_configs, rhs_configs)):

    def _interval(config):
      return config.begin_at_event, config.end_at_event

    if _interval(lhs_config) != _interval(rhs_config):
      raise ConfigError(
          '(begin_at_event, end_at_event) intervals do not match: '
          f'{lhs_path}[{i}] ({_interval(lhs_config)}) != '
          f'{rhs_path}[{i}] ({_interval(rhs_config)})')

@dataclasses.dataclass
class IntQuantConfig(_BaseConfig):
  # pyformat: disable
  """Config for integer quantization in aqt_ops.

  Attributes:
    bits:
      Number of bits to quantize to (e.g 4 for int4). Must be positive.
    preserve_zero:
      Whether or not zeros should be representable.
  """
  # pyformat: enable
  bits: int
  preserve_zero: bool = True

  def validate(self):
    if self.bits < 1:
      raise ConfigError(f'expected bits={self.bits} > 0')

  def compatible_with_int8(self) -> bool:
    """Checks if the config is compatible with int8."""
    return self.bits <= 8

@dataclasses.dataclass
class FloatConfig(_BaseConfig):
  """Config for non quantization in aqt_ops.

  Attributes:
    use_bf16: Whether or not to use bfloat16.
  """
  use_bf16: bool = True

@dataclasses.dataclass
class StatsConfig(_BaseConfig):
  # pyformat: disable
  """Config for aqt_ops.Stats.

  Attributes:
    ema_update_count:
      Scale calibration sample size.
      The weight of the newest batch will be 1 / ema_update_count.
      E.g. 1 means that only the current batch is used in calibration.
      Weights of examples in a batch are also correctly taken into account.
    share_stats_axes:
      Bound sharing. Required for all contraction axes. Each axis in this list
      must be in [0, rank) where rank is the rank of the tensor to quantize.
      This list must also be strictly sorted.
    filter_zeros:
      Ignores zeros when computing statistics. Useful for distributions with a
      zero spike like the output of Relu.
    lp_order:
      Defines moment used in lp_dev, needs to be even and positive.

    update_count_prior:
      Emulates a situation where some statistics have already been collected.
      Default of `1.0` prevents division by zero in prod, while setting it to
      `0` allows for precise logic in unit testing.
    mean_prior:
      Emulates a situation where some statistics have already been collected.
    l1_dev_prior:
      Emulates a situation where some statistics have already been collected.
    lp_dev_prior:
      Emulates a situation where some statistics have already been collected.
    max_dev_prior:
      Encode prior knowledge of the maximum absolute value observed; note that
      the `update_count_prior` does not affect the prior weight for the max so
      in practice this may be usually left at zero.

    tpu_cross_replica_sum:
      Setting it to false might improve performance, but note that checkpoint
      might contain wrong value.
  """
  # pyformat: enable
  ema_update_count: int
  share_stats_axes: List[int]
  filter_zeros: bool = True
  lp_order: int = 2

  update_count_prior: float = 1.0
  mean_prior: float = 0.0
  l1_dev_prior: float = 0.0
  lp_dev_prior: float = 0.0
  max_dev_prior: float = 0.0

  tpu_cross_replica_sum: bool = True

  def validate(self, data_shape: List[Optional[int]]):
    """Validates this StatsConfig for the provided data shape.

    Args:
      data_shape: the shape of the input tensor which will be quantized with
        self as the statistics configuration. If an entry is None, this
        indicates a dimension whose size is unknown at graph compilation time.

    Raises:
      ConfigError: if any of the specified share_stats_axes are not between
        [0, rank) where rank is len(data_shape), the share_stats_axes are
        not sorted, don't contain all unknown dimensions (None in data_shape),
        or if ema_update_count is not positive.
    """
    rank = len(data_shape)
    if any(ax < 0 or ax >= rank for ax in self.share_stats_axes):  # pylint: disable=not-an-iterable
      raise ConfigError(f'share_stats_axes ({self.share_stats_axes}) must be '
                        f'between 0 and rank ({rank}), not including rank')
    if not all(ax1 < ax2 for ax1, ax2 in zip(self.share_stats_axes[:-1],
                                             self.share_stats_axes[1:])):
      raise ConfigError(
          f'share_stats_axes ({self.share_stats_axes}) must be strictly sorted')

    unknown_axes = {i for i, dim in enumerate(data_shape) if dim is None}
    shared_axes = set(self.share_stats_axes)
    if not unknown_axes.issubset(shared_axes):
      raise ConfigError(f'expected share_stats_axes ({self.share_stats_axes}) '
                        'to contain unknown axes for given data shape '
                        f'({data_shape})')

    if self.ema_update_count < 1:
      raise ConfigError(
          f'expected ema_update_count={self.ema_update_count} >= 1')

    if self.lp_order < 1:
      raise ConfigError(f'expected lp_order={self.lp_order} >= 1')

@dataclasses.dataclass
class CalibrationConfig(_BaseConfig):
  """Config for aqt_ops.Stats.

  Attributes:
    const_bound_coeff: Bias used in bound calibration.
    l1_dev_coeff: Weight of the l1_dev in bound calibration.
    lp_dev_coeff: Weight of the lp_dev in bound calibration.
    max_dev_coeff: Weight of the max_dev in bound calibration.
  """
  const_bound_coeff: float = 0.0
  l1_dev_coeff: float = 0.0
  lp_dev_coeff: float = 0.0
  max_dev_coeff: float = 0.0

@dataclasses.dataclass
class AqtTensorConfig(_BaseConfig):
  """Config for aqt_ops.AqtTensor.

  Attributes:
    quant_config: Either IntQuantConfig or FloatConfig.
    calibration_config: Calibration config.
    freeze_scale_at_begin: Freezes the calibration when begin_at_event is
      reached. This breaks the feedback look between calibration scale and
      distribution observed by stats.
    begin_at_event: Start of this quantization interval. You need to make sure
      that there are enough updates before quantization is enabled with
      begin_at_event or set a proper prior or use const_bound. This is very
      important if freeze_scale_at_begin == true.
    end_at_event: End of this quantization interval.
  """
  quant_config: Union[IntQuantConfig, FloatConfig]
  calibration_config: CalibrationConfig
  freeze_scale_at_begin: bool
  begin_at_event: Optional[int] = None
  end_at_event: Optional[int] = None

  def validate(self):
    """Validates this tensor config."""
    if (self.begin_at_event is not None and self.end_at_event is not None and
        self.begin_at_event > self.end_at_event):
      raise ConfigError(f'expected begin_at_event={self.begin_at_event} <= '
                        f'end_at_event={self.end_at_event}')

    if not isinstance(self.quant_config, IntQuantConfig) and not isinstance(
        self.quant_config, FloatConfig):
      raise ConfigError(
          'quant_config must be one of {int_quant_config, float_config}.'
      )

  def to_dict(self) -> Dict[str, Any]:
    # AqtTensorConfig dataclass does not have `int_quant_config` and
    # `float_config` which are present in proto. Instead, it comes with
    # `quant_config`, but it is not a valid field in the corresponding proto.
    # So, they are manually added/removed below for proper conversion.
    dataclass_dict = dataclasses.asdict(self)
    cfg_type = 'int_quant_config' if isinstance(
        self.quant_config, IntQuantConfig) else 'float_config'
    dataclass_dict[cfg_type] = dataclass_dict.pop('quant_config')
    return dataclass_dict

@dataclasses.dataclass
class AqtScheduleConfig(_BaseConfig):
  """Config for an `aqt_tensor.TensorQuantizer` configuration schedule.

  Attributes:
    stats_config: Statistics configs describing what statistics to track.
    tensor_configs: Tensor configs to apply to the input tensor. *_at_event must
      describe sorted and disjoint intervals. *_at_event must be identical in
      the filter configs.
    use_quantized_variable:
      If true and in training mode, saves intermediate quantizations to
      user-provided variables. During inference, quantized variables
      are read from but not written to with new input values.
    inference_config_index:
      If not None, then the index into `tensor_configs` which indicates which
      tensor configuration to use during inference. This allows for static
      switching logic at inference time, which improves throughput. If None,
      then the most recently trained-on config is the one that's used.
  """
  stats_config: StatsConfig
  tensor_configs: List[AqtTensorConfig]
  use_quantized_variable: bool = False
  inference_config_index: Optional[int] = None

  def validate(self, data_shape: List[Optional[int]]):
    """Validates this AqtScheduleConfig for the provided data shape."""
    _validate_intervals(self.tensor_configs)
    if any(ax is None for ax in data_shape) and self.use_quantized_variable:
      raise ConfigError(
          'use of quantized variable with unknown dimensions is disallowed')
    if self.inference_config_index is not None and not (
        0 <= self.inference_config_index < len(self.tensor_configs)):
      raise ConfigError(
          f'inference_config_index ({self.inference_config_index}) must be '
          'at least 0 and less than len(tensor_configs) '
          f'({len(self.tensor_configs)})')

  def fill_gaps_with_float_config(self):
    """Fills gaps with FloatConfig to always have one active config."""
    def create_float_config(begin, end):
      tc = AqtTensorConfig(
          quant_config=FloatConfig(),
          calibration_config=CalibrationConfig(),
          freeze_scale_at_begin=True,
          begin_at_event=begin,
          end_at_event=end)
      return tc

    previous_end = None
    filled_configs = []
    inference_config_index = None
    for i, config in enumerate(self.tensor_configs):
      current_begin = config.begin_at_event

      if previous_end != current_begin and current_begin > 0:
        filled_configs.append(create_float_config(previous_end, current_begin))

      filled_configs.append(config)
      previous_end = config.end_at_event

      if self.inference_config_index == i:
        inference_config_index = len(filled_configs) - 1

    if previous_end is not None or not filled_configs:
      filled_configs.append(create_float_config(previous_end, None))

    self.tensor_configs = filled_configs
    # Note that self.inference_config_index is either None or in [0, len)
    # where len is the length of the previous self.tensor_configs. So the
    # update below either keeps it set to None or updates the index to the
    # reindexed value for the new self.tensor_configs.
    self.inference_config_index = inference_config_index

@dataclasses.dataclass
class AqtMatmulConfig(_BaseConfig):
  """Quantization config for a matmul.

  Attributes:
    lhs: quantization schedule for left-hand side argument
    rhs: quantization schedule for right-hand side argument
  """
  lhs: AqtScheduleConfig
  rhs: AqtScheduleConfig

<|endoftext|>


<commit_before>from rplugin.python3 import cmake
from unittest import TestCase as utTestCase
from unittest.mock import Mock as utMock
from unittest import main as utmain
import neovim
import subprocess
import os

# 1. Determine if Clean or Dirty
#    a. Clean - Goto Step #2
#    b. Dirty - Delete all Old Build Files
# 2. Run CMake Project Configuration
#    a. Passes - Goto Step #3
#    b. Fails - Delete all Old Build Files
# 3. Setup RTags Daemon
#    a. Passes - Goto Step #4
#    b. Fails - Kill Daemon & Repeat Step #3
# 4. Setup RTags Client
#    a. Passes - Done!
#    b. Fails - Kill Daemon & Exit

class TestCMake(utTestCase):
    def setUp(self):
        print("Setup for CMake unit test")
        subprocess.call(["touch", "/tmp/nvimsocket"])
        os.environ["NVIM_LISTEN_ADDRESS"] = "/tmp/nvimsocket"
        #subprocess.call(["nvim", "--embed", "-c", "\"terminal python3\""])
        self.nvimproc = subprocess.Popen(
            ["nvim", "--headless", "-c", "\"terminal python3\""])
        nvim = neovim.attach('socket', path=os.environ['NVIM_LISTEN_ADDRESS'])
        self.cmake_plugin = cmake.Main(nvim)

    def tearDown(self):
        print("Teardown for CMake unit test")
        self.nvimproc.terminate()
        del os.environ["NVIM_LISTEN_ADDRESS"]

    def test_InitClean(self):
        #build_area = utMock()
        try:
            os.chdir("tests/rplugin/python3/clean")
        except OSError:
            print("Test Error: Couldn't cd into 'clean'")
            raise
        for path in cmake.cmake_build_info["old_cmake_files"]:
            self.assertFalse(path.is_file())
        self.assertFalse(cmake.cmake_build_info["old_cmake_dir"].is_file())

if __name__ == '__main__':
    utmain()
<commit_msg>[Broken Test] More setup code for the unit tests.<commit_after>from rplugin.python3 import cmake
from unittest import TestCase as utTestCase
from unittest.mock import Mock as utMock
from unittest import main as utmain
from pathlib import Path
import neovim
import subprocess
import os

# 1. Determine if Clean or Dirty
#    a. Clean - Goto Step #2
#    b. Dirty - Delete all Old Build Files
# 2. Run CMake Project Configuration
#    a. Passes - Goto Step #3
#    b. Fails - Delete all Old Build Files
# 3. Setup RTags Daemon
#    a. Passes - Goto Step #4
#    b. Fails - Kill Daemon & Repeat Step #3
# 4. Setup RTags Client
#    a. Passes - Done!
#    b. Fails - Kill Daemon & Exit
NVIM_LISTEN_ADDRESS = "/tmp/nvim-CMakePluginTest"
nvim_remote_socket = Path(NVIM_LISTEN_ADDRESS)

class TestCMake(utTestCase):
    def setUp(self):
        #self.nvim_remote_socket = Path("/tmp/nvimsocket")
        #os.environ["NVIM_LISTEN_ADDRESS"] = "/tmp/nvimsocket"
        #subprocess.call(["nvim", "--headless", "-c", "\"terminal python3\""])
        #self.nvimproc = subprocess.Popen(
        #    ["nvim", "--headless", "-c", "\"terminal python3\""])
            #["NVIM_LISTEN_ADDRESS="+str(self.nvim_remote_socket), "nvim", "--headless", "-c", "\"terminal python3\""])
        #try:
        #    nvim = neovim.attach("socket",
        #                        path=os.environ["NVIM_LISTEN_ADDRESS"])
        #except OSError:
        #    print("Can't Launch Neovim in test fixture setup.")
        #    raise
        print("Setup for CMake unit test")
        if nvim_remote_socket.is_file():
            subprocess.call(["rm", str(nvim_remote_socket)])
        subprocess.call(["touch", str(nvim_remote_socket)])
        os.environ["NVIM_LISTEN_ADDRESS"] = str(nvim_remote_socket)
        self.nvimproc = subprocess.Popen(
            ["nvim", "--headless", "-c", "\"terminal python3\""])
        nvim = neovim.attach('socket', path=NVIM_LISTEN_ADDRESS)
        self.cmake_plugin = cmake.Main(nvim)

    def tearDown(self):
        print("Teardown for CMake unit test")
        self.nvimproc.terminate()
        del os.environ["NVIM_LISTEN_ADDRESS"]
        if nvim_remote_socket.is_file():
            subprocess.call(["rm", str(nvim_remote_socket)])

    def test_InitClean(self):
        #build_area = utMock()
        try:
            os.chdir("tests/rplugin/python3/clean")
        except OSError:
            print("Test Error: Couldn't cd into 'clean'")
            raise
        for path in cmake.cmake_build_info["old_cmake_files"]:
            self.assertFalse(path.is_file())
        self.assertFalse(cmake.cmake_build_info["old_cmake_dir"].is_file())
        try:
            os.chdir("tests/rplugin/python3")
        except OSError:
            print("Test Error: Couldn't cd out of 'clean'")
            raise

if __name__ == '__main__':
    utmain()
<|endoftext|>


<commit_before>import random
import datetime

class RandomDatetime(object):

    def __init__(self, pre_days=30, post_days=30):
        self.pre_days = pre_days
        self.post_days = post_days
        self.now = datetime.datetime.now()

    def datetime(self, pre=None, post=None):
        pre = pre or self.pre_days
        post = post or self.post_days
        delta = datetime.timedelta(random.randrange(- pre, post + 1))
        return self.now + delta

    def date(self, **kwds):
        return datetime.date(*self.datetime(**kwds).timetuple()[:3])

def timedeltastr(timedelta):
    hour = timedelta.seconds // 60 // 60
    min = timedelta.seconds // 60 - (hour * 60)
    return '{0:02d}:{1:02d}'.format(timedelta.days * 24 + hour, min)

def node(level, heading, todo=None, scheduled=None, deadline=None,
         closed=None, clock=[]):
    active_datestr = lambda x: x.strftime('<%Y-%m-%d %a>')
    inactive_datestr = lambda x: x.strftime('[%Y-%m-%d %a]')
    inactive_datetimestr = lambda x: x.strftime('[%Y-%m-%d %a %H:%M]')
    yield '*' * level
    yield ' '
    if todo:
        yield todo
        yield ' '
    yield heading
    yield '\n'
    if scheduled or deadline or closed:
        yield ' ' * level
    for (name, date, datestr) in [('CLOSED', closed, inactive_datestr),
                                  ('DEADLINE', deadline, active_datestr),
                                  ('SCHEDULED', scheduled, active_datestr)]:
        if date:
            yield ' '
            yield name
            yield ': '
            yield datestr(date)
    if scheduled or deadline or closed:
        yield '\n'
    for (clock_start, clock_end) in clock:
        yield ' ' * (level + 1)
        yield 'CLOCK: '
        yield inactive_datetimestr(clock_start)
        yield '--'
        yield inactive_datetimestr(clock_end)
        yield ' => '
        yield timedeltastr(clock_end - clock_start)
        yield '\n'

def makeorg(num, **kwds):
    heading_pops = ['aaa', 'bbb', 'ccc']
    true_or_false = [True, False]
    rd = RandomDatetime(**kwds)
    for i in range(num):
        kwds = {}
        if i == 0:
            kwds['level'] = 1
        else:
            kwds['level'] = random.randrange(1, 4)
        kwds['heading'] = random.choice(heading_pops)
        if random.choice(true_or_false):
            if random.choice(true_or_false):
                kwds['todo'] = 'TODO'
            else:
                kwds['closed'] = rd.date(post=0)
                kwds['todo'] = 'DONE'
        for sdc in ['scheduled', 'deadline']:
            if random.choice(true_or_false):
                kwds[sdc] = rd.date()
        if random.choice(true_or_false):
            kwds['clock'] = clock = []
            for _ in range(random.randrange(1, 5)):
                start = rd.datetime(post=0)
                end = start + datetime.timedelta(
                    0, random.randrange(1, 5) * 60 * 60)
                clock.append((start, end))
        for s in node(**kwds):
            yield s

def writeorg(file, *args, **kwds):
    file.writelines(makeorg(*args, **kwds))

def run(num):
    import sys
    writeorg(sys.stdout, num)
<commit_msg>Fix RandomDatetime.datetime when pre or post is 0<commit_after>import random
import datetime

class RandomDatetime(object):

    def __init__(self, pre_days=30, post_days=30):
        self.pre_days = pre_days
        self.post_days = post_days
        self.now = datetime.datetime.now()

    def datetime(self, pre=None, post=None):
        pre = self.pre_days if pre is None else pre
        post = self.post_days if post is None else post
        delta = datetime.timedelta(random.randrange(- pre, post + 1))
        return self.now + delta

    def date(self, **kwds):
        return datetime.date(*self.datetime(**kwds).timetuple()[:3])

def timedeltastr(timedelta):
    hour = timedelta.seconds // 60 // 60
    min = timedelta.seconds // 60 - (hour * 60)
    return '{0:02d}:{1:02d}'.format(timedelta.days * 24 + hour, min)

def node(level, heading, todo=None, scheduled=None, deadline=None,
         closed=None, clock=[]):
    active_datestr = lambda x: x.strftime('<%Y-%m-%d %a>')
    inactive_datestr = lambda x: x.strftime('[%Y-%m-%d %a]')
    inactive_datetimestr = lambda x: x.strftime('[%Y-%m-%d %a %H:%M]')
    yield '*' * level
    yield ' '
    if todo:
        yield todo
        yield ' '
    yield heading
    yield '\n'
    if scheduled or deadline or closed:
        yield ' ' * level
    for (name, date, datestr) in [('CLOSED', closed, inactive_datestr),
                                  ('DEADLINE', deadline, active_datestr),
                                  ('SCHEDULED', scheduled, active_datestr)]:
        if date:
            yield ' '
            yield name
            yield ': '
            yield datestr(date)
    if scheduled or deadline or closed:
        yield '\n'
    for (clock_start, clock_end) in clock:
        yield ' ' * (level + 1)
        yield 'CLOCK: '
        yield inactive_datetimestr(clock_start)
        yield '--'
        yield inactive_datetimestr(clock_end)
        yield ' => '
        yield timedeltastr(clock_end - clock_start)
        yield '\n'

def makeorg(num, **kwds):
    heading_pops = ['aaa', 'bbb', 'ccc']
    true_or_false = [True, False]
    rd = RandomDatetime(**kwds)
    for i in range(num):
        kwds = {}
        if i == 0:
            kwds['level'] = 1
        else:
            kwds['level'] = random.randrange(1, 4)
        kwds['heading'] = random.choice(heading_pops)
        if random.choice(true_or_false):
            if random.choice(true_or_false):
                kwds['todo'] = 'TODO'
            else:
                kwds['closed'] = rd.date(post=0)
                kwds['todo'] = 'DONE'
        for sdc in ['scheduled', 'deadline']:
            if random.choice(true_or_false):
                kwds[sdc] = rd.date()
        if random.choice(true_or_false):
            kwds['clock'] = clock = []
            for _ in range(random.randrange(1, 5)):
                start = rd.datetime(post=0)
                end = start + datetime.timedelta(
                    0, random.randrange(1, 5) * 60 * 60)
                clock.append((start, end))
        for s in node(**kwds):
            yield s

def writeorg(file, *args, **kwds):
    file.writelines(makeorg(*args, **kwds))

def run(num):
    import sys
    writeorg(sys.stdout, num)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#...............................licence...........................................
#
#     (C) Copyright 2008 Telefonica Investigacion y Desarrollo
#     S.A.Unipersonal (Telefonica I+D)
#
#     This file is part of Morfeo EzWeb Platform.
#
#     Morfeo EzWeb Platform is free software: you can redistribute it and/or modify
#     it under the terms of the GNU Affero General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     Morfeo EzWeb Platform is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the GNU Affero General Public License
#     along with Morfeo EzWeb Platform.  If not, see <http://www.gnu.org/licenses/>.
#
#     Info about members and contributors of the MORFEO project
#     is available at
#
#     http://morfeo-project.org
#
#...............................licence...........................................#

#

from django.utils import simplejson
from igadget.views import SaveIGadget
from preferences.views import update_tab_preferences, update_workspace_preferences
from workspace.models import WorkSpace, UserWorkSpace
from workspace.utils import createTab
from lxml import etree

NAME_XPATH = etree.ETXPath('/Template/Catalog.ResourceDescription/Name')
TAB_XPATH = etree.ETXPath('/Template/Catalog.ResourceDescription/IncludedResources/Tab')
RESOURCE_XPATH = etree.ETXPath('Resource')
POSITION_XPATH = etree.ETXPath('Position')
RENDERING_XPATH = etree.ETXPath('Rendering')
PREFERENCE_XPATH = etree.ETXPath('Preference')
PROPERTIES_XPATH = etree.ETXPath('Property')

def buildWorkspaceFromTemplate(template, user):

    xml = etree.fromstring(template)

    name = NAME_XPATH(xml)[0].text

    #Workspace creation
    workspace = WorkSpace(name=name, creator=user)
    workspace.save()

    #Adding user reference to workspace in the many to many relationship
    user_workspace = UserWorkSpace(user=user, workspace=workspace, active=False)
    user_workspace.save()

    preferences = PREFERENCE_XPATH(xml)
    new_values = {}
    for preference in preferences:
        new_values[preference.get('name')] = {
            'inherit': False,
            'value': preference.get('value'),
        }

    if len(new_values) > 0:
        update_workspace_preferences(workspace, new_values)

    tabs = TAB_XPATH(xml)

    forced_values = {
        'igadget': {},
    }
    for tabElement in tabs:
        tab, junk = createTab(tabElement.get('name'), user, workspace)

        preferences = PREFERENCE_XPATH(tabElement)
        new_values = {}
        for preference in preferences:
            new_values[preference.get('name')] = {
                'inherit': False,
                'value': preference.get('value'),
            }

        if len(new_values) > 0:
            update_tab_preferences(tab, new_values)

        resources = RESOURCE_XPATH(tabElement)
        for resource in resources:
            igadget_uri = "/workspace/" + str(workspace.id) + "/tab/" + str(tab.id) + "/igadgets"
            gadget_uri = "/user/" + user.username + "/gadgets/" + '/'.join([resource.get('vendor'), resource.get('name'), resource.get('version')])

            position = POSITION_XPATH(resource)[0]
            rendering = RENDERING_XPATH(resource)[0]

            initial_variable_values = {}
            igadget_forced_values = {}
            properties = PROPERTIES_XPATH(resource)
            for prop in properties:
                initial_variable_values[prop.get('name')] = prop.get('value')

            preferences = PREFERENCE_XPATH(resource)
            for pref in preferences:
                read_only = pref.get('readonly')
                if read_only and read_only == 'true':
                    igadget_forced_values[pref.get('name')] = pref.get('value')
                else:
                    initial_variable_values[pref.get('name')] = pref.get('value')

            igadget_data = {
                "left": int(position.get('x')),
                "top": int(position.get('y')),
                "icon_left": -1,
                "icon_top": -1,
                "zIndex": int(position.get('z')),
                "width": int(rendering.get('width')),
                "height": int(rendering.get('height')),
                "name": resource.get('title'),
                "menu_color": "FFFFFF",
                "layout": int(rendering.get('layout')),
                "uri": igadget_uri,
                "gadget": gadget_uri}

            igadget = SaveIGadget(igadget_data, user, tab, initial_variable_values)
            forced_values['igadget'][str(igadget['id'])] = igadget_forced_values

    workspace.forcedValues = simplejson.dumps(forced_values, ensure_ascii=False)
    workspace.save()

    return workspace
<commit_msg>[PATCH 21/61] Parse wiring connections from mashup templates.<commit_after># -*- coding: utf-8 -*-

#...............................licence...........................................
#
#     (C) Copyright 2008 Telefonica Investigacion y Desarrollo
#     S.A.Unipersonal (Telefonica I+D)
#
#     This file is part of Morfeo EzWeb Platform.
#
#     Morfeo EzWeb Platform is free software: you can redistribute it and/or modify
#     it under the terms of the GNU Affero General Public License as published by
#     the Free Software Foundation, either version 3 of the License, or
#     (at your option) any later version.
#
#     Morfeo EzWeb Platform is distributed in the hope that it will be useful,
#     but WITHOUT ANY WARRANTY; without even the implied warranty of
#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#     GNU Affero General Public License for more details.
#
#     You should have received a copy of the GNU Affero General Public License
#     along with Morfeo EzWeb Platform.  If not, see <http://www.gnu.org/licenses/>.
#
#     Info about members and contributors of the MORFEO project
#     is available at
#
#     http://morfeo-project.org
#
#...............................licence...........................................#

#

from django.utils import simplejson
from igadget.models import Variable
from igadget.views import SaveIGadget
from preferences.views import update_tab_preferences, update_workspace_preferences
from workspace.models import WorkSpace, UserWorkSpace
from workspace.utils import createTab
from connectable.models import In, Out, RelatedInOut
from connectable.views import createChannel
from lxml import etree

NAME_XPATH = etree.ETXPath('/Template/Catalog.ResourceDescription/Name')
TAB_XPATH = etree.ETXPath('/Template/Catalog.ResourceDescription/IncludedResources/Tab')
RESOURCE_XPATH = etree.ETXPath('Resource')
POSITION_XPATH = etree.ETXPath('Position')
RENDERING_XPATH = etree.ETXPath('Rendering')
PREFERENCE_XPATH = etree.ETXPath('Preference')
PROPERTIES_XPATH = etree.ETXPath('Property')
WIRING_XPATH = etree.ETXPath('/Template/Platform.Wiring')
CHANNEL_XPATH = etree.ETXPath('Channel')
IN_XPATH = etree.ETXPath('In')
OUT_XPATH = etree.ETXPath('Out')

def buildWorkspaceFromTemplate(template, user):

    xml = etree.fromstring(template)

    name = NAME_XPATH(xml)[0].text

    #Workspace creation
    workspace = WorkSpace(name=name, creator=user)
    workspace.save()

    #Adding user reference to workspace in the many to many relationship
    user_workspace = UserWorkSpace(user=user, workspace=workspace, active=False)
    user_workspace.save()

    preferences = PREFERENCE_XPATH(xml)
    new_values = {}
    igadget_id_mapping = {}
    for preference in preferences:
        new_values[preference.get('name')] = {
            'inherit': False,
            'value': preference.get('value'),
        }

    if len(new_values) > 0:
        update_workspace_preferences(workspace, new_values)

    tabs = TAB_XPATH(xml)

    forced_values = {
        'igadget': {},
    }
    for tabElement in tabs:
        tab, junk = createTab(tabElement.get('name'), user, workspace)

        preferences = PREFERENCE_XPATH(tabElement)
        new_values = {}
        for preference in preferences:
            new_values[preference.get('name')] = {
                'inherit': False,
                'value': preference.get('value'),
            }

        if len(new_values) > 0:
            update_tab_preferences(tab, new_values)

        resources = RESOURCE_XPATH(tabElement)
        for resource in resources:
            igadget_uri = "/workspace/" + str(workspace.id) + "/tab/" + str(tab.id) + "/igadgets"
            gadget_uri = "/user/" + user.username + "/gadgets/" + '/'.join([resource.get('vendor'), resource.get('name'), resource.get('version')])

            position = POSITION_XPATH(resource)[0]
            rendering = RENDERING_XPATH(resource)[0]

            initial_variable_values = {}
            igadget_forced_values = {}
            properties = PROPERTIES_XPATH(resource)
            for prop in properties:
                initial_variable_values[prop.get('name')] = prop.get('value')

            preferences = PREFERENCE_XPATH(resource)
            for pref in preferences:
                read_only = pref.get('readonly')
                if read_only and read_only == 'true':
                    igadget_forced_values[pref.get('name')] = pref.get('value')
                else:
                    initial_variable_values[pref.get('name')] = pref.get('value')

            igadget_data = {
                "left": int(position.get('x')),
                "top": int(position.get('y')),
                "icon_left": -1,
                "icon_top": -1,
                "zIndex": int(position.get('z')),
                "width": int(rendering.get('width')),
                "height": int(rendering.get('height')),
                "name": resource.get('title'),
                "menu_color": "FFFFFF",
                "layout": int(rendering.get('layout')),
                "uri": igadget_uri,
                "gadget": gadget_uri}

            igadget = SaveIGadget(igadget_data, user, tab, initial_variable_values)
            forced_values['igadget'][str(igadget.id)] = igadget_forced_values
            igadget_id_mapping[resource.get('id')] = igadget

    workspace.forcedValues = simplejson.dumps(forced_values, ensure_ascii=False)
    workspace.save()

    # wiring
    wiring = WIRING_XPATH(xml)[0]
    channels = CHANNEL_XPATH(wiring)
    channel_connectables = {}
    for channel in channels:
        channel_connectables[channel.get('id')] = {
            'connectable': createChannel(workspace, channel.get('name')),
            'element': channel,
        }

    for key in channel_connectables:
        channel = channel_connectables[key]
        ins = IN_XPATH(channel['element'])
        for in_ in ins:
            igadget_id = in_.get('igadget')
            igadget = igadget_id_mapping[igadget_id]
            name = in_.get('name')

            connectable = In.objects.get(variable__abstract_variable__name=name, variable__igadget=igadget)
            connectable.inouts.add(channel['connectable'])
            connectable.save()

        outs = OUT_XPATH(channel['element'])
        for out in outs:
            igadget_id = out.get('igadget')
            igadget = igadget_id_mapping[igadget_id]
            name = out.get('name')

            variable = Variable.objects.get(igadget=igadget, abstract_variable__name=name)
            connectable = Out.objects.get(abstract_variable=variable.abstract_variable)
            connectable.inouts.add(channel['connectable'])
            connectable.save()

        out_channels = CHANNEL_XPATH(channel['element'])
        for out_channel_element in out_channels:
            out_channel = channel_connectables[out_channel_element.get('id')]['connectable']
            relation = RelatedInOut(in_inout=channel['connectable'], out_inout=out_channel)
            relation.save()

    return workspace
<|endoftext|>