<commit_before>"""
sentry.db.backends.sqlalchemy
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.db.backends.base import SentryBackend

import datetime

from sqlalchemy import create_engine
from sqlalchemy.sql import select

from sentry.db.backends.sqlalchemy.models import metadata, model_map

class SQLAlchemyBackend(SentryBackend):
    def __init__(self, uri, **kwargs):
        self.engine = create_engine(uri, **kwargs)
        metadata.bind = self.engine

    def create_model(self, schema):
        metadata.create(model_map[schema])

    ## Hash table lookups

    def add(self, schema, **values):
        # generates a pk and sets the values
        pk = self.generate_key(schema)
        table = model_map[schema]
        table.insert().execute(id=pk, **values)
        return pk

    def delete(self, schema, pk):
        table = model_map[schema]
        table.delete(table.c.id==pk).execute()

    def set(self, schema, pk, **values):
        table = model_map[schema]
        table.update(table.c.id==pk).execute(**values)

    def get(self, schema, pk):
        table = model_map[schema]
        query = select([table], table.c.id==pk)
        return query.execute().fetchone()

    def incr(self, schema, pk, key, amount=1):
        table = model_map[schema]
        table.update(table.c.id==pk).execute(getattr(table.c, key)==getattr(table.c, key) + amount)

    # meta data is stored in a seperate key to avoid collissions and heavy getall pulls

    def set_meta(self, schema, pk, **values):
        self.conn.hmset(self._get_metadata_key(schema, pk), values)

    def get_meta(self, schema, pk):
        return self.conn.hgetall(self._get_metadata_key(schema, pk))

    def get_data(self, schema, pk):
        return self.conn.hgetall(self._get_data_key(schema, pk))

    def count(self, schema, index='default'):
        return self.conn.zcard(self._get_index_key(schema, index))

    def list(self, schema, index='default', offset=0, limit=-1, desc=False):
        if limit > 0:
            end = offset+limit
        else:
            end = limit
        pk_set = self.conn.zrange(self._get_index_key(schema, index), start=offset, end=end, desc=desc)
        return [(pk, self.get_data(schema, pk)) for pk in pk_set]

    ## Indexes using sorted sets

    def add_relation(self, from_schema, from_pk, to_schema, to_pk, score):
        # adds a relation to a sorted index for base instance
        if isinstance(score, datetime.datetime):
            score = score.strftime('%s.%m')
        self.conn.zadd(self._get_relation_key(from_schema, from_pk, to_schema), to_pk, float(score))

    def remove_relation(self, from_schema, from_pk, to_schema, to_pk=None):
        if to_pk:
            self.conn.zrem(self._get_relation_key(from_schema, from_pk, to_schema), to_pk)
        else:
            self.conn.delete(self._get_relation_key(from_schema, from_pk, to_schema))

    def list_relations(self, from_schema, from_pk, to_schema, offset=0, limit=-1, desc=False):
        # lists relations in a sorted index for base instance
        # XXX: this is O(n)+1, ugh
        if limit > 0:
            end = offset+limit
        else:
            end = limit

        pk_set = self.conn.zrange(self._get_relation_key(from_schema, from_pk, to_schema), start=offset, end=end, desc=desc)

        return [(pk, self.conn.hgetall(self._get_data_key(to_schema, pk))) for pk in pk_set]

    def add_to_index(self, schema, pk, index, score):
        # adds an instance to a sorted index
        if isinstance(score, datetime.datetime):
            score = score.strftime('%s.%m')
        self.conn.zadd(self._get_index_key(schema, index), pk, float(score))

    def remove_from_index(self, schema, pk, index):
        self.conn.zrem(self._get_index_key(schema, index), pk)

    ## Generic indexes

    # TODO: can we combine constraint indexes with sort indexes? (at least the API)

    def add_to_cindex(self, schema, pk, **kwargs):
        # adds an index to a composite index (for checking uniqueness)
        self.conn.sadd(self._get_constraint_key(schema, kwargs), pk)

    def remove_from_cindex(self, schema, pk, **kwargs):
        # adds an index to a composite index (for checking uniqueness)
        self.conn.srem(self._get_constraint_key(schema, kwargs), pk)

    def list_by_cindex(self, schema, **kwargs):
        # returns a list of keys associated with a constraint
        return list(self.conn.smembers(self._get_constraint_key(schema, kwargs)))
<commit_msg>Getting schooled by ronny on SQ and learning I cant make up my own syntax<commit_after>"""
sentry.db.backends.sqlalchemy
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

from sentry.db.backends.base import SentryBackend

import datetime

from sqlalchemy import create_engine
from sqlalchemy.sql import select

from sentry.db.backends.sqlalchemy.models import metadata, model_map

class SQLAlchemyBackend(SentryBackend):
    def __init__(self, uri, **kwargs):
        self.engine = create_engine(uri, **kwargs)
        metadata.bind = self.engine

    def create_model(self, schema):
        metadata.create(model_map[schema])

    ## Hash table lookups

    def add(self, schema, **values):
        # generates a pk and sets the values
        pk = self.generate_key(schema)
        table = model_map[schema]
        table.insert().execute(id=pk, **values)
        return pk

    def delete(self, schema, pk):
        table = model_map[schema]
        table.delete(table.c.id==pk).execute()

    def set(self, schema, pk, **values):
        table = model_map[schema]
        table.update(table.c.id==pk).execute(**values)

    def get(self, schema, pk):
        table = model_map[schema]
        query = select([table], table.c.id==pk)
        return query.execute().fetchone()

    def incr(self, schema, pk, key, amount=1):
        table = model_map[schema]
        table.update(table.c.id==pk, {key: getattr(table.c, key) + amount}).execute()

    # meta data is stored in a seperate key to avoid collissions and heavy getall pulls

    # TODO:
    def set_meta(self, schema, pk, **values):
        self.conn.hmset(self._get_metadata_key(schema, pk), values)

    def get_meta(self, schema, pk):
        return self.conn.hgetall(self._get_metadata_key(schema, pk))

    def get_data(self, schema, pk):
        return self.conn.hgetall(self._get_data_key(schema, pk))

    def count(self, schema, index='default'):
        return self.conn.zcard(self._get_index_key(schema, index))

    def list(self, schema, index='default', offset=0, limit=-1, desc=False):
        if limit > 0:
            end = offset+limit
        else:
            end = limit
        pk_set = self.conn.zrange(self._get_index_key(schema, index), start=offset, end=end, desc=desc)
        return [(pk, self.get_data(schema, pk)) for pk in pk_set]

    ## Indexes using sorted sets

    def add_relation(self, from_schema, from_pk, to_schema, to_pk, score):
        # adds a relation to a sorted index for base instance
        if isinstance(score, datetime.datetime):
            score = score.strftime('%s.%m')
        self.conn.zadd(self._get_relation_key(from_schema, from_pk, to_schema), to_pk, float(score))

    def remove_relation(self, from_schema, from_pk, to_schema, to_pk=None):
        if to_pk:
            self.conn.zrem(self._get_relation_key(from_schema, from_pk, to_schema), to_pk)
        else:
            self.conn.delete(self._get_relation_key(from_schema, from_pk, to_schema))

    def list_relations(self, from_schema, from_pk, to_schema, offset=0, limit=-1, desc=False):
        # lists relations in a sorted index for base instance
        # XXX: this is O(n)+1, ugh
        if limit > 0:
            end = offset+limit
        else:
            end = limit

        pk_set = self.conn.zrange(self._get_relation_key(from_schema, from_pk, to_schema), start=offset, end=end, desc=desc)

        return [(pk, self.conn.hgetall(self._get_data_key(to_schema, pk))) for pk in pk_set]

    def add_to_index(self, schema, pk, index, score):
        # adds an instance to a sorted index
        if isinstance(score, datetime.datetime):
            score = score.strftime('%s.%m')
        self.conn.zadd(self._get_index_key(schema, index), pk, float(score))

    def remove_from_index(self, schema, pk, index):
        self.conn.zrem(self._get_index_key(schema, index), pk)

    ## Generic indexes

    # TODO: can we combine constraint indexes with sort indexes? (at least the API)

    def add_to_cindex(self, schema, pk, **kwargs):
        # adds an index to a composite index (for checking uniqueness)
        self.conn.sadd(self._get_constraint_key(schema, kwargs), pk)

    def remove_from_cindex(self, schema, pk, **kwargs):
        # adds an index to a composite index (for checking uniqueness)
        self.conn.srem(self._get_constraint_key(schema, kwargs), pk)

    def list_by_cindex(self, schema, **kwargs):
        # returns a list of keys associated with a constraint
        return list(self.conn.smembers(self._get_constraint_key(schema, kwargs)))
<|endoftext|>


<commit_before>import time
import itertools
import operator

from django.core import management

from apps.webdriver_testing.webdriver_base import WebdriverTestCase
from apps.webdriver_testing import data_helpers
from apps.webdriver_testing.data_factories import UserFactory
from apps.webdriver_testing.data_factories import TeamMemberFactory
from apps.webdriver_testing.data_factories import TeamProjectFactory
from apps.webdriver_testing.data_factories import TeamVideoFactory
from apps.webdriver_testing.pages.site_pages import video_page
from apps.webdriver_testing.pages.site_pages.teams import videos_tab
from apps.webdriver_testing.pages.site_pages import watch_page

class TestCaseVideoResource(WebdriverTestCase):
    """TestSuite for videos via the api

    """
    NEW_BROWSER_PER_TEST_CASE = False

    @classmethod
    def setUpClass(cls):
        super(TestCaseVideoResource, cls).setUpClass()
        cls.user = UserFactory.create(username = 'user')
        cls.data_utils = data_helpers.DataHelpers()
        cls.data_utils.create_user_api_key(cls.user)
        cls.test_video = cls.data_utils.create_video()
        cls.open_team = TeamMemberFactory.create(
            team__name="Cool team",
            team__slug='team-with-projects',
            team__description='this is the coolest, most creative team ever',
            user = cls.user,
            ).team
        cls.project1 = TeamProjectFactory(
            team=cls.open_team,
            name='team project one',
            slug = 'team-project-one',
            description = 'subtitle project number 1',
            guidelines = 'do a good job',
            workflow_enabled=False)

        cls.project2 = TeamProjectFactory(
            team=cls.open_team,
            name='team project two',
            workflow_enabled=True)
        cls.video_pg = video_page.VideoPage(cls)
        cls.watch_pg = watch_page.WatchPage(cls)

    def tearDown(self):
        self.browser.get_screenshot_as_file('MYTMP/%s' % self.id())

    def test_video__list(self):
        """List the available videos that are in teams.

        GET /api2/partners/videos/
        video_url
        team
        project
        order_by
            title: ascending
            -title: descending
            created: older videos first
            -created : newer videos
        """
        tv = self.data_utils.create_video()
        TeamVideoFactory.create(team=self.open_team, video=tv,
                                added_by=self.user)
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/'
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertIn(tv.video_id, videos_list)

    def test_query__project(self):
        """List the available videos.

        GET /api2/partners/videos/
        video_url
        team
        project
        order_by
            title: ascending
            -title: descending
            created: older videos first
            -created : newer videos
        """
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/?project=%s' %self.project1.slug
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertEqual(5, len(videos_list))

    def test_query__team(self):
        """List the available videos.

        GET /api2/partners/videos/team=<team slug>
        """
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/?team=%s' %self.open_team.slug
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertEqual(5, len(videos_list))

    def test_sort__newest(self):
        """List the available videos.

        GET /api2/partners/videos/
        """
        team_vid_list = []
        for x in range(3):
            vid = TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1).video
            team_vid_list.append(vid.title)

            time.sleep(1)
        url_part = 'videos/?order_by=-created'
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('title')):
                videos_list.append(k)

        self.assertEqual(team_vid_list[-1], videos_list[0])

    def test_sort__title(self):
        """List the available videos sorted by title (desc)

        GET /api2/partners/videos/
        """
        for x in range(3):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1,
                video__title = 'Zzz-test-video')

        url_part = 'videos/?order_by=-title'
        status, response = self.data_utils.api_get_request(self.user, url_part)

        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('title')):
                videos_list.append(k)
        self.assertEqual('Zzz-test-video', videos_list[0])

    def test_video__create(self):
        """Add a new video.

        POST /api2/partners/videos/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user, url_part, url_data)
        self.video_pg.open_video_page(response['id'])
        #Check response metadata
        for k, v in url_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertTrue(self.video_pg.video_embed_present())

    def test_video__details(self):
        """Get video details.

        GET /api2/partners/videos/[video-id]/
        """

        test_video = self.data_utils.create_video()
        url_part = 'videos/%s/' % test_video.video_id
        s, r = self.data_utils.api_get_request(self.user, url_part)
        self.assertEqual(r['title'], test_video.title)
        self.assertIn(test_video.get_primary_videourl_obj().url,
                      r['all_urls'])
        self.video_pg.open_page(r['site_url'])
        self.assertEqual(self.video_pg.video_id(), r['id'])

    def test_update__speaker_metatdata(self):
        """Update video metadata, add speaker name field

        PUT /api2/partners/videos/[video-id]/
        """

        tv = self.data_utils.create_video()
        TeamVideoFactory(team=self.open_team, added_by=self.user, video=tv)

        vid_id = tv.video_id

        url_part = 'videos/%s/' % vid_id
        new_data = {'title': 'MVC webM output sample',
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter'),
                    'metadata' : {
                                             'speaker-name': 'Santa',
                                             'location': 'North Pole'
                                         }
                  }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.logger.info(response)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertIn(new_data['metadata']['speaker-name'],
            self.video_pg.speaker_name())

#        management.call_command('update_index', interactive=False)

#        test_text = new_data['metadata']['speaker-name']
#        self.watch_pg.open_watch_page()
#        results_pg = self.watch_pg.basic_search(test_text)
#        self.assertTrue(results_pg.search_has_results())

    def test_update__metatdata(self):
        """Update video metadata, title, description.

        PUT /api2/partners/videos/[video-id]/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user,
                                                   url_part, url_data)
        vid_id = response['id']

        url_part = 'videos/%s/' % vid_id
        new_data = {'title': 'MVC webM output sample',
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter'),
                  }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertEqual(new_data['description'],
            self.video_pg.description_text())

    def test_update__team(self):
        """Update the video metadata, add to team and edit video description.

        PUT /api2/partners/videos/[video-id]/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user,
                                                   url_part, url_data)
        vid_id = response['id']

        url_part = 'videos/%s/' % vid_id
        new_data = {'team': self.open_team.slug,
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter')
                   }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check the team is listed on the video page
        self.assertTrue(self.video_pg.team_slug(self.open_team.slug))

    def test_update__project(self):
        """Edit video, add it to a team and project.

        PUT /api2/partners/videos/[video-id]/
        """
        #Create the initial video via api and get the id
        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        s, r = self.data_utils.post_api_request(self.user,
                                       url_part, url_data)
        vid_id = r['id']

        #Update the video setting the team and project and new description.
        url_part = 'videos/%s/' % vid_id
        new_data = {'team': self.open_team.slug,
                    'project': self.project2.slug,
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter')
                   }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Open the projects page on the site and verify video in project.
        team_videos_tab = videos_tab.VideosTab(self)
        team_videos_tab.log_in(self.user.username, 'password')
        team_videos_tab.open_team_project(self.open_team.slug,
                                          self.project2.slug)
        self.assertTrue(team_videos_tab.video_present(url_data['title']))

<commit_msg>remove screenshot<commit_after>import time
import itertools
import operator

from django.core import management

from apps.webdriver_testing.webdriver_base import WebdriverTestCase
from apps.webdriver_testing import data_helpers
from apps.webdriver_testing.data_factories import UserFactory
from apps.webdriver_testing.data_factories import TeamMemberFactory
from apps.webdriver_testing.data_factories import TeamProjectFactory
from apps.webdriver_testing.data_factories import TeamVideoFactory
from apps.webdriver_testing.pages.site_pages import video_page
from apps.webdriver_testing.pages.site_pages.teams import videos_tab
from apps.webdriver_testing.pages.site_pages import watch_page

class TestCaseVideoResource(WebdriverTestCase):
    """TestSuite for videos via the api

    """
    NEW_BROWSER_PER_TEST_CASE = False

    @classmethod
    def setUpClass(cls):
        super(TestCaseVideoResource, cls).setUpClass()
        cls.user = UserFactory.create(username = 'user')
        cls.data_utils = data_helpers.DataHelpers()
        cls.data_utils.create_user_api_key(cls.user)
        cls.test_video = cls.data_utils.create_video()
        cls.open_team = TeamMemberFactory.create(
            team__name="Cool team",
            team__slug='team-with-projects',
            team__description='this is the coolest, most creative team ever',
            user = cls.user,
            ).team
        cls.project1 = TeamProjectFactory(
            team=cls.open_team,
            name='team project one',
            slug = 'team-project-one',
            description = 'subtitle project number 1',
            guidelines = 'do a good job',
            workflow_enabled=False)

        cls.project2 = TeamProjectFactory(
            team=cls.open_team,
            name='team project two',
            workflow_enabled=True)
        cls.video_pg = video_page.VideoPage(cls)
        cls.watch_pg = watch_page.WatchPage(cls)

    def test_video__list(self):
        """List the available videos that are in teams.

        GET /api2/partners/videos/
        video_url
        team
        project
        order_by
            title: ascending
            -title: descending
            created: older videos first
            -created : newer videos
        """
        tv = self.data_utils.create_video()
        TeamVideoFactory.create(team=self.open_team, video=tv,
                                added_by=self.user)
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/'
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertIn(tv.video_id, videos_list)

    def test_query__project(self):
        """List the available videos.

        GET /api2/partners/videos/
        video_url
        team
        project
        order_by
            title: ascending
            -title: descending
            created: older videos first
            -created : newer videos
        """
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/?project=%s' %self.project1.slug
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertEqual(5, len(videos_list))

    def test_query__team(self):
        """List the available videos.

        GET /api2/partners/videos/team=<team slug>
        """
        for x in range(5):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        url_part = 'videos/?team=%s' %self.open_team.slug
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('id')):
                videos_list.append(k)
        self.assertEqual(5, len(videos_list))

    def test_sort__newest(self):
        """List the available videos.

        GET /api2/partners/videos/
        """
        team_vid_list = []
        for x in range(3):
            vid = TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1).video
            team_vid_list.append(vid.title)

            time.sleep(1)
        url_part = 'videos/?order_by=-created'
        status, response = self.data_utils.api_get_request(self.user, url_part)
        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('title')):
                videos_list.append(k)

        self.assertEqual(team_vid_list[-1], videos_list[0])

    def test_sort__title(self):
        """List the available videos sorted by title (desc)

        GET /api2/partners/videos/
        """
        for x in range(3):
            TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1)
        TeamVideoFactory.create(team=self.open_team,
                added_by=self.user,
                project = self.project1,
                video__title = 'Zzz-test-video')

        url_part = 'videos/?order_by=-title'
        status, response = self.data_utils.api_get_request(self.user, url_part)

        video_objects =  response['objects']
        videos_list = []
        for k, v in itertools.groupby(video_objects,
            operator.itemgetter('title')):
                videos_list.append(k)
        self.assertEqual('Zzz-test-video', videos_list[0])

    def test_video__create(self):
        """Add a new video.

        POST /api2/partners/videos/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user, url_part, url_data)
        self.video_pg.open_video_page(response['id'])
        #Check response metadata
        for k, v in url_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertTrue(self.video_pg.video_embed_present())

    def test_video__details(self):
        """Get video details.

        GET /api2/partners/videos/[video-id]/
        """

        test_video = self.data_utils.create_video()
        url_part = 'videos/%s/' % test_video.video_id
        s, r = self.data_utils.api_get_request(self.user, url_part)
        self.assertEqual(r['title'], test_video.title)
        self.assertIn(test_video.get_primary_videourl_obj().url,
                      r['all_urls'])
        self.video_pg.open_page(r['site_url'])
        self.assertEqual(self.video_pg.video_id(), r['id'])

    def test_update__speaker_metatdata(self):
        """Update video metadata, add speaker name field

        PUT /api2/partners/videos/[video-id]/
        """

        tv = self.data_utils.create_video()
        TeamVideoFactory(team=self.open_team, added_by=self.user, video=tv)

        vid_id = tv.video_id

        url_part = 'videos/%s/' % vid_id
        new_data = {'title': 'MVC webM output sample',
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter'),
                    'metadata' : {
                                             'speaker-name': 'Santa',
                                             'location': 'North Pole'
                                         }
                  }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.logger.info(response)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertIn(new_data['metadata']['speaker-name'],
            self.video_pg.speaker_name())

#        management.call_command('update_index', interactive=False)

#        test_text = new_data['metadata']['speaker-name']
#        self.watch_pg.open_watch_page()
#        results_pg = self.watch_pg.basic_search(test_text)
#        self.assertTrue(results_pg.search_has_results())

    def test_update__metatdata(self):
        """Update video metadata, title, description.

        PUT /api2/partners/videos/[video-id]/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user,
                                                   url_part, url_data)
        vid_id = response['id']

        url_part = 'videos/%s/' % vid_id
        new_data = {'title': 'MVC webM output sample',
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter'),
                  }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check video displays on the site
        self.assertEqual(new_data['description'],
            self.video_pg.description_text())

    def test_update__team(self):
        """Update the video metadata, add to team and edit video description.

        PUT /api2/partners/videos/[video-id]/
        """

        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        status, response = self.data_utils.post_api_request(self.user,
                                                   url_part, url_data)
        vid_id = response['id']

        url_part = 'videos/%s/' % vid_id
        new_data = {'team': self.open_team.slug,
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter')
                   }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Check the team is listed on the video page
        self.assertTrue(self.video_pg.team_slug(self.open_team.slug))

    def test_update__project(self):
        """Edit video, add it to a team and project.

        PUT /api2/partners/videos/[video-id]/
        """
        #Create the initial video via api and get the id
        url_data = { 'video_url': ('http://qa.pculture.org/amara_tests/'
                                   'Birds_short.webmsd.webm'),
                     'title': 'Test video created via api',
                     'duration': 37 }
        url_part = 'videos/'
        s, r = self.data_utils.post_api_request(self.user,
                                       url_part, url_data)
        vid_id = r['id']

        #Update the video setting the team and project and new description.
        url_part = 'videos/%s/' % vid_id
        new_data = {'team': self.open_team.slug,
                    'project': self.project2.slug,
                    'description': ('This is a sample vid converted to webM '
                                   '720p using Miro Video Converter')
                   }
        status, response = self.data_utils.put_api_request(self.user,
                                                  url_part, new_data)
        self.video_pg.open_video_page(vid_id)

        #Check response metadata
        for k, v in new_data.iteritems():
            self.assertEqual(v, response[k])

        #Open the projects page on the site and verify video in project.
        team_videos_tab = videos_tab.VideosTab(self)
        team_videos_tab.log_in(self.user.username, 'password')
        team_videos_tab.open_team_project(self.open_team.slug,
                                          self.project2.slug)
        self.assertTrue(team_videos_tab.video_present(url_data['title']))

<|endoftext|>


<commit_before>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import time
import shutil
import threading
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptogrogphy modules
from M2Crypto import RSA

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        if self.opts['keep_jobs'] == 0:
            return
        diff = self.opts['keep_jobs'] * 60 * 60
        keep = int(time.time()) - diff
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        for jid in os.listdir(jid_root):
            if int(jid.split('.')[0]) < keep:
                shutil.rmtree(os.path.join(jid_root, jid))

    def start(self):
        '''
        Turn on the master server components
        '''
        self.opts['logger'].info('Starting the Salt Master')
        reqserv = ReqServer(self.opts)
        reqserv.run()
        while True:
            self._clear_old_jobs()
            time.sleep(60)

class Publisher(threading.Thread):
    '''
    The publihing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        threading.Thread.__init__(self)
        self.opts = opts
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PUB)

    def __bind(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        binder = 'tcp://' + self.opts['interface'] + ':'\
               + self.opts['publish_port']
        self.opts['logger'].info('Starting the Salt Publisher on ' + binder)
        self.socket.bind(binder)

    def publish(self, package):
        '''
        Publish out a command to the minions, takes a cmd structure
        '''
        self.opts['logger'].info('Publishing command')
        self.socket.send(package)

    def run(self):
        '''
        Start the publisher
        '''
        self.__bind()

class ReqServer():
    '''
    Starts up a threaded request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://' + self.opts['interface'] + ':'\
                   + self.opts['ret_port']
        self.socket = self.context.socket(zmq.REP)
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        self.opts['logger'].info('Preparing the root key for local'\
                + ' comunication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __bind(self):
        '''
        Binds the reply server
        '''
        self.opts['logger'].info('Setting up the master communication server')
        self.socket.bind(self.uri)
        while True:
            package = self.socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            self.socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = str(time.time())
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        self.opts['logger'].info('Clear payload recieved with commnad '\
                + load['cmd'])
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        self.opts['logger'].info('Pubkey payload recieved with commnad '\
                + load['cmd'])

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        self.opts['logger'].info('AES payload recieved with commnad '\
                + data['cmd'])
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are recieving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        self.opts['logger'].info('Authentication request from '\
                + load['id'])
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            self.opts['logger'].info('New public key placed in pending for '\
                    + load['id'])
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys in'\
                        + ' pending did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                self.opts['logger'].info('Authentication failed from host '\
                        + load['id'] + ', the key is in pending and'\
                        + ' needs to be accepted with saltkey -a '\
                        + load['id'])
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        self.opts['logger'].info('Authentication accepted from '\
                + load['id'])
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        self.opts['logger'].info('Got return from ' + load['id']\
                + ' for job ' + load['jid'])
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            self.opts['logger'].error('An inconsistency occured, a job was'\
                    + ' recieved with a job id that is not present on the'\
                    + ' master: ' + load['jid'])
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluser data out
        '''
        self.opts['logger'].debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        self.opts['logger'].debug('Cluster distributed: ' + str(ret))

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        self.publisher.publish(salt.payload.package(payload))
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

<commit_msg>Change the jid format<commit_after>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import time
import shutil
import threading
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptogrogphy modules
from M2Crypto import RSA

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        if self.opts['keep_jobs'] == 0:
            return
        diff = self.opts['keep_jobs'] * 60 * 60
        keep = int(time.time()) - diff
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        for jid in os.listdir(jid_root):
            if int(jid.split('.')[0]) < keep:
                shutil.rmtree(os.path.join(jid_root, jid))

    def start(self):
        '''
        Turn on the master server components
        '''
        self.opts['logger'].info('Starting the Salt Master')
        reqserv = ReqServer(self.opts)
        reqserv.run()
        while True:
            self._clear_old_jobs()
            time.sleep(60)

class Publisher(threading.Thread):
    '''
    The publihing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        threading.Thread.__init__(self)
        self.opts = opts
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.PUB)

    def __bind(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        binder = 'tcp://' + self.opts['interface'] + ':'\
               + self.opts['publish_port']
        self.opts['logger'].info('Starting the Salt Publisher on ' + binder)
        self.socket.bind(binder)

    def publish(self, package):
        '''
        Publish out a command to the minions, takes a cmd structure
        '''
        self.opts['logger'].info('Publishing command')
        self.socket.send(package)

    def run(self):
        '''
        Start the publisher
        '''
        self.__bind()

class ReqServer():
    '''
    Starts up a threaded request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://' + self.opts['interface'] + ':'\
                   + self.opts['ret_port']
        self.socket = self.context.socket(zmq.REP)
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        self.opts['logger'].info('Preparing the root key for local'\
                + ' comunication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __bind(self):
        '''
        Binds the reply server
        '''
        self.opts['logger'].info('Setting up the master communication server')
        self.socket.bind(self.uri)
        while True:
            package = self.socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            self.socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = time.strftime('%Y%m%d%H%M%S')
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        self.opts['logger'].info('Clear payload recieved with commnad '\
                + load['cmd'])
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        self.opts['logger'].info('Pubkey payload recieved with commnad '\
                + load['cmd'])

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        self.opts['logger'].info('AES payload recieved with commnad '\
                + data['cmd'])
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are recieving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        self.opts['logger'].info('Authentication request from '\
                + load['id'])
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            self.opts['logger'].info('New public key placed in pending for '\
                    + load['id'])
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                self.opts['logger'].error('Authentication attempt from '\
                        + load['id'] + ' failed, the public keys in'\
                        + ' pending did'\
                        + ' not match. This may be an attempt to compromise'\
                        + ' the Salt cluster.')
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                self.opts['logger'].info('Authentication failed from host '\
                        + load['id'] + ', the key is in pending and'\
                        + ' needs to be accepted with saltkey -a '\
                        + load['id'])
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        self.opts['logger'].info('Authentication accepted from '\
                + load['id'])
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        self.opts['logger'].info('Got return from ' + load['id']\
                + ' for job ' + load['jid'])
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            self.opts['logger'].error('An inconsistency occured, a job was'\
                    + ' recieved with a job id that is not present on the'\
                    + ' master: ' + load['jid'])
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluser data out
        '''
        self.opts['logger'].debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        self.opts['logger'].debug('Cluster distributed: ' + str(ret))

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        self.publisher.publish(salt.payload.package(payload))
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

<|endoftext|>


<commit_before>from __future__ import print_function
import re

conll12_fname = "/home/strubell/research/data/conll-2012-new/conll2012-train.txt"
semlink_fname = "/home/strubell/research/data/semlink-1.2.2c/semlink-wsj.txt"

remove_list = ['rel', 'LINK-SLC', 'LINK-PSV', 'LINK-PRO']

arg_re = re.compile(r"(ARG[0-5A])-[A-Za-z]+")

semlink_map = {}
arg_mapping_counts = {}
arg_mappings = {}
proposition_count = 0
with open(semlink_fname, 'r') as semlink_file:
  for line in semlink_file:
    line = line.strip()
    if line:
      proposition_count += 1
      split_line = line.split()

      # key is doc name without ending + sentence number
      key = (split_line[0].split('.')[0], split_line[1])

      # value is predicate + args
      args = split_line[10:]
      # take just the verbnet senses
      stripped_args_vn = map(lambda a: '-'.join(a.split('*')[-1].split('-')[1:]).split(';')[0].replace('-DSP', ''), args)
      stripped_args_fn = map(lambda a: '-'.join(a.split('*')[-1].split('-')[1:]).replace('-DSP', ''), args)

      # want to replace all ARG[0-9A]-[az]+ with ARG[0-9A]
      stripped_args = map(lambda a: arg_re.sub(r'\1', a), stripped_args_fn)

      stripped_removed_args = [a for a in stripped_args if a not in remove_list]

      # update mapping counts
      for arg in stripped_removed_args:
        if arg not in arg_mapping_counts:
          arg_mapping_counts[arg] = 0
        arg_mapping_counts[arg] += 1
        if '=' in arg:
          pb_arg, vn_arg = arg.split('=')
        else:
          pb_arg, vn_arg = arg, arg
        if pb_arg not in arg_mappings:
          arg_mappings[pb_arg] = {}
        if vn_arg not in arg_mappings[pb_arg]:
          arg_mappings[pb_arg][vn_arg] = 0
        arg_mappings[pb_arg][vn_arg] += 1

      value = (split_line[7].split('.')[0], ' '.join(stripped_removed_args))
      if key not in semlink_map:
        semlink_map[key] = []
      semlink_map[key].append(value)

print("Loaded %d semlink propositions" % proposition_count)
# print(arg_mapping_counts)
for arg in arg_mappings:
  print("%s: %s" % (arg, arg_mappings[arg]))

with open(conll12_fname, 'r') as conll12_file:
  # want to scan conll12 file until we find a sentence that is in semlink,
  # then process that sentence
  for line in conll12_file:
    line = line.strip()
    if line:
      split_line = line.split()
<commit_msg>semlink<commit_after>from __future__ import print_function
import re

conll12_fname = "/home/strubell/research/data/conll-2012-new/conll2012-train.txt"
semlink_fname = "/home/strubell/research/data/semlink-1.2.2c/semlink-wsj.txt"

remove_list = ['rel', 'LINK-SLC', 'LINK-PSV', 'LINK-PRO']

arg_re = re.compile(r"(ARG[0-5A])-[A-Za-z]+")

semlink_map = {}
arg_mapping_counts = {}
arg_mappings = {}
proposition_count = 0
with open(semlink_fname, 'r') as semlink_file:
  for line in semlink_file:
    line = line.strip()
    if line:
      proposition_count += 1
      split_line = line.split()

      # key is doc name without ending + sentence number
      key = (split_line[0].split('.')[0], split_line[1])

      # value is predicate + args
      args = split_line[10:]
      # take just the verbnet senses
      stripped_args_vn = map(lambda a: '-'.join(a.split('*')[-1].split('-')[1:]).split(';')[0].replace('-DSP', ''), args)

      # verbnet and framenet senses
      stripped_args_fn = map(lambda a: '-'.join(a.split('*')[-1].split('-')[1:]).replace('-DSP', ''), args)

      # want to replace all ARG[0-9A]-[az]+ with ARG[0-9A]
      stripped_args = map(lambda a: arg_re.sub(r'\1', a), stripped_args_vn)

      stripped_removed_args = [a for a in stripped_args if a not in remove_list]

      # update mapping counts
      for arg in stripped_removed_args:
        if arg not in arg_mapping_counts:
          arg_mapping_counts[arg] = 0
        arg_mapping_counts[arg] += 1
        if '=' in arg:
          pb_arg, vn_arg = arg.split('=')
        else:
          pb_arg, vn_arg = arg, arg
        if pb_arg not in arg_mappings:
          arg_mappings[pb_arg] = {}
        if vn_arg not in arg_mappings[pb_arg]:
          arg_mappings[pb_arg][vn_arg] = 0
        arg_mappings[pb_arg][vn_arg] += 1

      value = (split_line[7].split('.')[0], ' '.join(stripped_removed_args))
      if key not in semlink_map:
        semlink_map[key] = []
      semlink_map[key].append(value)

print("Loaded %d semlink propositions" % proposition_count)
# print(arg_mapping_counts)
for arg in arg_mappings:
  print("%s: %s" % (arg, arg_mappings[arg]))

with open(conll12_fname, 'r') as conll12_file:
  # want to scan conll12 file until we find a sentence that is in semlink,
  # then process that sentence
  for line in conll12_file:
    line = line.strip()
    if line:
      split_line = line.split()
<|endoftext|>


<commit_before>#Copyright (c) 2007-8, Playful Invention Company.
#Copyright (c) 2008-9, Walter Bender

#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.

import gtk
from math import sin,cos,pi
class taTurtle: pass

from tasprites import *
from tasetup import load_image

colors = {}
DEGTOR = 2*pi/360

color_table = (
    0xFF0000,0xFF0D00,0xFF1A00,0xFF2600,0xFF3300,
    0xFF4000,0xFF4D00,0xFF5900,0xFF6600,0xFF7300,
    0xFF8000,0xFF8C00,0xFF9900,0xFFA600,0xFFB300,
    0xFFBF00,0xFFCC00,0xFFD900,0xFFE600,0xFFF200,
    0xFFFF00,0xE6FF00,0xCCFF00,0xB3FF00,0x99FF00,
    0x80FF00,0x66FF00,0x4DFF00,0x33FF00,0x1AFF00,
    0x00FF00,0x00FF0D,0x00FF1A,0x00FF26,0x00FF33,
    0x00FF40,0x00FF4D,0x00FF59,0x00FF66,0x00FF73,
    0x00FF80,0x00FF8C,0x00FF99,0x00FFA6,0x00FFB3,
    0x00FFBF,0x00FFCC,0x00FFD9,0x00FFE6,0x00FFF2,
    0x00FFFF,0x00F2FF,0x00E6FF,0x00D9FF,0x00CCFF,
    0x00BFFF,0x00B3FF,0x00A6FF,0x0099FF,0x008CFF,
    0x0080FF,0x0073FF,0x0066FF,0x0059FF,0x004DFF,
    0x0040FF,0x0033FF,0x0026FF,0x001AFF,0x000DFF,
    0x0000FF,0x0D00FF,0x1A00FF,0x2600FF,0x3300FF,
    0x4000FF,0x4D00FF,0x5900FF,0x6600FF,0x7300FF,
    0x8000FF,0x8C00FF,0x9900FF,0xA600FF,0xB300FF,
    0xBF00FF,0xCC00FF,0xD900FF,0xE600FF,0xF200FF,
    0xFF00FF,0xFF00E6,0xFF00CC,0xFF00B3,0xFF0099,
    0xFF0080,0xFF0066,0xFF004D,0xFF0033,0xFF001A)

def tNew(tw,w,h):
    t = taTurtle()
    t.tw, t.width, t.height = tw, w, h
    t.canvas = sprNew(tw,0,0,gtk.gdk.Pixmap(tw.area,w,h,-1))
    t.canvas.type = 'canvas'
    setlayer(t.canvas,600)
    t.shapelist = \
        [load_image(tw.path, 'shapes','t'+str(i)) for i in range(36)]
    t.spr = sprNew(tw,100,100,t.shapelist[0])
    t.spr.type = 'turtle'
    setlayer(t.spr, 630)
    t.gc = t.canvas.image.new_gc()
    t.shade = 0
    clearscreen(t)
    return t

def clearscreen(t):
    t.xcor, t.ycor, t.heading = 0,0,0
    rect = gtk.gdk.Rectangle(0,0,t.width,t.height)
    t.gc.set_foreground(t.tw.bgcolor)
    t.canvas.image.draw_rectangle(t.gc, True, *rect)
    invalt(t,0,0,t.width,t.height)
    setpensize(t,5)
    setcolor(t,0)
    settextcolor(t,70)
    settextsize(t,32)
    setshade(t,50)
    t.pendown = True
    move_turtle(t)
    turn_turtle(t)
    return None

def forward(t, n):
    n *= t.tw.coord_scale
    t.gc.set_foreground(t.tw.fgcolor)
    oldx, oldy = t.xcor, t.ycor
    try:
        t.xcor += n*sin(t.heading*DEGTOR)
        t.ycor += n*cos(t.heading*DEGTOR)
    except:
        pass
    if t.pendown:
        draw_line(t,oldx,oldy,t.xcor,t.ycor)
    move_turtle(t)
    return None

def seth(t,n):
    try:
        t.heading=n
    except:
        pass
    t.heading%=360
    turn_turtle(t)
    return None

def right(t,n):
    try:
        t.heading+=n
    except:
        pass
    t.heading%=360
    turn_turtle(t)
    return None

def arc(t,a,r):
    t.gc.set_foreground(t.tw.fgcolor)
    r *= t.tw.coord_scale
    try:
        if a<0: larc(t,-a,r)
        else: rarc(t,a,r)
    except:
        pass
    move_turtle(t)
    turn_turtle(t)

def rarc(t,a,r):
    if r<0: r=-r; a=-a
    cx = t.xcor+r*cos(t.heading*DEGTOR)
    cy = t.ycor-r*sin(t.heading*DEGTOR)
    x,y,w,h=t.width/2+int(cx-r),t.height/2-int(cy+r),int(2*r),int(2*r)
    if t.pendown:
        t.canvas.image.draw_arc(t.gc,False,x,y,w,h, \
                                int(180-t.heading-a)*64,int(a)*64)
    invalt(t,x-t.pensize*t.tw.coord_scale/2-3,y-t.pensize*t.tw.coord_scale/2-3,\
             w+t.pensize*t.tw.coord_scale+6,h+t.pensize*t.tw.coord_scale+6)
    right(t,a)
    t.xcor=cx-r*cos(t.heading*DEGTOR)
    t.ycor=cy+r*sin(t.heading*DEGTOR)

def larc(t,a,r):
    if r<0: r=-r; a=-a
    cx = t.xcor-r*cos(t.heading*DEGTOR)
    cy = t.ycor+r*sin(t.heading*DEGTOR)
    x,y,w,h=t.width/2+int(cx-r),t.height/2-int(cy+r),int(2*r),int(2*r)
    if t.pendown:
        t.canvas.image.draw_arc(t.gc,False,x,y,w,h,int(360-t.heading)*64, \
                                int(a)*64)
    invalt(t,x-t.pensize*t.tw.coord_scale/2-3,y-t.pensize*t.tw.coord_scale/2-3,\
             w+t.pensize*t.tw.coord_scale+6,h+t.pensize*t.tw.coord_scale+6)
    right(t,-a)
    t.xcor=cx+r*cos(t.heading*DEGTOR)
    t.ycor=cy-r*sin(t.heading*DEGTOR)

def setxy(t,x,y):
    x *= t.tw.coord_scale
    y *= t.tw.coord_scale
    try:
        t.xcor,t.ycor = x,y
    except:
        pass
    move_turtle(t)

def setpensize(t,ps):
    try:
        if ps<0:
            ps=0;
        t.pensize = ps
    except:
        pass
    t.gc.set_line_attributes(int(t.pensize*t.tw.coord_scale), \
                             gtk.gdk.LINE_SOLID, \
                             gtk.gdk.CAP_ROUND, gtk.gdk.JOIN_MITER)
    return None

def setcolor(t,c):
    try:
        t.color = c
    except:
        pass
    set_fgcolor(t)
    return None

def settextcolor(t,c):
    try:
        t.tcolor = c
    except:
        pass
    set_textcolor(t)
    return None

def settextsize(t,c):
    try:
        t.tw.textsize = c
    except:
        pass
    return None

def setshade(t,s):
    try:
        t.shade = s
    except:
        pass
    set_fgcolor(t)
    set_textcolor(t)
    return None

def fillscreen(t,c,s):
    oldc, olds = t.color,t.shade
    try:
        setcolor(t,c); setshade(t,s)
        rect = gtk.gdk.Rectangle(0,0,t.width,t.height)
        t.gc.set_foreground(t.tw.fgcolor)
        t.canvas.image.draw_rectangle(t.gc, True, *rect)
        invalt(t,0,0,t.width,t.height)
        setcolor(t,oldc); setshade(t,olds)
    except:
        pass
    return None

def set_fgcolor(t):
    sh = (wrap100(t.shade)-50)/50.0
    rgb = color_table[wrap100(t.color)]
    r,g,b = (rgb>>8)&0xff00,rgb&0xff00,(rgb<<8)&0xff00
    r,g,b = calc_shade(r,sh),calc_shade(g,sh),calc_shade(b,sh)
    t.tw.rgb = [r>>8,g>>8,b>>8]
    t.tw.fgcolor = t.tw.cm.alloc_color(r,g,b)

def set_textcolor(t):
    sh = (wrap100(t.shade)-50)/50.0
    rgb = color_table[wrap100(t.tcolor)]
    r,g,b = (rgb>>8)&0xff00,rgb&0xff00,(rgb<<8)&0xff00
    r,g,b = calc_shade(r,sh),calc_shade(g,sh),calc_shade(b,sh)
    t.tw.textcolor = t.tw.cm.alloc_color(r,g,b)

def wrap100(n):
    n = int(n)
    n %= 200
    if n>99: n=199-n
    return n

def calc_shade(c,s):
    if s<0: return int(c*(1+s*.8))
    return int(c+(65536-c)*s*.9)

def setpen(t,bool):
    t.pendown = bool

def draw_pixbuf(t,pixbuf,a,b,x,y,w,h):
    w *= t.tw.coord_scale
    h *= t.tw.coord_scale
    t.canvas.image.draw_pixbuf(t.gc, pixbuf, a, b, x, y)
    invalt(t,x,y,w,h)

def draw_text(t, label, x, y, size, w):
    w *= t.tw.coord_scale
    t.gc.set_foreground(t.tw.textcolor)
    fd = pango.FontDescription('Sans')
    try:
        fd.set_size(int(size*t.tw.coord_scale)*pango.SCALE)
    except:
        pass
    if type(label) == str or type(label) == unicode:
        pl = t.tw.window.create_pango_layout(label.replace("\0"," "))
    elif type(label) == float or type(label) == int:
        pl = t.tw.window.create_pango_layout(str(label))
    else:
        print type(label)
        pl = t.tw.window.create_pango_layout(str(label))
    pl.set_font_description(fd)
    pl.set_width(int(w)*pango.SCALE)
    t.canvas.image.draw_layout(t.gc,int(x),int(y),pl)
    w,h = pl.get_pixel_size()
    invalt(t,x,y,w,h)

def draw_line(t,x1,y1,x2,y2):
    x1,y1 = t.width/2+int(x1), t.height/2-int(y1)
    x2,y2 = t.width/2+int(x2), t.height/2-int(y2)
    if x1<x2: minx,maxx=x1,x2
    else: minx,maxx=x2,x1
    if y1<y2: miny,maxy=y1,y2
    else: miny,maxy=y2,y1
    w,h=maxx-minx,maxy-miny
    t.canvas.image.draw_line(t.gc,x1,y1,x2,y2)
    invalt(t,minx-t.pensize*t.tw.coord_scale/2-3, \
             miny-t.pensize*t.tw.coord_scale/2-3, \
           w+t.pensize*t.tw.coord_scale+6, \
           h+t.pensize*t.tw.coord_scale+6)

def turn_turtle(t):
    setshape(t.spr, t.shapelist[(int(t.heading+5)%360)/10])

def move_turtle(t):
    x,y = t.width/2+int(t.xcor), t.height/2-int(t.ycor)
    move(t.spr, (t.canvas.x+x-30,t.canvas.y+y-30))
    invalt(t,x-30,y-30,60,60)

def invalt(t,x,y,w,h):
    rect = gtk.gdk.Rectangle(int(x+t.canvas.x),int(y+t.canvas.y), \
                             int(w),int(h))
    t.tw.area.invalidate_rect(rect, False)

<commit_msg>set text color to pen color<commit_after>#Copyright (c) 2007-8, Playful Invention Company.
#Copyright (c) 2008-9, Walter Bender

#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:

#The above copyright notice and this permission notice shall be included in
#all copies or substantial portions of the Software.

#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#THE SOFTWARE.

import gtk
from math import sin,cos,pi
class taTurtle: pass

from tasprites import *
from tasetup import load_image

colors = {}
DEGTOR = 2*pi/360

color_table = (
    0xFF0000,0xFF0D00,0xFF1A00,0xFF2600,0xFF3300,
    0xFF4000,0xFF4D00,0xFF5900,0xFF6600,0xFF7300,
    0xFF8000,0xFF8C00,0xFF9900,0xFFA600,0xFFB300,
    0xFFBF00,0xFFCC00,0xFFD900,0xFFE600,0xFFF200,
    0xFFFF00,0xE6FF00,0xCCFF00,0xB3FF00,0x99FF00,
    0x80FF00,0x66FF00,0x4DFF00,0x33FF00,0x1AFF00,
    0x00FF00,0x00FF0D,0x00FF1A,0x00FF26,0x00FF33,
    0x00FF40,0x00FF4D,0x00FF59,0x00FF66,0x00FF73,
    0x00FF80,0x00FF8C,0x00FF99,0x00FFA6,0x00FFB3,
    0x00FFBF,0x00FFCC,0x00FFD9,0x00FFE6,0x00FFF2,
    0x00FFFF,0x00F2FF,0x00E6FF,0x00D9FF,0x00CCFF,
    0x00BFFF,0x00B3FF,0x00A6FF,0x0099FF,0x008CFF,
    0x0080FF,0x0073FF,0x0066FF,0x0059FF,0x004DFF,
    0x0040FF,0x0033FF,0x0026FF,0x001AFF,0x000DFF,
    0x0000FF,0x0D00FF,0x1A00FF,0x2600FF,0x3300FF,
    0x4000FF,0x4D00FF,0x5900FF,0x6600FF,0x7300FF,
    0x8000FF,0x8C00FF,0x9900FF,0xA600FF,0xB300FF,
    0xBF00FF,0xCC00FF,0xD900FF,0xE600FF,0xF200FF,
    0xFF00FF,0xFF00E6,0xFF00CC,0xFF00B3,0xFF0099,
    0xFF0080,0xFF0066,0xFF004D,0xFF0033,0xFF001A)

def tNew(tw,w,h):
    t = taTurtle()
    t.tw, t.width, t.height = tw, w, h
    t.canvas = sprNew(tw,0,0,gtk.gdk.Pixmap(tw.area,w,h,-1))
    t.canvas.type = 'canvas'
    setlayer(t.canvas,600)
    t.shapelist = \
        [load_image(tw.path, 'shapes','t'+str(i)) for i in range(36)]
    t.spr = sprNew(tw,100,100,t.shapelist[0])
    t.spr.type = 'turtle'
    setlayer(t.spr, 630)
    t.gc = t.canvas.image.new_gc()
    t.shade = 0
    clearscreen(t)
    return t

def clearscreen(t):
    t.xcor, t.ycor, t.heading = 0,0,0
    rect = gtk.gdk.Rectangle(0,0,t.width,t.height)
    t.gc.set_foreground(t.tw.bgcolor)
    t.canvas.image.draw_rectangle(t.gc, True, *rect)
    invalt(t,0,0,t.width,t.height)
    setpensize(t,5)
    setcolor(t,0)
    settextcolor(t,70)
    settextsize(t,32)
    setshade(t,50)
    t.pendown = True
    move_turtle(t)
    turn_turtle(t)
    return None

def forward(t, n):
    n *= t.tw.coord_scale
    t.gc.set_foreground(t.tw.fgcolor)
    oldx, oldy = t.xcor, t.ycor
    try:
        t.xcor += n*sin(t.heading*DEGTOR)
        t.ycor += n*cos(t.heading*DEGTOR)
    except:
        pass
    if t.pendown:
        draw_line(t,oldx,oldy,t.xcor,t.ycor)
    move_turtle(t)
    return None

def seth(t,n):
    try:
        t.heading=n
    except:
        pass
    t.heading%=360
    turn_turtle(t)
    return None

def right(t,n):
    try:
        t.heading+=n
    except:
        pass
    t.heading%=360
    turn_turtle(t)
    return None

def arc(t,a,r):
    t.gc.set_foreground(t.tw.fgcolor)
    r *= t.tw.coord_scale
    try:
        if a<0: larc(t,-a,r)
        else: rarc(t,a,r)
    except:
        pass
    move_turtle(t)
    turn_turtle(t)

def rarc(t,a,r):
    if r<0: r=-r; a=-a
    cx = t.xcor+r*cos(t.heading*DEGTOR)
    cy = t.ycor-r*sin(t.heading*DEGTOR)
    x,y,w,h=t.width/2+int(cx-r),t.height/2-int(cy+r),int(2*r),int(2*r)
    if t.pendown:
        t.canvas.image.draw_arc(t.gc,False,x,y,w,h, \
                                int(180-t.heading-a)*64,int(a)*64)
    invalt(t,x-t.pensize*t.tw.coord_scale/2-3,y-t.pensize*t.tw.coord_scale/2-3,\
             w+t.pensize*t.tw.coord_scale+6,h+t.pensize*t.tw.coord_scale+6)
    right(t,a)
    t.xcor=cx-r*cos(t.heading*DEGTOR)
    t.ycor=cy+r*sin(t.heading*DEGTOR)

def larc(t,a,r):
    if r<0: r=-r; a=-a
    cx = t.xcor-r*cos(t.heading*DEGTOR)
    cy = t.ycor+r*sin(t.heading*DEGTOR)
    x,y,w,h=t.width/2+int(cx-r),t.height/2-int(cy+r),int(2*r),int(2*r)
    if t.pendown:
        t.canvas.image.draw_arc(t.gc,False,x,y,w,h,int(360-t.heading)*64, \
                                int(a)*64)
    invalt(t,x-t.pensize*t.tw.coord_scale/2-3,y-t.pensize*t.tw.coord_scale/2-3,\
             w+t.pensize*t.tw.coord_scale+6,h+t.pensize*t.tw.coord_scale+6)
    right(t,-a)
    t.xcor=cx+r*cos(t.heading*DEGTOR)
    t.ycor=cy-r*sin(t.heading*DEGTOR)

def setxy(t,x,y):
    x *= t.tw.coord_scale
    y *= t.tw.coord_scale
    try:
        t.xcor,t.ycor = x,y
    except:
        pass
    move_turtle(t)

def setpensize(t,ps):
    try:
        if ps<0:
            ps=0;
        t.pensize = ps
    except:
        pass
    t.gc.set_line_attributes(int(t.pensize*t.tw.coord_scale), \
                             gtk.gdk.LINE_SOLID, \
                             gtk.gdk.CAP_ROUND, gtk.gdk.JOIN_MITER)
    return None

def setcolor(t,c):
    try:
        t.color = c
        t.tcolor = c # using pen color for text color
    except:
        pass
    set_fgcolor(t)
    set_textcolor(t)
    return None

def settextcolor(t,c):
    try:
        t.tcolor = c
    except:
        pass
    set_textcolor(t)
    return None

def settextsize(t,c):
    try:
        t.tw.textsize = c
    except:
        pass
    return None

def setshade(t,s):
    try:
        t.shade = s
    except:
        pass
    set_fgcolor(t)
    set_textcolor(t)
    return None

def fillscreen(t,c,s):
    oldc, olds = t.color,t.shade
    try:
        setcolor(t,c); setshade(t,s)
        rect = gtk.gdk.Rectangle(0,0,t.width,t.height)
        t.gc.set_foreground(t.tw.fgcolor)
        t.canvas.image.draw_rectangle(t.gc, True, *rect)
        invalt(t,0,0,t.width,t.height)
        setcolor(t,oldc); setshade(t,olds)
    except:
        pass
    return None

def set_fgcolor(t):
    sh = (wrap100(t.shade)-50)/50.0
    rgb = color_table[wrap100(t.color)]
    r,g,b = (rgb>>8)&0xff00,rgb&0xff00,(rgb<<8)&0xff00
    r,g,b = calc_shade(r,sh),calc_shade(g,sh),calc_shade(b,sh)
    t.tw.rgb = [r>>8,g>>8,b>>8]
    t.tw.fgcolor = t.tw.cm.alloc_color(r,g,b)

def set_textcolor(t):
    sh = (wrap100(t.shade)-50)/50.0
    rgb = color_table[wrap100(t.tcolor)]
    r,g,b = (rgb>>8)&0xff00,rgb&0xff00,(rgb<<8)&0xff00
    r,g,b = calc_shade(r,sh),calc_shade(g,sh),calc_shade(b,sh)
    t.tw.textcolor = t.tw.cm.alloc_color(r,g,b)

def wrap100(n):
    n = int(n)
    n %= 200
    if n>99: n=199-n
    return n

def calc_shade(c,s):
    if s<0: return int(c*(1+s*.8))
    return int(c+(65536-c)*s*.9)

def setpen(t,bool):
    t.pendown = bool

def draw_pixbuf(t,pixbuf,a,b,x,y,w,h):
    w *= t.tw.coord_scale
    h *= t.tw.coord_scale
    t.canvas.image.draw_pixbuf(t.gc, pixbuf, a, b, x, y)
    invalt(t,x,y,w,h)

def draw_text(t, label, x, y, size, w):
    w *= t.tw.coord_scale
    t.gc.set_foreground(t.tw.textcolor)
    fd = pango.FontDescription('Sans')
    try:
        fd.set_size(int(size*t.tw.coord_scale)*pango.SCALE)
    except:
        pass
    if type(label) == str or type(label) == unicode:
        pl = t.tw.window.create_pango_layout(label.replace("\0"," "))
    elif type(label) == float or type(label) == int:
        pl = t.tw.window.create_pango_layout(str(label))
    else:
        print type(label)
        pl = t.tw.window.create_pango_layout(str(label))
    pl.set_font_description(fd)
    pl.set_width(int(w)*pango.SCALE)
    t.canvas.image.draw_layout(t.gc,int(x),int(y),pl)
    w,h = pl.get_pixel_size()
    invalt(t,x,y,w,h)

def draw_line(t,x1,y1,x2,y2):
    x1,y1 = t.width/2+int(x1), t.height/2-int(y1)
    x2,y2 = t.width/2+int(x2), t.height/2-int(y2)
    if x1<x2: minx,maxx=x1,x2
    else: minx,maxx=x2,x1
    if y1<y2: miny,maxy=y1,y2
    else: miny,maxy=y2,y1
    w,h=maxx-minx,maxy-miny
    t.canvas.image.draw_line(t.gc,x1,y1,x2,y2)
    invalt(t,minx-t.pensize*t.tw.coord_scale/2-3, \
             miny-t.pensize*t.tw.coord_scale/2-3, \
           w+t.pensize*t.tw.coord_scale+6, \
           h+t.pensize*t.tw.coord_scale+6)

def turn_turtle(t):
    setshape(t.spr, t.shapelist[(int(t.heading+5)%360)/10])

def move_turtle(t):
    x,y = t.width/2+int(t.xcor), t.height/2-int(t.ycor)
    move(t.spr, (t.canvas.x+x-30,t.canvas.y+y-30))
    invalt(t,x-30,y-30,60,60)

def invalt(t,x,y,w,h):
    rect = gtk.gdk.Rectangle(int(x+t.canvas.x),int(y+t.canvas.y), \
                             int(w),int(h))
    t.tw.area.invalidate_rect(rect, False)

<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product_dict(
    [
        {'shape': (3, 4)},
        {'shape': ()},
    ],
    [
        {'in_type': numpy.float16},
        {'in_type': numpy.float32},
        {'in_type': numpy.float64},
    ],
    [
        {'out_type': numpy.float16},
        {'out_type': numpy.float32},
        {'out_type': numpy.float64},
    ]
))
class TestCast(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.in_type)
        self.g = numpy.random.uniform(-1, 1, self.shape).astype(self.out_type)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.cast(x, self.out_type)
        self.assertEqual(y.data.shape, x.data.shape)
        self.assertEqual(y.data.dtype, self.out_type)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, g_data):
        def func(x):
            return functions.cast(x, self.out_type)

        gradient_check.check_backward(
            func, x_data, g_data, dtype='d',
            eps=2.0 ** -2, atol=1e-2, rtol=1e-3)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.g)

class TestNoCast(unittest.TestCase):

    def setUp(self):
        self.dtype = numpy.float32
        self.x = numpy.empty(1, self.dtype)

    def check_forward_no_cast(self, x_data):
        y = functions.cast(x_data, self.dtype)
        self.assertIsInstance(y, chainer.Variable)
        self.assertIs(y.data, x_data)

    def test_forward_no_cast_array(self):
        y = functions.cast(self.x, self.dtype)
        self.assertIsInstance(y, chainer.Variable)
        self.assertIs(y.data, self.x)

    def test_forward_no_cast_variable(self):
        x = chainer.Variable(self.x)
        y = functions.cast(x, self.dtype)
        self.assertIs(y, x)

testing.run_module(__name__, __file__)
<commit_msg>Replace F.cast test asserts<commit_after>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product_dict(
    [
        {'shape': (3, 4)},
        {'shape': ()},
    ],
    [
        {'in_type': numpy.float16},
        {'in_type': numpy.float32},
        {'in_type': numpy.float64},
    ],
    [
        {'out_type': numpy.float16},
        {'out_type': numpy.float32},
        {'out_type': numpy.float64},
    ]
))
class TestCast(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.in_type)
        self.g = numpy.random.uniform(-1, 1, self.shape).astype(self.out_type)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.cast(x, self.out_type)
        assert y.data.shape == x.data.shape
        assert y.data.dtype == self.out_type

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, g_data):
        def func(x):
            return functions.cast(x, self.out_type)

        gradient_check.check_backward(
            func, x_data, g_data, dtype='d',
            eps=2.0 ** -2, atol=1e-2, rtol=1e-3)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.g)

class TestNoCast(unittest.TestCase):

    def setUp(self):
        self.dtype = numpy.float32
        self.x = numpy.empty(1, self.dtype)

    def check_forward_no_cast(self, x_data):
        y = functions.cast(x_data, self.dtype)
        assert isinstance(y, chainer.Variable)
        assert y.data is x_data

    def test_forward_no_cast_array(self):
        y = functions.cast(self.x, self.dtype)
        assert isinstance(y, chainer.Variable)
        assert y.data is self.x

    def test_forward_no_cast_variable(self):
        x = chainer.Variable(self.x)
        y = functions.cast(x, self.dtype)
        assert y is x

testing.run_module(__name__, __file__)
<|endoftext|>