<commit_before># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
import json
from default import with_context
from nose.tools import assert_equal, assert_raises
from test_api import TestAPI
from pybossa.core import project_repo

from factories import ProjectFactory, TaskFactory, TaskRunFactory, UserFactory

class TestApiCommon(TestAPI):

    def setUp(self):
        super(TestApiCommon, self).setUp()

    @with_context
    def test_limits_query(self):
        """Test API GET limits works"""
        owner = UserFactory.create()
        projects = ProjectFactory.create_batch(30, owner=owner)
        for project in projects:
            task = TaskFactory.create(project=project)
            TaskRunFactory.create(task=task)

        res = self.app.get('/api/project')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/project?limit=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)

        # DEPRECATED
        res = self.app.get('/api/project?limit=10&offset=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == projects[10].name, data[0]

        # Keyset pagination
        url = '/api/project?limit=10&last_id=%s' % projects[9].id
        res = self.app.get(url)
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == projects[10].name, data[0]

        res = self.app.get('/api/task')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/taskrun')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        UserFactory.create_batch(30)

        res = self.app.get('/api/user')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/user?limit=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)

        # DEPRECATED
        res = self.app.get('/api/user?limit=10&offset=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == 'user11', data

        res = self.app.get('/api/user?limit=10&last_id=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == 'user11', data

    @with_context
    def test_get_query_with_api_key_and_all(self):
        """ Test API GET query with an API-KEY requesting all results"""
        admin, owner, user = UserFactory.create_batch(3)
        project = ProjectFactory.create(owner=owner, info={'total': 150, 'onesignal': 'something', 'onesignal_app_id': 1}, published=True)
        task = TaskFactory.create(project=project, info={'url': 'my url'})
        taskrun = TaskRunFactory.create(task=task, user=admin,
                                        info={'answer': 'annakarenina'})

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + user.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' not in project_res['info'].keys(), data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

        tmp = project_repo.get(project.id)

        assert tmp.id == project.id, tmp
        assert tmp.info['total'] == 150, tmp

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + owner.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' in project_res['info'].keys(), data
                assert project_res['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + admin.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' in project_res['info'].keys(), data
                assert project_res['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

    @with_context
    def test_get_query_with_api_key_context(self):
        """ Test API GET query with an API-KEY requesting only APIKEY results."""
        users = UserFactory.create_batch(4)
        project_oc = ProjectFactory.create(owner=users[0], info={'total': 150})
        projects = ProjectFactory.create_batch(3, owner=users[1])
        task_oc = TaskFactory.create(project=project_oc, info={'url': 'my url'})
        taskrun_oc = TaskRunFactory.create(task=task_oc, user=users[0],
                                        info={'answer': 'annakarenina'})
        for p in projects:
            task_tmp = TaskFactory.create(project=p)
            TaskRunFactory.create(task=task_tmp)

        # For project owner with associated data
        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + users[0].api_key
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project = data[0]
                assert project['owner_id'] == users[0].id, project['owner_id']
                assert project['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['project_id'] == project_oc.id, task
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['project_id'] == project_oc.id, taskrun
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

        # For authenticated with non-associated data
        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + users[3].api_key

            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

    @with_context
    def test_query_search_wrongfield(self):
        """ Test API query search works"""
        # Test first a non-existant field for all end-points
        for endpoint in self.endpoints:
            res = self.app.get("/api/%s?wrongfield=value" % endpoint)
            err = json.loads(res.data)
            assert res.status_code == 415, err
            assert err['status'] == 'failed', err
            assert err['action'] == 'GET', err
            assert err['exception_cls'] == 'AttributeError', err

    @with_context
    def test_query_search_fulltext(self):
        """ Test API query search fulltext works"""
        # Test first a non-existant field for all end-points
        TaskFactory.create(info={'foo': 'fox'})
        TaskFactory.create(info={'foo': 'foxes something'})
        res = self.app.get('/api/task?all=1&info=foo::fox&fulltextsearch=1')
        data = json.loads(res.data)
        assert len(data) == 2, res.data
        for d in data:
            assert 'fox' in d['info']['foo']
            assert 'rank' in d.keys()
            assert 'headline' in d.keys()

        # Without the fulltextsearch
        res = self.app.get('/api/task?all=1&info=foo::fox')
        data = json.loads(res.data)
        assert len(data) == 1, res.data
        for d in data:
            assert 'fox' in d['info']['foo']
            assert 'rank' not in d.keys()
            assert 'headline'  not in d.keys()

    @with_context
    def test_query_sql_injection(self):
        """Test API SQL Injection is not allowed works"""

        q = '1%3D1;SELECT%20*%20FROM%20task%20WHERE%201=1'
        res = self.app.get('/api/task?' + q)
        error = json.loads(res.data)
        assert res.status_code == 415, error
        assert error['action'] == 'GET', error
        assert error['status'] == 'failed', error
        assert error['target'] == 'task', error

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api/apappp?' + q)
        assert res.status_code == 404, res.data

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api/' + q)
        assert res.status_code == 404, res.data

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api' + q)
        assert res.status_code == 404, res.data

    @with_context
    def test_jsonpify(self):
        """Test API jsonpify decorator works."""
        project = ProjectFactory.create()
        res = self.app.get('/api/project/%s?callback=mycallback' % project.id)
        err_msg = "mycallback should be included in the response"
        print res.data
        assert "mycallback" in res.data, err_msg
        err_msg = "Status code should be 200"
        assert res.status_code == 200, err_msg

    def test_cors(self):
        """Test CORS decorator works."""
        res = self.app.options('/api/project/1',
                           headers={'Access-Control-Request-Method': 'GET',
                                    'Access-Control-Request-Headers': 'Authorization',
                           })
        err_msg = "CORS should be enabled"
        assert res.headers['Access-Control-Allow-Origin'] == '*', err_msg
        methods = ['PUT', 'HEAD', 'DELETE', 'OPTIONS', 'GET']
        for m in methods:
            err_msg = "Access-Control-Allow-Methods: %s is missing" % m
            assert m in res.headers['Access-Control-Allow-Methods'], err_msg
        assert res.headers['Access-Control-Max-Age'] == '21600', err_msg
        test_headers = ['Content-Type', 'Authorization']
        for header in test_headers:
            err_msg = "Access-Control-Allow-Headers: %s is missing" % header
            headers={'Access-Control-Request-Method': 'GET',
                     'Access-Control-Request-Headers': header}
            res = self.app.options('/api/project/1', headers=headers)
            assert res.headers['Access-Control-Allow-Headers'] == header, err_msg
<commit_msg>Tests.<commit_after># -*- coding: utf8 -*-
# This file is part of PYBOSSA.
#
# Copyright (C) 2015 Scifabric LTD.
#
# PYBOSSA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PYBOSSA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PYBOSSA.  If not, see <http://www.gnu.org/licenses/>.
import json
from default import with_context
from nose.tools import assert_equal, assert_raises
from test_api import TestAPI
from pybossa.core import project_repo

from factories import ProjectFactory, TaskFactory, TaskRunFactory, UserFactory

class TestApiCommon(TestAPI):

    def setUp(self):
        super(TestApiCommon, self).setUp()

    @with_context
    def test_limits_query(self):
        """Test API GET limits works"""
        owner = UserFactory.create()
        projects = ProjectFactory.create_batch(30, owner=owner)
        project_created = ProjectFactory.create(created='2000-01-01T12:08:47.134025')
        for project in projects:
            task = TaskFactory.create(project=project)
            TaskRunFactory.create(task=task)

        res = self.app.get('/api/project')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/project?limit=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)

        # DEPRECATED
        res = self.app.get('/api/project?limit=10&offset=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == projects[10].name, data[0]

        # Keyset pagination
        url = '/api/project?limit=10&last_id=%s' % projects[9].id
        res = self.app.get(url)
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == projects[10].name, data[0]

        res = self.app.get('/api/task')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/taskrun')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        UserFactory.create_batch(30)

        res = self.app.get('/api/user')
        data = json.loads(res.data)
        assert len(data) == 20, len(data)

        res = self.app.get('/api/user?limit=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)

        # DEPRECATED
        res = self.app.get('/api/user?limit=10&offset=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == 'user11', data

        res = self.app.get('/api/user?limit=10&last_id=10')
        data = json.loads(res.data)
        assert len(data) == 10, len(data)
        assert data[0].get('name') == 'user11', data

        # By date created
        res = self.app.get('/api/project?created=2000-01')
        data = json.loads(res.data)
        assert len(data) == 1, len(data)
        assert data[0].get('id') == project_created.id

    @with_context
    def test_get_query_with_api_key_and_all(self):
        """ Test API GET query with an API-KEY requesting all results"""
        admin, owner, user = UserFactory.create_batch(3)
        project = ProjectFactory.create(owner=owner, info={'total': 150, 'onesignal': 'something', 'onesignal_app_id': 1}, published=True)
        task = TaskFactory.create(project=project, info={'url': 'my url'})
        taskrun = TaskRunFactory.create(task=task, user=admin,
                                        info={'answer': 'annakarenina'})

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + user.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' not in project_res['info'].keys(), data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

        tmp = project_repo.get(project.id)

        assert tmp.id == project.id, tmp
        assert tmp.info['total'] == 150, tmp

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + owner.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' in project_res['info'].keys(), data
                assert project_res['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + admin.api_key + '&all=1'
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project_res = data[0]
                assert 'total' in project_res['info'].keys(), data
                assert project_res['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'user':
                assert len(data) == 3, data
                user_res = data[0]
                assert user_res['name'] == 'user1', data
                assert res.mimetype == 'application/json', res

    @with_context
    def test_get_query_with_api_key_context(self):
        """ Test API GET query with an API-KEY requesting only APIKEY results."""
        users = UserFactory.create_batch(4)
        project_oc = ProjectFactory.create(owner=users[0], info={'total': 150})
        projects = ProjectFactory.create_batch(3, owner=users[1])
        task_oc = TaskFactory.create(project=project_oc, info={'url': 'my url'})
        taskrun_oc = TaskRunFactory.create(task=task_oc, user=users[0],
                                        info={'answer': 'annakarenina'})
        for p in projects:
            task_tmp = TaskFactory.create(project=p)
            TaskRunFactory.create(task=task_tmp)

        # For project owner with associated data
        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + users[0].api_key
            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 1, data
                project = data[0]
                assert project['owner_id'] == users[0].id, project['owner_id']
                assert project['info']['total'] == 150, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 1, data
                task = data[0]
                assert task['project_id'] == project_oc.id, task
                assert task['info']['url'] == 'my url', data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 1, data
                taskrun = data[0]
                assert taskrun['project_id'] == project_oc.id, taskrun
                assert taskrun['info']['answer'] == 'annakarenina', data
                assert res.mimetype == 'application/json', res

        # For authenticated with non-associated data
        for endpoint in self.endpoints:
            url = '/api/' + endpoint + '?api_key=' + users[3].api_key

            res = self.app.get(url)
            data = json.loads(res.data)

            if endpoint == 'project':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'task':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

            if endpoint == 'taskrun':
                assert len(data) == 0, data
                assert res.mimetype == 'application/json', res

    @with_context
    def test_query_search_wrongfield(self):
        """ Test API query search works"""
        # Test first a non-existant field for all end-points
        for endpoint in self.endpoints:
            res = self.app.get("/api/%s?wrongfield=value" % endpoint)
            err = json.loads(res.data)
            assert res.status_code == 415, err
            assert err['status'] == 'failed', err
            assert err['action'] == 'GET', err
            assert err['exception_cls'] == 'AttributeError', err

    @with_context
    def test_query_search_fulltext(self):
        """ Test API query search fulltext works"""
        # Test first a non-existant field for all end-points
        TaskFactory.create(info={'foo': 'fox'})
        TaskFactory.create(info={'foo': 'foxes something'})
        res = self.app.get('/api/task?all=1&info=foo::fox&fulltextsearch=1')
        data = json.loads(res.data)
        assert len(data) == 2, res.data
        for d in data:
            assert 'fox' in d['info']['foo']
            assert 'rank' in d.keys()
            assert 'headline' in d.keys()

        # Without the fulltextsearch
        res = self.app.get('/api/task?all=1&info=foo::fox')
        data = json.loads(res.data)
        assert len(data) == 1, res.data
        for d in data:
            assert 'fox' in d['info']['foo']
            assert 'rank' not in d.keys()
            assert 'headline'  not in d.keys()

    @with_context
    def test_query_sql_injection(self):
        """Test API SQL Injection is not allowed works"""

        q = '1%3D1;SELECT%20*%20FROM%20task%20WHERE%201=1'
        res = self.app.get('/api/task?' + q)
        error = json.loads(res.data)
        assert res.status_code == 415, error
        assert error['action'] == 'GET', error
        assert error['status'] == 'failed', error
        assert error['target'] == 'task', error

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api/apappp?' + q)
        assert res.status_code == 404, res.data

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api/' + q)
        assert res.status_code == 404, res.data

        q = 'project_id=1%3D1;SELECT%20*%20FROM%20task%20WHERE%201'
        res = self.app.get('/api' + q)
        assert res.status_code == 404, res.data

    @with_context
    def test_jsonpify(self):
        """Test API jsonpify decorator works."""
        project = ProjectFactory.create()
        res = self.app.get('/api/project/%s?callback=mycallback' % project.id)
        err_msg = "mycallback should be included in the response"
        print res.data
        assert "mycallback" in res.data, err_msg
        err_msg = "Status code should be 200"
        assert res.status_code == 200, err_msg

    def test_cors(self):
        """Test CORS decorator works."""
        res = self.app.options('/api/project/1',
                           headers={'Access-Control-Request-Method': 'GET',
                                    'Access-Control-Request-Headers': 'Authorization',
                           })
        err_msg = "CORS should be enabled"
        assert res.headers['Access-Control-Allow-Origin'] == '*', err_msg
        methods = ['PUT', 'HEAD', 'DELETE', 'OPTIONS', 'GET']
        for m in methods:
            err_msg = "Access-Control-Allow-Methods: %s is missing" % m
            assert m in res.headers['Access-Control-Allow-Methods'], err_msg
        assert res.headers['Access-Control-Max-Age'] == '21600', err_msg
        test_headers = ['Content-Type', 'Authorization']
        for header in test_headers:
            err_msg = "Access-Control-Allow-Headers: %s is missing" % header
            headers={'Access-Control-Request-Method': 'GET',
                     'Access-Control-Request-Headers': header}
            res = self.app.options('/api/project/1', headers=headers)
            assert res.headers['Access-Control-Allow-Headers'] == header, err_msg
<|endoftext|>


<commit_before>import sys
import pytest
import tempfile
import os
from ectyper import ectyper
import subprocess
import pandas
import logging
import re

TEST_ROOT = os.path.dirname(__file__)
logging.basicConfig(level=logging.INFO)
LOG=logging.getLogger("TEST")

def set_input(input,
              percent_iden=None,
              verify = True,
              output=tempfile.mkdtemp(),
              cores=1,
              print_sequence=False):
    """
    Create the sys.argv[] without need for commandline input.
    :param input: Input file given by testing function
    :param percent_iden: Percent identity for comparison
    :param output: Location of output
    :return: None
    """
    args = ['-i', input,
            #'-r', os.path.join(TEST_ROOT, 'Data/test_sketch.msh'),
            '-c', str(cores)]

    if percent_iden:
        args += ['-d', str(percent_iden)]
    if verify:
        args += ['--verify']
    if output:
        args += ['-o', output]
    if print_sequence:
        args += ['--sequence']
    sys.argv[1:] = args

def test_integration_invalid_file(caplog):
    """
    Giving a non-fasta file in fasta-file name.
    :return: None
    """
    caplog.set_level(logging.DEBUG)
    file = os.path.join(TEST_ROOT, 'Data/test_dir/badfasta.fasta')
    set_input(input=file)
    ectyper.run_program()
    assert "Non fasta / fastq file" in caplog.text

def test_integration_no_file():
    """
    Giving no input to the program.
    :return: None
    """
    file = ''
    set_input(file)
    with pytest.raises(SystemExit) as se:
        ectyper.run_program()
    assert se.type == SystemExit
    assert se.value.code == "No files were found"

def test_integration_valid_file(caplog):
    """
    Ensure a valid E. coli fasta passes
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fna')
    set_input(file)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli\tO103\tH2\tO103:H2\tPASS\tHIGH" in caplog.text

def test_integration_yersinia(caplog):
    """
    Ensure a non-E. coli gets categorized as such
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Yersinia.fasta')
    set_input(file)
    ectyper.run_program()
    assert "Yersinia pestis" in caplog.text

def test_integration_validfasta_noverify(caplog):
    """
    Tests for fasta files without E.coli species verify function (--verify) do not fail as per issue #76 (https://github.com/phac-nml/ecoli_serotyping/issues/76)
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fna')
    set_input(file, verify=False)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli O103:H2 str. 12009\tO103\tH2\tO103:H2" in caplog.text

def test_valid_fastq_file(caplog):
    """
    Given a valid fastq file, get the correct results.
    Use a temp dir for the test output
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fastq')
    set_input(file)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli\tO22\tH8" in caplog.text

def test_multiple_directories(caplog):
    """
    Check a number of small files, some good, some bad,
    within a nested directory structure.

    :param caplog: Capture logging output for pytest
    :return: None
    """
    the_dir = os.path.join(TEST_ROOT, 'Data/test_dir')
    set_input(the_dir, cores=4, print_sequence=True)
    ectyper.run_program()
    assert "sample2\tEscherichia coli\tO148\tH44" in caplog.text
    assert "sample3\tEscherichia coli\tO148\tH44" in caplog.text
    assert "sample4\tEscherichia coli\tO148\tH44" in caplog.text
    assert "badfasta\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "sample.fasta\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "sampletar\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "test_junk\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text

def test_mash_sketch_and_assembly_metadata():
    """
    Test if all accessions in mash sketch are a complete subset of the assembly stats superset.
    Ensure that all accession numbers are represented in the meta data assembly stats
    """
    ectyper.speciesIdentification.get_refseq_mash()
    ROOT_DIR = os.path.abspath(os.path.join(TEST_ROOT, '..'))
    MASHSTATSMETAFILE=os.path.join(TEST_ROOT+"/tmp/mash_refseq_meta.txt")
    MASHINFILE = os.path.join(ROOT_DIR, 'ectyper/Data/refseq.genomes.k21s1000.msh')
    ASSEMBLYREFSEQMETAFILE = os.path.join(ROOT_DIR, 'ectyper/Data/assembly_summary_refseq.txt')

    cmd = ["mash info -t " +  MASHINFILE   + " > " + MASHSTATSMETAFILE]
    print("File written to {}".format(MASHSTATSMETAFILE))
    subprocess.run(cmd, shell=True)
    mashsketchdatadf = pandas.read_csv(MASHSTATSMETAFILE,sep="\t")
    mashaccessions=[re.findall("(GCF_\d+)\.+",item)[0] for item in mashsketchdatadf.iloc[:,2].values.tolist()]
    LOG.info("Extracted {} MASH RefSeq accessions".format(len(mashaccessions)))

    genomeassemblystatrefseqsdf = pandas.read_csv(ASSEMBLYREFSEQMETAFILE, sep="\t", skiprows=1)
    #print(genomeassemblystatrefseqsdf.iloc[:, 0].values.tolist()[0:10])
    metaaccessionsrefseq = [re.findall("(GCF_\d+)\.+",item)[0]  for item  in genomeassemblystatrefseqsdf.iloc[:, 0].values.tolist()]
    #print(metaaccessionsrefseq[0:10])
    metaaccessionsrefseqdict = dict.fromkeys(metaaccessionsrefseq, True)

    LOG.info("Extracted {} assembly meta data accessions".format(len(metaaccessionsrefseq)))

    notfoundaccessions = []
    for accession in mashaccessions:
        if not metaaccessionsrefseqdict.get(accession):
            notfoundaccessions.append(accession)

    LOG.info("Not found accessions in meta data {}".format(len(notfoundaccessions)))
    print(notfoundaccessions)

    #genomeassemblystatsdf.loc[1:2, ["# assembly_accession",'bioproject', 'biosample',"wgs_master",
    # "refseq_category","taxid","species_taxid", "organism_name","ftp_path"]]

    # assembly_accession    bioproject      biosample       wgs_master      refseq_category taxid   species_taxid   organism_name   infraspecific_name      isolate version_status  assembly_level  release_type    genome_rep      seq_rel_date    asm_name        submitter       gbrs_paired_asm paired_asm_comp ftp_path        excluded_from_refseq    relation_to_type_material<commit_msg>test_mash_sketch_and_assembly_metadata() was corrected to successfully pass the mash sketch accession numbers extraction test.Try #2<commit_after>import sys
import pytest
import tempfile
import os
from ectyper import ectyper
import subprocess
import pandas
import logging
import re

TEST_ROOT = os.path.dirname(__file__)
logging.basicConfig(level=logging.INFO)
LOG=logging.getLogger("TEST")

def set_input(input,
              percent_iden=None,
              verify = True,
              output=tempfile.mkdtemp(),
              cores=1,
              print_sequence=False):
    """
    Create the sys.argv[] without need for commandline input.
    :param input: Input file given by testing function
    :param percent_iden: Percent identity for comparison
    :param output: Location of output
    :return: None
    """
    args = ['-i', input,
            #'-r', os.path.join(TEST_ROOT, 'Data/test_sketch.msh'),
            '-c', str(cores)]

    if percent_iden:
        args += ['-d', str(percent_iden)]
    if verify:
        args += ['--verify']
    if output:
        args += ['-o', output]
    if print_sequence:
        args += ['--sequence']
    sys.argv[1:] = args

def test_integration_invalid_file(caplog):
    """
    Giving a non-fasta file in fasta-file name.
    :return: None
    """
    caplog.set_level(logging.DEBUG)
    file = os.path.join(TEST_ROOT, 'Data/test_dir/badfasta.fasta')
    set_input(input=file)
    ectyper.run_program()
    assert "Non fasta / fastq file" in caplog.text

def test_integration_no_file():
    """
    Giving no input to the program.
    :return: None
    """
    file = ''
    set_input(file)
    with pytest.raises(SystemExit) as se:
        ectyper.run_program()
    assert se.type == SystemExit
    assert se.value.code == "No files were found"

def test_integration_valid_file(caplog):
    """
    Ensure a valid E. coli fasta passes
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fna')
    set_input(file)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli\tO103\tH2\tO103:H2\tPASS\tHIGH" in caplog.text

def test_integration_yersinia(caplog):
    """
    Ensure a non-E. coli gets categorized as such
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Yersinia.fasta')
    set_input(file)
    ectyper.run_program()
    assert "Yersinia pestis" in caplog.text

def test_integration_validfasta_noverify(caplog):
    """
    Tests for fasta files without E.coli species verify function (--verify) do not fail as per issue #76 (https://github.com/phac-nml/ecoli_serotyping/issues/76)
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fna')
    set_input(file, verify=False)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli O103:H2 str. 12009\tO103\tH2\tO103:H2" in caplog.text

def test_valid_fastq_file(caplog):
    """
    Given a valid fastq file, get the correct results.
    Use a temp dir for the test output
    :return: None
    """
    file = os.path.join(TEST_ROOT, 'Data/Escherichia.fastq')
    set_input(file)
    ectyper.run_program()
    assert "Escherichia\tEscherichia coli\tO22\tH8" in caplog.text

def test_multiple_directories(caplog):
    """
    Check a number of small files, some good, some bad,
    within a nested directory structure.

    :param caplog: Capture logging output for pytest
    :return: None
    """
    the_dir = os.path.join(TEST_ROOT, 'Data/test_dir')
    set_input(the_dir, cores=4, print_sequence=True)
    ectyper.run_program()
    assert "sample2\tEscherichia coli\tO148\tH44" in caplog.text
    assert "sample3\tEscherichia coli\tO148\tH44" in caplog.text
    assert "sample4\tEscherichia coli\tO148\tH44" in caplog.text
    assert "badfasta\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "sample.fasta\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "sampletar\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text
    assert "test_junk\t-\t-\t-\t-:-\t-\t-\t-\t-\tNon fasta / fastq file" in caplog.text

def test_mash_sketch_and_assembly_metadata():
    """
    Test if all accessions in mash sketch are a complete subset of the assembly stats superset.
    Ensure that all accession numbers are represented in the meta data assembly stats
    """
    ectyper.speciesIdentification.get_refseq_mash()
    ROOT_DIR = os.path.abspath(os.path.join(TEST_ROOT, '..'))
    MASHSTATSMETAFILE=os.path.join(TEST_ROOT+"/mash_refseq_meta.txt")
    MASHINFILE = os.path.join(ROOT_DIR, 'ectyper/Data/refseq.genomes.k21s1000.msh')
    ASSEMBLYREFSEQMETAFILE = os.path.join(ROOT_DIR, 'ectyper/Data/assembly_summary_refseq.txt')

    cmd = ["mash info -t " +  MASHINFILE   + " > " + MASHSTATSMETAFILE]
    print("File written to {}".format(MASHSTATSMETAFILE))
    subprocess.run(cmd, shell=True)
    mashsketchdatadf = pandas.read_csv(MASHSTATSMETAFILE,sep="\t")
    mashaccessions=[re.findall("(GCF_\d+)\.+",item)[0] for item in mashsketchdatadf.iloc[:,2].values.tolist()]
    LOG.info("Extracted {} MASH RefSeq accessions".format(len(mashaccessions)))

    genomeassemblystatrefseqsdf = pandas.read_csv(ASSEMBLYREFSEQMETAFILE, sep="\t", skiprows=1)
    #print(genomeassemblystatrefseqsdf.iloc[:, 0].values.tolist()[0:10])
    metaaccessionsrefseq = [re.findall("(GCF_\d+)\.+",item)[0]  for item  in genomeassemblystatrefseqsdf.iloc[:, 0].values.tolist()]
    #print(metaaccessionsrefseq[0:10])
    metaaccessionsrefseqdict = dict.fromkeys(metaaccessionsrefseq, True)

    LOG.info("Extracted {} assembly meta data accessions".format(len(metaaccessionsrefseq)))

    notfoundaccessions = []
    for accession in mashaccessions:
        if not metaaccessionsrefseqdict.get(accession):
            notfoundaccessions.append(accession)

    LOG.info("Not found accessions in meta data {}".format(len(notfoundaccessions)))
    print(notfoundaccessions)

    #genomeassemblystatsdf.loc[1:2, ["# assembly_accession",'bioproject', 'biosample',"wgs_master",
    # "refseq_category","taxid","species_taxid", "organism_name","ftp_path"]]

    # assembly_accession    bioproject      biosample       wgs_master      refseq_category taxid   species_taxid   organism_name   infraspecific_name      isolate version_status  assembly_level  release_type    genome_rep      seq_rel_date    asm_name        submitter       gbrs_paired_asm paired_asm_comp ftp_path        excluded_from_refseq    relation_to_type_material<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''\
usage: venv-update [-h] [virtualenv_dir] [requirements [requirements ...]]

Update a (possibly non-existant) virtualenv directory using a requirements.txt listing
When this script completes, the virtualenv should have the same packages as if it were
removed, then rebuilt.

To set the index server, export a PIP_INDEX_URL variable.
    See also: https://pip.readthedocs.org/en/stable/user_guide/#environment-variables

positional arguments:
  virtualenv_dir  Destination virtualenv directory (default: virtualenv_run)
  requirements    Requirements files. (default: requirements.txt)

optional arguments:
  -h, --help      show this help message and exit

Version control at: https://github.com/yelp/pip-faster
'''
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from os.path import exists

VENV_UPDATE_REQS_OVERRIDE = 'requirements.d/venv-update.txt'
__version__ = '1.0.dev2'

# This script must not rely on anything other than
#   stdlib>=2.6 and virtualenv>1.11

def parseargs(args):
    if set(args) & set(('-h', '--help')):
        print(__doc__, end='')
        exit(0)

    args = list(args)
    virtualenv_dir = None
    requirements = []
    remaining = []

    for arg in args:
        if arg.startswith('-'):
            remaining.append(arg)
        elif virtualenv_dir is None:
            virtualenv_dir = arg
        else:
            requirements.append(arg)

    if not virtualenv_dir:
        virtualenv_dir = 'virtualenv_run'
    if not requirements:
        requirements = ['requirements.txt']

    return virtualenv_dir, tuple(requirements), tuple(remaining)

def timid_relpath(arg):
    from os.path import isabs, relpath
    if isabs(arg):
        result = relpath(arg)
        if len(result) < len(arg):
            return result

    return arg

def shellescape(args):
    from pipes import quote
    return ' '.join(quote(timid_relpath(arg)) for arg in args)

def colorize(cmd):
    from os import isatty

    if isatty(1):
        template = '\033[01;36m>\033[m \033[01;32m{0}\033[m'
    else:
        template = '> {0}'

    return template.format(shellescape(cmd))

def run(cmd):
    from subprocess import check_call
    check_call(('echo', colorize(cmd)))
    check_call(cmd)

def info(msg):
    # use a subprocess to ensure correct output interleaving.
    from subprocess import check_call
    check_call(('echo', msg))

def validate_venv(venv_path, venv_args):
    """Ensure we have a valid virtualenv."""
    import json
    from sys import executable, version
    # we count any existing virtualenv invalidated if any of these relevant values changes
    validation = (
        version,  # includes e.g. pypy version
        venv_args,
        venv_path,
    )
    # normalize types, via json round-trip
    validation = json.loads(json.dumps(validation))

    from os.path import join, abspath
    venv_path = abspath(venv_path)  # this removes trailing slashes as well
    state_path = join(venv_path, '.venv-update.state')

    from os.path import isdir
    if isdir(venv_path):
        try:
            with open(state_path) as state:
                previous_state = json.load(state)  # due to a coverage bug :pragma:nobranch:
        except IOError:
            previous_state = {}

        if previous_state.get('validation') == validation:
            info('Keeping virtualenv from previous run.')
            return
        else:
            info('Removing invalidated virtualenv.')
            info('(%r != %r)' % (validation, validation))
            # TODO: error out if venv_path is nonempty and doesn't look like a virtualenv
            run(('rm', '-rf', venv_path))

    from distutils.spawn import find_executable as which  # pylint:disable=import-error
    virtualenv = which('virtualenv')
    run((virtualenv, venv_path,) + venv_args)

    if isdir(venv_path):
        with open(state_path, 'w') as state:
            json.dump(
                dict(executable=executable, validation=validation),
                state,
            )

def wait_for_all_subprocesses():
    from os import wait
    try:
        while True:
            wait()
    except OSError as error:
        if error.errno == 10:  # no child processes
            return
        else:
            raise

def mtime_offset(reference, hours_offset):
    from os.path import getmtime
    mtime = getmtime(reference)

    return mtime + hours_offset * 60 * 60

def touch(filename, timestamp):
    if timestamp is not None:
        timestamp = (timestamp, timestamp)  # atime, mtime

    from os import utime
    utime(filename, timestamp)

def mark_venv_valid(venv_path):
    wait_for_all_subprocesses()
    touch(venv_path, None)

def mark_venv_invalid(venv_path, reqs):
    # LBYL, to attempt to avoid any exception during exception handling
    from os.path import isdir
    if isdir(venv_path) and exists(reqs[0]):
        info('')
        info("Something went wrong! Sending '%s' back in time, so make knows it's invalid." % timid_relpath(venv_path))
        info('Waiting for all subprocesses to finish...')
        wait_for_all_subprocesses()
        info('DONE')
        touch(venv_path, mtime_offset(reqs[0], -24))
        info('')

def dotpy(filename):
    if filename.endswith(('.pyc', '.pyo', '.pyd')):
        return filename[:-1]
    else:
        return filename

def venv_executable(venv_path, executable):
    from os.path import join
    return join(venv_path, 'bin', executable)

def venv_python(venv_path):
    return venv_executable(venv_path, 'python')

def venv_update(venv_path, reqs, venv_args):
    """we have an arbitrary python interpreter active, (possibly) outside the virtualenv we want.

    make a fresh venv at the right spot, make sure it has pip-faster, and use it
    """
    from os.path import abspath
    venv_path = abspath(venv_path)
    validate_venv(venv_path, venv_args)
    python = venv_python(venv_path)
    if not exists(python):
        return 'virtualenv executable not found: %s' % python

    # ensure that a compatible version of pip is installed
    run((python, '-m', 'pip.__main__', '--version'))

    from os import environ
    pipdir = environ['HOME'] + '/.pip'
    # We could combine these caches to one directory, but pip would search everything twice, going slower.
    pip_wheels = pipdir + '/wheelhouse'

    pip_install = (
        python, '-m', 'pip.__main__', 'install',
        '--find-links=file://' + pip_wheels,
    )

    if exists(VENV_UPDATE_REQS_OVERRIDE):
        args = ('-r', VENV_UPDATE_REQS_OVERRIDE)
    else:
        args = ('pip-faster==' + __version__,)

    # TODO: short-circuit when pip-faster is already there.
    run(pip_install + args)

    run((python, '-m', 'pip_faster', 'install', '--prune', '--upgrade') + sum(
        [('-r', req) for req in reqs],
        (),
    ))

def raise_on_failure(mainfunc):
    """raise if and only if mainfunc fails"""
    from subprocess import CalledProcessError
    try:
        errors = mainfunc()
        if errors:
            exit(errors)
    except CalledProcessError as error:
        exit(error.returncode)
    except SystemExit as error:
        if error.code:
            raise
    except KeyboardInterrupt:  # I don't plan to test-cover this.  :pragma:nocover:
        exit(1)

def main():
    from sys import argv, path
    del path[:1]  # we don't (want to) import anything from pwd or the script's directory
    venv_path, reqs, venv_args = parseargs(argv[1:])

    try:
        raise_on_failure(lambda: venv_update(venv_path, reqs, venv_args))
    except BaseException:
        mark_venv_invalid(venv_path, reqs)
        raise
    else:
        mark_venv_valid(venv_path)

if __name__ == '__main__':
    exit(main())
<commit_msg>this is 1.0rc1<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''\
usage: venv-update [-h] [virtualenv_dir] [requirements [requirements ...]]

Update a (possibly non-existant) virtualenv directory using a requirements.txt listing
When this script completes, the virtualenv should have the same packages as if it were
removed, then rebuilt.

To set the index server, export a PIP_INDEX_URL variable.
    See also: https://pip.readthedocs.org/en/stable/user_guide/#environment-variables

positional arguments:
  virtualenv_dir  Destination virtualenv directory (default: virtualenv_run)
  requirements    Requirements files. (default: requirements.txt)

optional arguments:
  -h, --help      show this help message and exit

Version control at: https://github.com/yelp/pip-faster
'''
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

from os.path import exists

VENV_UPDATE_REQS_OVERRIDE = 'requirements.d/venv-update.txt'
__version__ = '1.0rc1'

# This script must not rely on anything other than
#   stdlib>=2.6 and virtualenv>1.11

def parseargs(args):
    if set(args) & set(('-h', '--help')):
        print(__doc__, end='')
        exit(0)

    args = list(args)
    virtualenv_dir = None
    requirements = []
    remaining = []

    for arg in args:
        if arg.startswith('-'):
            remaining.append(arg)
        elif virtualenv_dir is None:
            virtualenv_dir = arg
        else:
            requirements.append(arg)

    if not virtualenv_dir:
        virtualenv_dir = 'virtualenv_run'
    if not requirements:
        requirements = ['requirements.txt']

    return virtualenv_dir, tuple(requirements), tuple(remaining)

def timid_relpath(arg):
    from os.path import isabs, relpath
    if isabs(arg):
        result = relpath(arg)
        if len(result) < len(arg):
            return result

    return arg

def shellescape(args):
    from pipes import quote
    return ' '.join(quote(timid_relpath(arg)) for arg in args)

def colorize(cmd):
    from os import isatty

    if isatty(1):
        template = '\033[01;36m>\033[m \033[01;32m{0}\033[m'
    else:
        template = '> {0}'

    return template.format(shellescape(cmd))

def run(cmd):
    from subprocess import check_call
    check_call(('echo', colorize(cmd)))
    check_call(cmd)

def info(msg):
    # use a subprocess to ensure correct output interleaving.
    from subprocess import check_call
    check_call(('echo', msg))

def validate_venv(venv_path, venv_args):
    """Ensure we have a valid virtualenv."""
    import json
    from sys import executable, version
    # we count any existing virtualenv invalidated if any of these relevant values changes
    validation = (
        version,  # includes e.g. pypy version
        venv_args,
        venv_path,
    )
    # normalize types, via json round-trip
    validation = json.loads(json.dumps(validation))

    from os.path import join, abspath
    venv_path = abspath(venv_path)  # this removes trailing slashes as well
    state_path = join(venv_path, '.venv-update.state')

    from os.path import isdir
    if isdir(venv_path):
        try:
            with open(state_path) as state:
                previous_state = json.load(state)  # due to a coverage bug :pragma:nobranch:
        except IOError:
            previous_state = {}

        if previous_state.get('validation') == validation:
            info('Keeping virtualenv from previous run.')
            return
        else:
            info('Removing invalidated virtualenv.')
            info('(%r != %r)' % (validation, validation))
            # TODO: error out if venv_path is nonempty and doesn't look like a virtualenv
            run(('rm', '-rf', venv_path))

    from distutils.spawn import find_executable as which  # pylint:disable=import-error
    virtualenv = which('virtualenv')
    run((virtualenv, venv_path,) + venv_args)

    if isdir(venv_path):
        with open(state_path, 'w') as state:
            json.dump(
                dict(executable=executable, validation=validation),
                state,
            )

def wait_for_all_subprocesses():
    from os import wait
    try:
        while True:
            wait()
    except OSError as error:
        if error.errno == 10:  # no child processes
            return
        else:
            raise

def mtime_offset(reference, hours_offset):
    from os.path import getmtime
    mtime = getmtime(reference)

    return mtime + hours_offset * 60 * 60

def touch(filename, timestamp):
    if timestamp is not None:
        timestamp = (timestamp, timestamp)  # atime, mtime

    from os import utime
    utime(filename, timestamp)

def mark_venv_valid(venv_path):
    wait_for_all_subprocesses()
    touch(venv_path, None)

def mark_venv_invalid(venv_path, reqs):
    # LBYL, to attempt to avoid any exception during exception handling
    from os.path import isdir
    if isdir(venv_path) and exists(reqs[0]):
        info('')
        info("Something went wrong! Sending '%s' back in time, so make knows it's invalid." % timid_relpath(venv_path))
        info('Waiting for all subprocesses to finish...')
        wait_for_all_subprocesses()
        info('DONE')
        touch(venv_path, mtime_offset(reqs[0], -24))
        info('')

def dotpy(filename):
    if filename.endswith(('.pyc', '.pyo', '.pyd')):
        return filename[:-1]
    else:
        return filename

def venv_executable(venv_path, executable):
    from os.path import join
    return join(venv_path, 'bin', executable)

def venv_python(venv_path):
    return venv_executable(venv_path, 'python')

def venv_update(venv_path, reqs, venv_args):
    """we have an arbitrary python interpreter active, (possibly) outside the virtualenv we want.

    make a fresh venv at the right spot, make sure it has pip-faster, and use it
    """
    from os.path import abspath
    venv_path = abspath(venv_path)
    validate_venv(venv_path, venv_args)
    python = venv_python(venv_path)
    if not exists(python):
        return 'virtualenv executable not found: %s' % python

    # ensure that a compatible version of pip is installed
    run((python, '-m', 'pip.__main__', '--version'))

    from os import environ
    pipdir = environ['HOME'] + '/.pip'
    # We could combine these caches to one directory, but pip would search everything twice, going slower.
    pip_wheels = pipdir + '/wheelhouse'

    pip_install = (
        python, '-m', 'pip.__main__', 'install',
        '--find-links=file://' + pip_wheels,
    )

    if exists(VENV_UPDATE_REQS_OVERRIDE):
        args = ('-r', VENV_UPDATE_REQS_OVERRIDE)
    else:
        args = ('pip-faster==' + __version__,)

    # TODO: short-circuit when pip-faster is already there.
    run(pip_install + args)

    run((python, '-m', 'pip_faster', 'install', '--prune', '--upgrade') + sum(
        [('-r', req) for req in reqs],
        (),
    ))

def raise_on_failure(mainfunc):
    """raise if and only if mainfunc fails"""
    from subprocess import CalledProcessError
    try:
        errors = mainfunc()
        if errors:
            exit(errors)
    except CalledProcessError as error:
        exit(error.returncode)
    except SystemExit as error:
        if error.code:
            raise
    except KeyboardInterrupt:  # I don't plan to test-cover this.  :pragma:nocover:
        exit(1)

def main():
    from sys import argv, path
    del path[:1]  # we don't (want to) import anything from pwd or the script's directory
    venv_path, reqs, venv_args = parseargs(argv[1:])

    try:
        raise_on_failure(lambda: venv_update(venv_path, reqs, venv_args))
    except BaseException:
        mark_venv_invalid(venv_path, reqs)
        raise
    else:
        mark_venv_valid(venv_path)

if __name__ == '__main__':
    exit(main())
<|endoftext|>


<commit_before>import c3d
import io
import os
import unittest
import numpy as np

def genByteWordArr(word, shape):
	'''	Generate a multi-dimensional byte array from a specific word.
	'''
	arr = np.array(word)
	for d in shape[::-1]:
		arr = arr[np.newaxis].repeat(d, 0)
	return arr, [len(word)] + [d for d in shape]

def genRndByteArr(wordlen, shape, pad):
	'''	Generate a multi-dimensional byte array with random data.
	'''
	tot_len = wordlen + pad*wordlen
	arr = np.empty(shape, dtype=np.dtype('S'+str(tot_len)))
	for i in np.ndindex(arr.shape):
		bytes = np.random.randint(21, 126, wordlen).astype(np.uint8)
		if pad:
			bytes = np.hstack((bytes, np.array([b'255']*wordlen, dtype=np.uint8)))
		arr[i] = bytes.tobytes()
	return arr, [tot_len] + [d for d in shape]

def genRndFloatArr(shape, rnd, range=(-1e6, 1e6)):
	'''	Generate a multi-dimensional array of 32 bit floating point data.
	'''
	return rnd.uniform(range[0], range[1], shape)

class ParameterTest(unittest.TestCase):
	def setUp(self):
		self.rnd = np.random.default_rng()
		self.dtypes = c3d.DataTypes(c3d.PROCESSOR_INTEL)

	def test_a_parse_float32_array(self):
		flt_range = (-1e6, 1e6)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.float32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.float_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'float_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading float array'

	def test_b_parse_int32_array(self):
		flt_range = (-1e6, 1e6)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int32_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_c_parse_uint32_array(self):
		flt_range = (0, 1e6)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint32_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_d_parse_int16_array(self):
		flt_range = (-1e4, 1e4)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int16)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int16_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_e_parse_uint16_array(self):
		flt_range = (0, 1e4)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint16)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint16_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_e_parse_int8_array(self):
		flt_range = (-127, 127)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int8)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int8_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_f_parse_uint8_array(self):
		flt_range = (0, 255)

		shapes = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

		for shape in shapes:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint8)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint8_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_g_parse_byte_array(self):
		'''	Verify byte arrays are parsed correctly
		'''
		word = b'WRIST'

		# 1 dims
		arr = np.array(word).repeat(3).repeat(3).repeat(3)
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=arr.shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array
		assert arr.shape[1:] == arr_out.shape, "Mismatch in 'bytes_array' converted shape"
		assert np.all(arr.tobytes() == arr_out), 'Mismatch in reading single dimensional byte array'

		# 4 dims
		arr, shape = genByteWordArr(word, [5, 4, 3])
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'bytes_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert np.all(arr[i[::-1]] == arr_out[i]), "Mismatch in 'bytes_array' converted value at index %s" % str(i)

		# 5 dims
		arr, shape = genByteWordArr(word, [6, 5, 4, 3])
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'bytes_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert np.all(arr[i[::-1]] == arr_out[i]), "Mismatch in 'bytes_array' converted value at index %s" % str(i)

	def test_h_parse_string_array(self):
		'''	Verify repeated word arrays are parsed correctly
		'''
		word = b'ANCLE'

		# 3 dims
		arr, shape = genByteWordArr(word, [7, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 4 dims
		arr, shape = genByteWordArr(word, [5, 4, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 5 dims
		arr, shape = genByteWordArr(word, [6, 5, 4, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

	def test_i_parse_random_string_array(self):
		'''	Verify random word arrays are parsed correctly
		'''
		##
		# RND

		# 3 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 4 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 5, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 5 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 6, 5, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

if __name__ == '__main__':
	unittest.main()
<commit_msg>Made shape array a class variable<commit_after>import c3d
import io
import os
import unittest
import numpy as np

def genByteWordArr(word, shape):
	'''	Generate a multi-dimensional byte array from a specific word.
	'''
	arr = np.array(word)
	for d in shape[::-1]:
		arr = arr[np.newaxis].repeat(d, 0)
	return arr, [len(word)] + [d for d in shape]

def genRndByteArr(wordlen, shape, pad):
	'''	Generate a multi-dimensional byte array with random data.
	'''
	tot_len = wordlen + pad*wordlen
	arr = np.empty(shape, dtype=np.dtype('S'+str(tot_len)))
	for i in np.ndindex(arr.shape):
		bytes = np.random.randint(21, 126, wordlen).astype(np.uint8)
		if pad:
			bytes = np.hstack((bytes, np.array([b'255']*wordlen, dtype=np.uint8)))
		arr[i] = bytes.tobytes()
	return arr, [tot_len] + [d for d in shape]

def genRndFloatArr(shape, rnd, range=(-1e6, 1e6)):
	'''	Generate a multi-dimensional array of 32 bit floating point data.
	'''
	return rnd.uniform(range[0], range[1], shape)

class ParameterArrayTest(unittest.TestCase):
	'''	Test read Parameter arrays
	'''

	SHAPES = [[7, 6, 5], [7, 5, 3], [7, 3], [19]]

	def setUp(self):
		self.rnd = np.random.default_rng()
		self.dtypes = c3d.DataTypes(c3d.PROCESSOR_INTEL)

	def test_a_parse_float32_array(self):
		flt_range = (-1e6, 1e6)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.float32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.float_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'float_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading float array'

	def test_b_parse_int32_array(self):
		flt_range = (-1e6, 1e6)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int32_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_c_parse_uint32_array(self):
		flt_range = (0, 1e6)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint32)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint32_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_d_parse_int16_array(self):
		flt_range = (-1e4, 1e4)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int16)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int16_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_e_parse_uint16_array(self):
		flt_range = (0, 1e4)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint16)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint16_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_e_parse_int8_array(self):
		flt_range = (-127, 127)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.int8)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.int8_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'int32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading int32 array'

	def test_f_parse_uint8_array(self):
		flt_range = (0, 255)

		for shape in ParameterArrayTest.SHAPES:
			arr = self.rnd.uniform(flt_range[0], flt_range[1], size=shape).astype(np.uint8)
			P = c3d.Param('FLOAT_TEST', self.dtypes, bytes_per_element=4, dimensions=arr.shape, bytes=arr.T.tobytes())
			arr_out = P.uint8_array
			assert arr.T.shape == arr_out.shape, "Mismatch in 'uint32_array' converted shape"
			assert np.all(arr.T == arr_out), 'Value mismatch when reading uint32 array'

	def test_g_parse_byte_array(self):
		'''	Verify byte arrays are parsed correctly
		'''
		word = b'WRIST'

		# 1 dims
		arr = np.array(word).repeat(3).repeat(3).repeat(3)
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=arr.shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array
		assert arr.shape[1:] == arr_out.shape, "Mismatch in 'bytes_array' converted shape"
		assert np.all(arr.tobytes() == arr_out), 'Mismatch in reading single dimensional byte array'

		# 4 dims
		arr, shape = genByteWordArr(word, [5, 4, 3])
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'bytes_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert np.all(arr[i[::-1]] == arr_out[i]), "Mismatch in 'bytes_array' converted value at index %s" % str(i)

		# 5 dims
		arr, shape = genByteWordArr(word, [6, 5, 4, 3])
		P = c3d.Param('BYTE_TEST', self.dtypes, bytes_per_element=1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.bytes_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'bytes_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert np.all(arr[i[::-1]] == arr_out[i]), "Mismatch in 'bytes_array' converted value at index %s" % str(i)

	def test_h_parse_string_array(self):
		'''	Verify repeated word arrays are parsed correctly
		'''
		word = b'ANCLE'

		# 3 dims
		arr, shape = genByteWordArr(word, [7, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 4 dims
		arr, shape = genByteWordArr(word, [5, 4, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 5 dims
		arr, shape = genByteWordArr(word, [6, 5, 4, 3])
		P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
		arr_out = P.string_array

		assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
			(str(arr_out.shape), str(arr.T.shape))
		for i in np.ndindex(arr_out.shape):
			assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
				"Mismatch in 'string_array' converted value at index %s" % str(i)

	def test_i_parse_random_string_array(self):
		'''	Verify random word arrays are parsed correctly
		'''
		##
		# RND

		# 3 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 4 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 5, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

		# 5 dims
		for wlen in range(10):
			arr, shape = genRndByteArr(wlen, [7, 6, 5, 3], wlen > 5)
			P = c3d.Param('STRING_TEST', self.dtypes, bytes_per_element=-1, dimensions=shape, bytes=arr.T.tobytes())
			arr_out = P.string_array

			assert arr.T.shape == arr_out.shape, "Mismatch in 'string_array' converted shape. Was %s, expected %s" %\
				(str(arr_out.shape), str(arr.T.shape))
			for i in np.ndindex(arr_out.shape):
				assert self.dtypes.decode_string(arr[i[::-1]]) == arr_out[i],\
					"Mismatch in 'string_array' converted value at index %s" % str(i)

if __name__ == '__main__':
	unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- Coding:utf-8 -*-
#
# Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors: Jan Safranek <jsafrane@redhat.com>

from test_base import StorageTestBase
import unittest
import pywbem

class TestSetPartitionStyle(StorageTestBase):
    """
        Test LMI_DiskPartitionConfigurationService.SetPartitionStyle
        with different parameters.
    """

    DISK_CLASS = "LMI_StorageExtent"
    MBR_CLASS = "LMI_DiskPartition"
    GPT_CLASS = "LMI_GenericDiskPartition"

    STYLE_EMBR = 4100
    STYLE_MBR = 2
    STYLE_GPT = 3

    def setUp(self):
        """ Find disk partition service. """
        super(TestSetPartitionStyle, self).setUp()
        self.service = self.wbemconnection.EnumerateInstanceNames(
                "LMI_DiskPartitionConfigurationService")[0]

    def _check_name(self, partname, classname):
        """ Check that CIMInstanceName represents a partition. """
        self.assertEqual(partname['SystemName'], self.SYSTEM_NAME)
        self.assertEqual(partname['SystemCreationClassName'], self.SYSTEM_CLASS_NAME)
        self.assertEqual(partname['CreationClassName'], classname)

    def _create_disk_name(self, device_id):
        """ Return CIMInstanceName for given DeviceID."""
        name = pywbem.CIMInstanceName(
                classname=self.DISK_CLASS,
                keybindings={
                        'DeviceID': device_id,
                        'SystemCreationClassName': self.SYSTEM_CLASS_NAME,
                        'SystemName': self.SYSTEM_NAME,
                        'CreationClassName': self.DISK_CLASS})
        return name

    def _check_capabilities_name(self, capabilities_name, partition_style):
        """
            Check that given partition capabilities name represent givent
            partition_style.
        """
        capabilities = self.wbemconnection.GetInstance(capabilities_name)
        self.assertEqual(capabilities[0]['PartitionStyle'], partition_style)

    def _check_partition_table_type(self, diskname, partition_style):
        """
            Check that disk (represented by CIMInstanceName diskname)
            has partition table partition_style.
        """
        capabilities = self.wbemconnection.Associators(
                diskname,
                AssocClass="LMI_InstalledPartitionTable")
        # there must be only one associated Capabilities instance
        self.assertEqual(len(capabilities), 1)
        self.assertEqual(capabilities[0]['PartitionStyle'], partition_style)

    def _get_capabilities_name(self, style):
        """
            Return CIMInstanceName with partition capabilities representing
            given partition style.
        """
        if style == self.STYLE_EMBR:
            style_name = "EMBR"
        elif style == self.STYLE_MBR:
            style_name = "MBR"
        elif style == self.STYLE_GPT:
            style_name = "GPT"

        return pywbem.CIMInstanceName(
                classname="LMI_DiskPartitionConfigurationCapabilities",
                keybindings={
                        'InstanceID': "LMI:LMI_DiskPartitionConfigurationCapabilities:" + style_name
                })

    def test_no_params(self):
        """ Try SetPartitionStyle with no parameters"""
        # Extent = None -> error
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service)

    def test_no_partition_style(self):
        """ Test SetPartitionStyle with no PartitionStyle parameter."""
        diskname = self._create_disk_name(self.disks[0])
        # Extent = sda -> success, with default capabilities (GPT)
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_GPT)
        self.assertDictEqual(outparams, {})

    def test_gpt(self):
        """ Test SetPartitionStyle with GPT capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_GPT)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_GPT)
        self.assertDictEqual(outparams, {})

    def test_mbr(self):
        """ Test SetPartitionStyle with GPT capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_MBR)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_MBR)
        self.assertDictEqual(outparams, {})

    def test_embr(self):
        """ Test SetPartitionStyle with EMBR capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_EMBR)
        # Extent = sda, partStyle = MBR -> success
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)

    def test_on_partition(self):
        """ Test SetPartitionStyle on a partition."""
        diskname = self._create_disk_name(self.disks[0])
        # create a partition table on disk
        part_style = self._get_capabilities_name(self.STYLE_MBR)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_MBR)
        self.assertDictEqual(outparams, {})

        # create partition on the disk
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "CreateOrModifyPartition",
                self.service,
                extent=diskname)
        self.assertEqual(retval, 0)
        partition = outparams['partition']

        # try to create partition table on it
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service,
                Extent=partition,
                PartitionStyle=part_style)

        # remove the partition
        self.wbemconnection.DeleteInstance(partition)

    #TODO: add SetPartitionStyle on RAID, LVM etc.

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed a typo.<commit_after>#!/usr/bin/python
# -*- Coding:utf-8 -*-
#
# Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors: Jan Safranek <jsafrane@redhat.com>

from test_base import StorageTestBase
import unittest
import pywbem

class TestSetPartitionStyle(StorageTestBase):
    """
        Test LMI_DiskPartitionConfigurationService.SetPartitionStyle
        with different parameters.
    """

    DISK_CLASS = "LMI_StorageExtent"
    MBR_CLASS = "LMI_DiskPartition"
    GPT_CLASS = "LMI_GenericDiskPartition"

    STYLE_EMBR = 4100
    STYLE_MBR = 2
    STYLE_GPT = 3

    def setUp(self):
        """ Find disk partition service. """
        super(TestSetPartitionStyle, self).setUp()
        self.service = self.wbemconnection.EnumerateInstanceNames(
                "LMI_DiskPartitionConfigurationService")[0]

    def _check_name(self, partname, classname):
        """ Check that CIMInstanceName represents a partition. """
        self.assertEqual(partname['SystemName'], self.SYSTEM_NAME)
        self.assertEqual(partname['SystemCreationClassName'], self.SYSTEM_CLASS_NAME)
        self.assertEqual(partname['CreationClassName'], classname)

    def _create_disk_name(self, device_id):
        """ Return CIMInstanceName for given DeviceID."""
        name = pywbem.CIMInstanceName(
                classname=self.DISK_CLASS,
                keybindings={
                        'DeviceID': device_id,
                        'SystemCreationClassName': self.SYSTEM_CLASS_NAME,
                        'SystemName': self.SYSTEM_NAME,
                        'CreationClassName': self.DISK_CLASS})
        return name

    def _check_capabilities_name(self, capabilities_name, partition_style):
        """
            Check that given partition capabilities name represent givent
            partition_style.
        """
        capabilities = self.wbemconnection.GetInstance(capabilities_name)
        self.assertEqual(capabilities[0]['PartitionStyle'], partition_style)

    def _check_partition_table_type(self, diskname, partition_style):
        """
            Check that disk (represented by CIMInstanceName diskname)
            has partition table partition_style.
        """
        capabilities = self.wbemconnection.Associators(
                diskname,
                AssocClass="LMI_InstalledPartitionTable")
        # there must be only one associated Capabilities instance
        self.assertEqual(len(capabilities), 1)
        self.assertEqual(capabilities[0]['PartitionStyle'], partition_style)

    def _get_capabilities_name(self, style):
        """
            Return CIMInstanceName with partition capabilities representing
            given partition style.
        """
        if style == self.STYLE_EMBR:
            style_name = "EMBR"
        elif style == self.STYLE_MBR:
            style_name = "MBR"
        elif style == self.STYLE_GPT:
            style_name = "GPT"

        return pywbem.CIMInstanceName(
                classname="LMI_DiskPartitionConfigurationCapabilities",
                keybindings={
                        'InstanceID': "LMI:LMI_DiskPartitionConfigurationCapabilities:" + style_name
                })

    def test_no_params(self):
        """ Try SetPartitionStyle with no parameters"""
        # Extent = None -> error
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service)

    def test_no_partition_style(self):
        """ Test SetPartitionStyle with no PartitionStyle parameter."""
        diskname = self._create_disk_name(self.disks[0])
        # Extent = sda -> success, with default capabilities (GPT)
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_GPT)
        self.assertDictEqual(outparams, {})

    def test_gpt(self):
        """ Test SetPartitionStyle with GPT capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_GPT)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_GPT)
        self.assertDictEqual(outparams, {})

    def test_mbr(self):
        """ Test SetPartitionStyle with MBR capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_MBR)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_MBR)
        self.assertDictEqual(outparams, {})

    def test_embr(self):
        """ Test SetPartitionStyle with EMBR capabilities."""
        diskname = self._create_disk_name(self.disks[0])
        part_style = self._get_capabilities_name(self.STYLE_EMBR)
        # Extent = sda, partStyle = MBR -> success
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)

    def test_on_partition(self):
        """ Test SetPartitionStyle on a partition."""
        diskname = self._create_disk_name(self.disks[0])
        # create a partition table on disk
        part_style = self._get_capabilities_name(self.STYLE_MBR)
        # Extent = sda, partStyle = MBR -> success
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "SetPartitionStyle",
                self.service,
                Extent=diskname,
                PartitionStyle=part_style)
        self.assertEqual(retval, 0)
        self._check_partition_table_type(diskname, self.STYLE_MBR)
        self.assertDictEqual(outparams, {})

        # create partition on the disk
        (retval, outparams) = self.wbemconnection.InvokeMethod(
                "CreateOrModifyPartition",
                self.service,
                extent=diskname)
        self.assertEqual(retval, 0)
        partition = outparams['partition']

        # try to create partition table on it
        self.assertRaises(pywbem.CIMError, self.wbemconnection.InvokeMethod,
                "SetPartitionStyle",
                self.service,
                Extent=partition,
                PartitionStyle=part_style)

        # remove the partition
        self.wbemconnection.DeleteInstance(partition)

    #TODO: add SetPartitionStyle on RAID, LVM etc.

if __name__ == '__main__':
    unittest.main()
<|endoftext|>