<commit_before>from __future__ import with_statement

import urlparse

from django.core import exceptions, urlresolvers
from django.test import client, utils
from django.utils import simplejson as json

from tastypie import authorization as tastypie_authorization
from tastypie.test import ResourceTestCase, TestApiClient

from tastypie_elasticsearch import resources as tastypie_elasticsearch_resources

import elasticsearch

import resources

# TODO: Test set operations
# TODO: Test bulk operations
# TODO: Test ordering, filtering
# TODO: Use Tastypie's testcase class for tests?

@utils.override_settings(DEBUG=True)
class TastypieElasticsearchTest(ResourceTestCase):
    api_name = 'v1'

    def setUp(self):
        super(TastypieElasticsearchTest, self).setUp()

        self.resource_class = resources.TestResource
        self.resource_class._meta.index = "test"
        #self.resource_class._meta.cache = NoCache()

        #
        # ensure index is up and created
        #
        es_server = resources.TestResource._meta.es_server
        index = self.resource_class._meta.index

        h, p = es_server.split(":")
        hosts = [{"host":h, "port":p}]
        self.es = elasticsearch.Elasticsearch(hosts,
            timeout=30)

        if self.es.indices.exists(index):
            self.es.indices.delete(index)
        self.es.indices.create(index)

    def tearDown(self):
        index = self.resource_class._meta.index
        #self.es.indices.delete(index)

    def resourceListURI(self, resource_name):
        return urlresolvers.reverse('api_dispatch_list', kwargs={'api_name': self.api_name, 'resource_name': resource_name})

    def resourcePK(self, resource_uri):
        match = urlresolvers.resolve(resource_uri)
        return match.kwargs['pk']

    def resourceDetailURI(self, resource_name, resource_pk):
        return urlresolvers.reverse('api_dispatch_detail', kwargs={'api_name': self.api_name,
            'resource_name': resource_name, 'pk': resource_pk})

    def fullURItoAbsoluteURI(self, uri):
        scheme, netloc, path, query, fragment = urlparse.urlsplit(uri)
        return urlparse.urlunsplit((None, None, path, query, fragment))

class BasicTest(TastypieElasticsearchTest):

    def testCRUD(self):
        # Testing POST

        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        #create an object
        obj = dict(_id=1, first_name="John", last_name="Doe")

        response = self.api_client.post(base_url, format="json", data=obj)
        self.assertHttpCreated(response)

        obj_uri = response.get("Location")

        # get the list
        response = self.api_client.get(base_url)
        self.assertHttpOK(response)

        response = json.loads(response.content)
        self.assertEqual(len(response["objects"]), 1)

        #get the object
        response = self.api_client.get(obj_uri)
        self.assertHttpOK(response)

        # update object
        obj1 = json.loads(response.content)
        del obj1["resource_uri"]

        obj1["first_name"] = u"Person 1 UPDATED"
        response = self.api_client.post(base_url, format="json", data=obj1)
        self.assertHttpCreated(response)

        #get the *updated?* object
        response = self.api_client.get(obj_uri)
        self.assertHttpOK(response)
        #and compare it
        data = json.loads(response.content)
        for k, v in obj1.iteritems():
            self.assertEqual(data[k], v)

        #delete the object
        response = self.api_client.delete(obj_uri)
        self.assertHttpAccepted(response)

        #get the *already deleted* object, not found expected
        response = self.api_client.get(obj_uri)
        self.assertHttpNotFound(response)

    def test_bulk(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        for i in range(10):
            #create an object
            obj = dict(first_name="Person %d" % i, number=i)
            response = self.api_client.post(base_url,
                format="json", data=obj)
            self.assertHttpCreated(response)

    def test_invalid(self):
        # Invalid ObjectId
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        response = self.api_client.get(base_url + 'foobar/')
        self.assertEqual(response.status_code, 404)

    def test_ordering(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        for i in range(100):
            response = self.api_client.post(base_url,
                format="json", data=dict(
                    name="Person %d" % (i + 1), number=i+1))
            self.assertHttpCreated(response)

        for key in ["number"]:
            for ordering in [key, "-%s" % key]:

                response = self.api_client.get(base_url,
                    data={'order_by': ordering})
                self.assertEqual(response.status_code, 200)

                data = json.loads(response.content)

                sq = []
                objects = data.get("objects")
                if objects:
                    for obj in objects:
                        self.assertTrue(obj.has_key("number"))
                        number = obj.get("number")
                        sq.append(number)

                    ran = ([i for i in reversed(range(100 - (len(objects) - 1), 100 + 1))]
                            if ordering.startswith("-") else range(1, len(objects) + 1))
                    self.assertEqual(sq, ran)

    def test_pagination(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        count = 100

        for i in range(count):
            response = self.api_client.post(base_url,
                format="json", data=dict(name="Person %d" % (i + 1), number=i+1))
            self.assertEqual(response.status_code, 201)

        base_url = self.resourceListURI(resource_name)

        response = self.api_client.get(base_url,
            data={'offset': '42', 'limit': 7, "order_by": "number"})
        self.assertEqual(response.status_code, 200)

        response = json.loads(response.content)

        self.assertEqual(response['meta']['total_count'], 100)
        self.assertEqual(response['meta']['offset'], 42)
        self.assertEqual(response['meta']['limit'], 7)
        self.assertEqual(len(response['objects']), 7)

        for i, obj in enumerate(response['objects']):
            self.assertEqual(obj['name'], "Person %s" % (43 + i))

        offset = response['objects'][0]['number']

        response = self.api_client.get(
            self.resourceListURI(resource_name),
            data={'offset': offset, 'limit': 7, "order_by": "number"})
        self.assertEqual(response.status_code, 200)
        response = json.loads(response.content)

        self.assertEqual(response['meta']['total_count'], count)
        self.assertEqual(response['meta']['offset'], offset)
        self.assertEqual(response['meta']['limit'], 7)
        self.assertEqual(len(response['objects']), 7)

        for i, obj in enumerate(response['objects']):
            self.assertEqual(obj['name'], "Person %s" % (44 + i))

        # invalid limit parameter, fail
        response = self.api_client.get(
            self.resourceListURI(resource_name),
            data={'offset': offset, 'limit': -7, "order_by": "number"})
        self.assertEqual(response.status_code, 400)

    def test_percolator(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        for i in range(100):
            response = self.api_client.post(base_url,
                format="json", data=dict(
                    name="Person %d" % (i + 1), number=i+1))

        #get the object
        response = self.api_client.post(base_url + 'percolate/',
            format='json', data={"body":{"name":"python"}})

        self.assertHttpOK(response)

        result = json.loads(response.content)

        self.assertTrue("meta" in result.keys())
        meta = result.get("meta")

        self.assertTrue("matches" in meta)
        self.assertTrue(isinstance(meta.get('matches'), list))

        <commit_msg>unittests for bulk<commit_after>from __future__ import with_statement

import urlparse

from django.core import exceptions, urlresolvers
from django.test import client, utils
from django.utils import simplejson as json

from tastypie import authorization as tastypie_authorization
from tastypie.test import ResourceTestCase, TestApiClient

from tastypie_elasticsearch import resources as tastypie_elasticsearch_resources

import elasticsearch

import resources

# TODO: Test set operations
# TODO: Test bulk operations
# TODO: Test ordering, filtering
# TODO: Use Tastypie's testcase class for tests?

@utils.override_settings(DEBUG=True)
class TastypieElasticsearchTest(ResourceTestCase):
    api_name = 'v1'

    def setUp(self):
        super(TastypieElasticsearchTest, self).setUp()

        self.resource_class = resources.TestResource
        self.resource_class._meta.index = "test"
        #self.resource_class._meta.cache = NoCache()

        #
        # ensure index is up and created
        #
        es_server = resources.TestResource._meta.es_server
        index = self.resource_class._meta.index

        h, p = es_server.split(":")
        hosts = [{"host":h, "port":p}]
        self.es = elasticsearch.Elasticsearch(hosts,
            timeout=30)

        if self.es.indices.exists(index):
            self.es.indices.delete(index)
        self.es.indices.create(index)

    def tearDown(self):
        index = self.resource_class._meta.index
        #self.es.indices.delete(index)

    def resourceListURI(self, resource_name):
        return urlresolvers.reverse('api_dispatch_list', kwargs={'api_name': self.api_name, 'resource_name': resource_name})

    def resourcePK(self, resource_uri):
        match = urlresolvers.resolve(resource_uri)
        return match.kwargs['pk']

    def resourceDetailURI(self, resource_name, resource_pk):
        return urlresolvers.reverse('api_dispatch_detail', kwargs={'api_name': self.api_name,
            'resource_name': resource_name, 'pk': resource_pk})

    def fullURItoAbsoluteURI(self, uri):
        scheme, netloc, path, query, fragment = urlparse.urlsplit(uri)
        return urlparse.urlunsplit((None, None, path, query, fragment))

class BasicTest(TastypieElasticsearchTest):

    def testCRUD(self):
        # Testing POST

        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        #create an object
        obj = dict(_id=1, first_name="John", last_name="Doe")

        response = self.api_client.post(base_url, format="json", data=obj)
        self.assertHttpCreated(response)

        obj_uri = response.get("Location")

        # get the list
        response = self.api_client.get(base_url)
        self.assertHttpOK(response)

        response = json.loads(response.content)
        self.assertEqual(len(response["objects"]), 1)

        #get the object
        response = self.api_client.get(obj_uri)
        self.assertHttpOK(response)

        # update object
        obj1 = json.loads(response.content)
        del obj1["resource_uri"]

        obj1["first_name"] = u"Person 1 UPDATED"
        response = self.api_client.post(base_url, format="json", data=obj1)
        self.assertHttpCreated(response)

        #get the *updated?* object
        response = self.api_client.get(obj_uri)
        self.assertHttpOK(response)
        #and compare it
        data = json.loads(response.content)
        for k, v in obj1.iteritems():
            self.assertEqual(data[k], v)

        #delete the object
        response = self.api_client.delete(obj_uri)
        self.assertHttpAccepted(response)

        #get the *already deleted* object, not found expected
        response = self.api_client.get(obj_uri)
        self.assertHttpNotFound(response)

    def test_bulk(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        #add an object to be updated
        obj = dict(_id=9999999, first_name="John", last_name="Doe", untouched=1)
        response = self.api_client.post(base_url, format="json", data=obj)
        self.assertHttpCreated(response)
        obj_uri = response.get("Location")
        obj_uri = '/' + '/'.join(obj_uri.split('/')[3:])

        objects = []
        deleted_objects = []

        obj = {
            '_id': 9999999,
            'first_name': 'John',
            'last_name': 'Doe Updated',
            'updated': True,
            'resource_uri': obj_uri,
        }
        objects.append(obj)

        for i in range(5):
            obj = {
                "_id": (i+1000),
                "first_name": "Person %d" % (i+1000),
                "last_name": "Smith",
                "bulk": True,
            }
            objects.append(obj)

        data = {
            "objects":objects,
            "deleted_objects":deleted_objects,
        }

        response = self.api_client.patch(base_url, format="json", data=data)
        self.assertHttpAccepted(response)

    def test_ordering(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        for i in range(100):
            response = self.api_client.post(base_url,
                format="json", data=dict(
                    name="Person %d" % (i + 1), number=i+1))
            self.assertHttpCreated(response)

        for key in ["number"]:
            for ordering in [key, "-%s" % key]:

                response = self.api_client.get(base_url,
                    data={'order_by': ordering})
                self.assertEqual(response.status_code, 200)

                data = json.loads(response.content)

                sq = []
                objects = data.get("objects")
                if objects:
                    for obj in objects:
                        self.assertTrue(obj.has_key("number"))
                        number = obj.get("number")
                        sq.append(number)

                    ran = ([i for i in reversed(range(100 - (len(objects) - 1), 100 + 1))]
                            if ordering.startswith("-") else range(1, len(objects) + 1))
                    self.assertEqual(sq, ran)

    def test_pagination(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        count = 100

        for i in range(count):
            response = self.api_client.post(base_url,
                format="json", data=dict(name="Person %d" % (i + 1), number=i+1))
            self.assertEqual(response.status_code, 201)

        base_url = self.resourceListURI(resource_name)

        response = self.api_client.get(base_url,
            data={'offset': '42', 'limit': 7, "order_by": "number"})
        self.assertEqual(response.status_code, 200)

        response = json.loads(response.content)

        self.assertEqual(response['meta']['total_count'], 100)
        self.assertEqual(response['meta']['offset'], 42)
        self.assertEqual(response['meta']['limit'], 7)
        self.assertEqual(len(response['objects']), 7)

        for i, obj in enumerate(response['objects']):
            self.assertEqual(obj['name'], "Person %s" % (43 + i))

        offset = response['objects'][0]['number']

        response = self.api_client.get(
            self.resourceListURI(resource_name),
            data={'offset': offset, 'limit': 7, "order_by": "number"})
        self.assertEqual(response.status_code, 200)
        response = json.loads(response.content)

        self.assertEqual(response['meta']['total_count'], count)
        self.assertEqual(response['meta']['offset'], offset)
        self.assertEqual(response['meta']['limit'], 7)
        self.assertEqual(len(response['objects']), 7)

        for i, obj in enumerate(response['objects']):
            self.assertEqual(obj['name'], "Person %s" % (44 + i))

        # invalid limit parameter, fail
        response = self.api_client.get(
            self.resourceListURI(resource_name),
            data={'offset': offset, 'limit': -7, "order_by": "number"})
        self.assertEqual(response.status_code, 400)

    def test_percolator(self):
        resource_name = 'test'
        base_url = self.resourceListURI(resource_name)

        for i in range(100):
            response = self.api_client.post(base_url,
                format="json", data=dict(
                    name="Person %d" % (i + 1), number=i+1))

        #get the object
        response = self.api_client.post(base_url + 'percolate/',
            format='json', data={"body":{"name":"python"}})

        self.assertHttpOK(response)

        result = json.loads(response.content)

        self.assertTrue("meta" in result.keys())
        meta = result.get("meta")

        self.assertTrue("matches" in meta)
        self.assertTrue(isinstance(meta.get('matches'), list))

        <|endoftext|>


<commit_before>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, ip,
                                    s_abranch, s_dependent, s_force, s_trigger,
                                    hash_upload_to_master,
                                    hash_download_from_master,
                                    clean_unpack, mkdir)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

CODEBASE = make_cb(['polli-sb', 'benchbuild', 'polli'])
FORCE_CODEBASE = make_new_cb(['polli-sb', 'benchbuild', 'polli'])

P = util.Property
BF = util.BuildFactory

ACCEPTED_BUILDERS = \
    slaves.get_hostlist(slaves.infosun,
                        predicate=lambda host: host["host"] in {'debussy', 'ligeti'})

def configure(c):
    sb_steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            '-DCMAKE_BUILD_TYPE=Release',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install",
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    sb_steps.extend(upload_pj)

    download_pj = hash_download_from_master("public_html/polyjit_sb.tar.gz",
                                            "polyjit_sb.tar.gz", "polyjit")
    slurm_steps = [
        define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/"))
    ]
    slurm_steps.extend(download_pj)
    slurm_steps.extend(clean_unpack("polyjit_sb.tar.gz", "llvm"))
    slurm_steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),
    ])
    slurm_steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('bb_src', ip('%(prop:scratch)s/benchbuild')),

        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install',
            P("BENCHBUILD_ROOT")),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', env={
            'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
            'BB_TMP_DIR': '/scratch/pjtest/src/',
            'BB_TEST_DIR': P("testinputs"),
            'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
            'BB_SLURM_PARTITION': 'anywhere',
            'BB_SLURM_NODE_DIR': '/local/hdd/buildbot-polyjit/',
            'BB_SLURM_ACCOUNT': 'anywhere',
            'BB_SLURM_TIMELIMIT': '24:00:00',
            'BB_CONTAINER_MOUNTS': ip('["%(prop:llvm)s", "%(prop:bb_src)s"]'),
            'BB_CONTAINER_PREFIXES': '["/opt/benchbuild", "/", "/usr", "/usr/local"]',
            'BB_ENV_PATH':
                ip('["%(prop:llvm)s/bin", "/scratch/pjtest/erlent/build"]'),
            'BB_ENV_LD_LIBRARY_PATH':
                ip('["%(prop:llvm)s/lib", "/scratch/pjtest/erlent/build"]'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:bb_src)s'),
            'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
            'BB_UNIONFS_ENABLE': 'false'
            },
            workdir=P('scratch')),
        # This only works on infosun machines
        cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
            ip("%(prop:bb_src)s")),
        mkdir(ip("%(prop:scratch)s/results"))
    ])

    c['builders'].append(
        builder("polyjit-superbuild", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(sb_steps)))
    c['builders'].append(
        builder("polyjit-superbuild-slurm", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(slurm_steps)))

def schedule(c):
    superbuild_sched = s_abranch("bs_polyjit-superbuild",
                                 CODEBASE, ["polyjit-superbuild"],
                                 treeStableTimer=10 * 60)
    c['schedulers'].extend([
        superbuild_sched,
        s_force("fs_polyjit-superbuild", FORCE_CODEBASE,
                ["polyjit-superbuild"]),
        s_trigger("ts_polyjit-superbuild", CODEBASE,
                  ["polyjit-superbuild"]),

        s_dependent("ds_polyjit-superbuild-slurm",
                    superbuild_sched,
                    ["polyjit-superbuild-slurm"]),
        s_force("fs_polyjit-superbuild-slurm", FORCE_CODEBASE,
                ["polyjit-superbuild-slurm"])
    ])

register(sys.modules[__name__])
<commit_msg>superbuild: oops<commit_after>import sys

from polyjit.buildbot.builders import register
from polyjit.buildbot import slaves
from polyjit.buildbot.utils import (builder, define, git, cmd, ip,
                                    s_abranch, s_dependent, s_force, s_trigger,
                                    hash_upload_to_master,
                                    hash_download_from_master,
                                    clean_unpack, mkdir)
from polyjit.buildbot.repos import make_cb, make_new_cb, codebases
from polyjit.buildbot.master import URL
from buildbot.plugins import util
from buildbot.changes import filter

CODEBASE = make_cb(['polli-sb', 'benchbuild', 'polli'])
FORCE_CODEBASE = make_new_cb(['polli-sb', 'benchbuild', 'polli'])

P = util.Property
BF = util.BuildFactory

ACCEPTED_BUILDERS = \
    slaves.get_hostlist(slaves.infosun,
                        predicate=lambda host: host["host"] in {'debussy', 'ligeti'})

def configure(c):
    sb_steps = [
        define("SUPERBUILD_ROOT", ip("%(prop:builddir)s/polli-sb")),
        define("UCHROOT_SUPERBUILD_ROOT", "/mnt/polli-sb"),
        define("POLYJIT_DEFAULT_BRANCH", "master"),

        git('polli-sb', 'master', codebases, workdir=P("SUPERBUILD_ROOT"),
            mode="full", method="fresh"),
        cmd(ip('%(prop:cmake_prefix)s/bin/cmake'), P("SUPERBUILD_ROOT"),
            '-DCMAKE_BUILD_TYPE=Release',
            '-DCMAKE_INSTALL_PREFIX=./_install',
            '-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -DLLVM_ENABLE_STATS',
            '-DBUILD_SHARED_LIBS=Off',
            '-DPOLYJIT_PAPI_PREFIX=/scratch/pjtest/opt/papi-5.5.1/install',
            ip('-DPOLYJIT_BRANCH_CLANG=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_LLVM=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLI=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            ip('-DPOLYJIT_BRANCH_POLLY=%(prop:POLYJIT_DEFAULT_BRANCH)s'),
            '-G', 'Ninja',
            usePTY=True,
            name="cmake",
            logEnviron=True,
            description="[uchroot] cmake: release build",
            descriptionDone="[uchroot] configured."),
        cmd("/usr/bin/ninja", "llvm-configure",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="configure LLVM",
            description="configuring LLVM",
            descriptionDone="configured LLVM",
            timeout=4800),
        cmd("/usr/bin/ninja", "polli-configure",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="configure PolyJIT",
            description="configuring PolyJIT",
            descriptionDone="configured PolyJIT",
            timeout=4800),
        cmd("/usr/bin/ninja",
            usePTY=True,
            logEnviron=True,
            haltOnFailure=True,
            name="build jit",
            description="[uchroot] building PolyJIT",
            descriptionDone="[uchroot] built PolyJIT",
            timeout=4800),
        cmd("tar", "czf", "../polyjit_sb.tar.gz", "-C", "./_install", ".")
    ]

    upload_pj = hash_upload_to_master(
        "polyjit_sb.tar.gz",
        "../polyjit_sb.tar.gz",
        "public_html/polyjit_sb.tar.gz", URL)
    sb_steps.extend(upload_pj)

    download_pj = hash_download_from_master("public_html/polyjit_sb.tar.gz",
                                            "polyjit_sb.tar.gz", "polyjit")
    slurm_steps = [
        define("scratch", ip("/scratch/pjtest/sb-%(prop:buildnumber)s/"))
    ]
    slurm_steps.extend(download_pj)
    slurm_steps.extend(clean_unpack("polyjit_sb.tar.gz", "llvm"))
    slurm_steps.extend([
        define("BENCHBUILD_ROOT", ip("%(prop:builddir)s/build/benchbuild/")),
        git('benchbuild', 'master', codebases, workdir=P("BENCHBUILD_ROOT")),
    ])
    slurm_steps.extend([
        define('benchbuild', ip('%(prop:scratch)s/env/bin/benchbuild')),
        define('llvm', ip('%(prop:scratch)s/llvm')),
        define('bb_src', ip('%(prop:scratch)s/benchbuild')),

        cmd('virtualenv', '-ppython3', ip('%(prop:scratch)s/env')),
        cmd(ip('%(prop:scratch)s/env/bin/pip3'), 'install',
            P("BENCHBUILD_ROOT")),
        mkdir(P("scratch")),
        cmd("rsync", "-var", "llvm", P("scratch")),
        cmd(P('benchbuild'), 'bootstrap', '-s', env={
            'BB_CONFIG_FILE': '/scratch/pjtest/.benchbuild.json',
            'BB_TMP_DIR': '/scratch/pjtest/src/',
            'BB_TEST_DIR': P("testinputs"),
            'BB_GENTOO_AUTOTEST_LOC': '/scratch/pjtest/gentoo-autotest',
            'BB_SLURM_PARTITION': 'anywhere',
            'BB_SLURM_NODE_DIR': '/local/hdd/buildbot-polyjit/',
            'BB_SLURM_ACCOUNT': 'anywhere',
            'BB_SLURM_TIMELIMIT': '24:00:00',
            'BB_CONTAINER_MOUNTS': ip('["%(prop:llvm)s", "%(prop:bb_src)s"]'),
            'BB_CONTAINER_PREFIXES': '["/opt/benchbuild", "/", "/usr", "/usr/local"]',
            'BB_ENV_PATH':
                ip('["%(prop:llvm)s/bin", "/scratch/pjtest/erlent/build"]'),
            'BB_ENV_LD_LIBRARY_PATH':
                ip('["%(prop:llvm)s/lib", "/scratch/pjtest/erlent/build"]'),
            'BB_LLVM_DIR': ip('%(prop:scratch)s/llvm'),
            'BB_LIKWID_PREFIX': '/usr',
            'BB_PAPI_INCLUDE': '/usr/include',
            'BB_PAPI_LIBRARY': '/usr/lib',
            'BB_SRC_DIR': ip('%(prop:bb_src)s'),
            'BB_SLURM_LOGS': ip('%(prop:scratch)s/slurm.log'),
            'BB_UNIONFS_ENABLE': 'false'
            },
            workdir=P('scratch')),
        # This only works on infosun machines
        cmd("ln", "-s", ip("/scratch/pjtest/benchbuild-src/"),
            ip("%(prop:bb_src)s")),
        mkdir(ip("%(prop:scratch)s/results"))
    ])

    c['builders'].append(
        builder("polyjit-superbuild", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(sb_steps)))
    c['builders'].append(
        builder("polyjit-superbuild-slurm", None,
                ACCEPTED_BUILDERS, tags=['polyjit'],
                collapseRequests=True,
                factory=BF(slurm_steps)))

def schedule(c):
    superbuild_sched = s_abranch("bs_polyjit-superbuild",
                                 CODEBASE, ["polyjit-superbuild"],
                                 treeStableTimer=10 * 60)
    c['schedulers'].extend([
        superbuild_sched,
        s_force("fs_polyjit-superbuild", FORCE_CODEBASE,
                ["polyjit-superbuild"]),
        s_trigger("ts_polyjit-superbuild", CODEBASE,
                  ["polyjit-superbuild"]),

        s_dependent("ds_polyjit-superbuild-slurm",
                    superbuild_sched,
                    ["polyjit-superbuild-slurm"]),
        s_force("fs_polyjit-superbuild-slurm", FORCE_CODEBASE,
                ["polyjit-superbuild-slurm"])
    ])

register(sys.modules[__name__])
<|endoftext|>


<commit_before>#!/usr/bin/env python

from t2t.consensus import get_consensus_stats, taxa_score, hash_cons, \
    taxa_score_hash, merge_taxa_strings_and_scores
from unittest import TestCase, main
from numpy import array, array_equal

class ConsensusTests(TestCase):

    def setUp(self):
        pass

    def test_merge_taxa_strings_and_scores(self):
        """merge taxa scores and the strings"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        scores = taxa_score(master, [rep1, rep2, rep3, rep4])
        exp = {
            'a': [('k__k1', 1.0), ('p__p1', 1.0), ('c__c1', 1.0),
                  ('o__o1', 1.0), ('f__f1', 1.0), ('g__g1', 1.0),
                  ('s__s1', 1.0)],
            'b': [('k__k1', 1.0), ('p__p1', 1.0), ('c__c2', 0.75), (None, 1.0),
                  ('f__f2', 1.0), ('g__g1', 1.0), ('s__s2', 0.75)],
            'c': [(None, 1.0), (None, 1.0), (None, 1.0), (None, 1.0),
                  (None, 1.0), (None, 1.0), (None, 1.0)],
            'd': [('k__k2', 0.75), ('p__x', 0.25), ('c__c1', 1.0),
                  ('o__o10', 0.25), ('f__f2', 1.0), ('g__g2', 1.0),
                  ('s__s3', 1.0)]}
        obs = merge_taxa_strings_and_scores(master, scores)
        self.assertEqual(obs, exp)

    def test_taxa_score(self):
        """score taxa strings in the face of replicates"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        # if malformed, as in rep3 rep4 for d, count as contridiction
        exp = {'a': array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),
               'b': array([1.0, 1.0, 0.75, 1.0, 1.0, 1.0, 0.75]),
               'c': array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),
               'd': array([0.75, 0.25, 1.0, 0.25, 1.0, 1.0, 1.0])}
        obs = taxa_score(master, [rep1, rep2, rep3, rep4])
        self.assertEqual(obs.keys(), exp.keys())
        for k in exp:
            self.assertTrue(array_equal(obs[k], exp[k]))

    def test_taxa_score_hash(self):
        """test hash based consensus scoring"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        # if malformed, as in rep3 rep4 for d, count as contridiction
        exp = {'a': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
               'b': [1.0, 1.0, 0.75, 1.0, 1.0, 1.0, 0.75],
               'c': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
               'd': [0.75, 0.25, 1.0, 0.25, 1.0, 1.0, 1.0]}
        obs = taxa_score_hash(master, [rep1, rep2, rep3, rep4])
        self.assertEqual(obs.keys(), exp.keys())
        for k in exp:
            self.assertTrue(array_equal(obs[k], exp[k]))

    def test_hash_cons(self):
        """test turning consensus strings into hashes"""
        input = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        exp = array([*map(hash, input['a']),
                     *map(hash, input['d']),
                     *map(hash, input['c']),
                     *map(hash, input['b'])])
        obs = hash_cons(input, ['a', 'd', 'c', 'b'], 7)
        self.assertTrue(array_equal(obs.flatten(), exp))

        exp = array([*map(hash, input['a']),
                     *map(hash, input['d']),
                     *map(hash, [0, 0, 0, 0, 0, 0, 0]),
                     *map(hash, input['c']),
                     *map(hash, input['b'])])
        obs = hash_cons(input, ['a', 'd', 'e', 'c', 'b'], 7)
        self.assertTrue(array_equal(obs.flatten(), exp))

    def test_get_consensus_stats(self):
        """Produces the correct stats"""
        input = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        exp_nseqs = {
            'k': (3, 1), 'p': (2, 2), 'c': (3, 1), 'o': (1, 3), 'f': (3, 1),
            'g': (3, 1), 's': (3, 1)}
        exp_names = {'k': set(['k__k1', 'k__k2']), 'p': set(['p__p1']),
                     'c': set(['c__c1', 'c__c2']), 'o': set(['o__o1']),
                     'f': set(['f__f1', 'f__f2']), 'g': set(['g__g1',
                                                             'g__g2']),
                     's': set(['s__s1', 's__s2', 's__s3'])}
        exp_names = {'k':2,'p':1,'c':2,'o':1,'f':2,'g':2,'s':3}
        obs_nseqs, obs_names = get_consensus_stats(input)
        self.assertEqual(obs_nseqs, exp_nseqs)
        self.assertEqual(obs_names, exp_names)

if __name__ == '__main__':
    main()
<commit_msg>leave it outcommented<commit_after>#!/usr/bin/env python

from t2t.consensus import get_consensus_stats, taxa_score, hash_cons, \
    taxa_score_hash, merge_taxa_strings_and_scores
from unittest import TestCase, main
from numpy import array, array_equal

class ConsensusTests(TestCase):

    def setUp(self):
        pass

    def test_merge_taxa_strings_and_scores(self):
        """merge taxa scores and the strings"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        scores = taxa_score(master, [rep1, rep2, rep3, rep4])
        exp = {
            'a': [('k__k1', 1.0), ('p__p1', 1.0), ('c__c1', 1.0),
                  ('o__o1', 1.0), ('f__f1', 1.0), ('g__g1', 1.0),
                  ('s__s1', 1.0)],
            'b': [('k__k1', 1.0), ('p__p1', 1.0), ('c__c2', 0.75), (None, 1.0),
                  ('f__f2', 1.0), ('g__g1', 1.0), ('s__s2', 0.75)],
            'c': [(None, 1.0), (None, 1.0), (None, 1.0), (None, 1.0),
                  (None, 1.0), (None, 1.0), (None, 1.0)],
            'd': [('k__k2', 0.75), ('p__x', 0.25), ('c__c1', 1.0),
                  ('o__o10', 0.25), ('f__f2', 1.0), ('g__g2', 1.0),
                  ('s__s3', 1.0)]}
        obs = merge_taxa_strings_and_scores(master, scores)
        self.assertEqual(obs, exp)

    def test_taxa_score(self):
        """score taxa strings in the face of replicates"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        # if malformed, as in rep3 rep4 for d, count as contridiction
        exp = {'a': array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),
               'b': array([1.0, 1.0, 0.75, 1.0, 1.0, 1.0, 0.75]),
               'c': array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]),
               'd': array([0.75, 0.25, 1.0, 0.25, 1.0, 1.0, 1.0])}
        obs = taxa_score(master, [rep1, rep2, rep3, rep4])
        self.assertEqual(obs.keys(), exp.keys())
        for k in exp:
            self.assertTrue(array_equal(obs[k], exp[k]))

    def test_taxa_score_hash(self):
        """test hash based consensus scoring"""
        master = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        rep1 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep2 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s9'],
            'c': [None, None, None, None, None, None, None]}
        rep3 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'd': ['k__k2', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        rep4 = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c10', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k1', 'p__', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        # if malformed, as in rep3 rep4 for d, count as contridiction
        exp = {'a': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
               'b': [1.0, 1.0, 0.75, 1.0, 1.0, 1.0, 0.75],
               'c': [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
               'd': [0.75, 0.25, 1.0, 0.25, 1.0, 1.0, 1.0]}
        obs = taxa_score_hash(master, [rep1, rep2, rep3, rep4])
        self.assertEqual(obs.keys(), exp.keys())
        for k in exp:
            self.assertTrue(array_equal(obs[k], exp[k]))

    def test_hash_cons(self):
        """test turning consensus strings into hashes"""
        input = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', 'p__x', 'c__c1', 'o__o10', 'f__f2', 'g__g2',
                  's__s3']}
        exp = array([*map(hash, input['a']),
                     *map(hash, input['d']),
                     *map(hash, input['c']),
                     *map(hash, input['b'])])
        obs = hash_cons(input, ['a', 'd', 'c', 'b'], 7)
        self.assertTrue(array_equal(obs.flatten(), exp))

        exp = array([*map(hash, input['a']),
                     *map(hash, input['d']),
                     *map(hash, [0, 0, 0, 0, 0, 0, 0]),
                     *map(hash, input['c']),
                     *map(hash, input['b'])])
        obs = hash_cons(input, ['a', 'd', 'e', 'c', 'b'], 7)
        self.assertTrue(array_equal(obs.flatten(), exp))

    def test_get_consensus_stats(self):
        """Produces the correct stats"""
        input = {
            'a': ['k__k1', 'p__p1', 'c__c1', 'o__o1', 'f__f1', 'g__g1',
                  's__s1'],
            'b': ['k__k1', 'p__p1', 'c__c2', None, 'f__f2', 'g__g1', 's__s2'],
            'c': [None, None, None, None, None, None, None],
            'd': ['k__k2', '', 'c__c1', 'f__f2', 'f__f2', 'g__g2', 's__s3']}
        exp_nseqs = {
            'k': (3, 1), 'p': (2, 2), 'c': (3, 1), 'o': (1, 3), 'f': (3, 1),
            'g': (3, 1), 's': (3, 1)}
        exp_names = {'k': set(['k__k1', 'k__k2']), 'p': set(['p__p1']),
                     'c': set(['c__c1', 'c__c2']), 'o': set(['o__o1']),
                     'f': set(['f__f1', 'f__f2']), 'g': set(['g__g1',
                                                             'g__g2']),
                     's': set(['s__s1', 's__s2', 's__s3'])}
        #  exp_names = {'k':2,'p':1,'c':2,'o':1,'f':2,'g':2,'s':3}
        obs_nseqs, obs_names = get_consensus_stats(input)
        self.assertEqual(obs_nseqs, exp_nseqs)
        self.assertEqual(obs_names, exp_names)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># For requests to external APIs, timeout after 3 seconds:
API_REQUESTS_TIMEOUT = 3.05
<commit_msg>Increase the PMG API timeout, since it's slow at the moment<commit_after># For requests to external APIs, timeout after 3 seconds:
API_REQUESTS_TIMEOUT = 6.05
<|endoftext|>


<commit_before>from __future__ import unicode_literals, print_function
import requests

GROUPS = (
    ('fictional-character', 'https://wdq.wmflabs.org/api?q=claim[31:95074]'),
    ('philosophers', 'https://wdq.wmflabs.org/api?q=claim[106:4964182]'),
    ('warriors', 'https://wdq.wmflabs.org/api?q=claim[106:1250916]'),
    ('explorers', 'https://wdq.wmflabs.org/api?q=claim[106:11900058]'),
    ('monarchs', 'https://wdq.wmflabs.org/api?q=claim[39:116]'),
)

def get_label(item_id):
    if isinstance(item_id, int):
        item_id = "Q{}".format(item_id)
    url = "https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&languages=en&props=labels&ids={}".format(item_id)
    data = requests.get(url).json()['entities'][item_id]
    return data.get('labels',{}).get('en', {}).get('value')

import re
NON_ASCII = re.compile(r"[^A-Za-z0-9\- ]")
def filter_name(name):
    return not NON_ASCII.search(name)

def main():
    for title, url in GROUPS:
        items = requests.get(url).json()['items']
        for item in items:
            label = get_label(item)
            if label:
                print("{}:{}:{}:{}".format(title, item, 'simple' if filter_name(label) else 'complex', label))

if __name__ == "__main__":
    main()
<commit_msg>keep http connection<commit_after>from __future__ import unicode_literals, print_function
import requests
from requests.adapters import HTTPAdapter

GROUPS = (
    ('fictional-character', 'https://wdq.wmflabs.org/api?q=claim[31:95074]'),
    ('philosophers', 'https://wdq.wmflabs.org/api?q=claim[106:4964182]'),
    ('warriors', 'https://wdq.wmflabs.org/api?q=claim[106:1250916]'),
    ('explorers', 'https://wdq.wmflabs.org/api?q=claim[106:11900058]'),
    ('monarchs', 'https://wdq.wmflabs.org/api?q=claim[39:116]'),
)

session = requests.Session()
session.mount('https://www.wikidata.org', HTTPAdapter(max_retries=42))
session.mount('https://wdq.wmflabs.org', HTTPAdapter(max_retries=42))

def get_label(item_id):
    if isinstance(item_id, int):
        item_id = "Q{}".format(item_id)
    url = "https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&languages=en&props=labels&ids={}".format(item_id)
    data = session.get(url).json()['entities'][item_id]
    return data.get('labels',{}).get('en', {}).get('value')

import re
NON_ASCII = re.compile(r"[^A-Za-z0-9\- ]")
def filter_name(name):
    return not NON_ASCII.search(name)

def main():
    for title, url in GROUPS:
        items = session.get(url).json()['items']
        for item in items:
            label = get_label(item)
            if label:
                print("{}:{}:{}:{}".format(title, item, 'simple' if filter_name(label) else 'complex', label).encode('utf-8'))

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from dateutil import parser as date_parser
import pytest
import pytz
from web_test_base import *

class TestIATIDashboard(WebTestBase):
    requests_to_load = {
        'Dashboard Homepage': {
            'url': 'http://dashboard.iatistandard.org/'
        },
        'Page Generated Late in Process': {
            'url': 'http://dashboard.iatistandard.org/element/iati-activity_result_description_narrative.html'
        }
    }

    def test_contains_links(self, loaded_request):
        """
        Test that each page contains links to the defined URLs.
        """
        result = utility.get_links_from_page(loaded_request)

        assert "https://github.com/IATI/IATI-Dashboard/" in result

    @pytest.mark.xfail(strict=True)
    def test_recently_generated(self, loaded_request):
        """
        Tests that the dashboard was generated in the past 7 days.
        """
        max_delay = timedelta(days=7)
        generation_time_xpath = '//*[@id="footer"]/div/p/em[1]'
        data_time_xpath = '//*[@id="footer"]/div/p/em[2]'

        generation_time_arr = utility.get_text_from_xpath(loaded_request, generation_time_xpath)
        data_time_arr = utility.get_text_from_xpath(loaded_request, data_time_xpath)

        generation_time = date_parser.parse(generation_time_arr[0])
        data_time = date_parser.parse(data_time_arr[0])
        now = datetime.now(pytz.utc)

        assert len(generation_time_arr) == 1
        assert len(data_time_arr) == 1
        assert (now - max_delay) < generation_time
        assert (now - max_delay) < data_time
<commit_msg>Update test_dashboard.py<commit_after>from datetime import datetime, timedelta
from dateutil import parser as date_parser
import pytest
import pytz
from web_test_base import *

class TestIATIDashboard(WebTestBase):
    requests_to_load = {
        'Dashboard Homepage': {
            'url': 'http://dashboard.iatistandard.org/'
        },
        'Page Generated Late in Process': {
            'url': 'http://dashboard.iatistandard.org/element/iati-activity_result_description_narrative.html'
        }
    }

    def test_contains_links(self, loaded_request):
        """
        Test that each page contains links to the defined URLs.
        """
        result = utility.get_links_from_page(loaded_request)

        assert "https://github.com/IATI/IATI-Dashboard/" in result

    def test_recently_generated(self, loaded_request):
        """
        Tests that the dashboard was generated in the past 7 days.
        """
        max_delay = timedelta(days=7)
        generation_time_xpath = '//*[@id="footer"]/div/p/em[1]'
        data_time_xpath = '//*[@id="footer"]/div/p/em[2]'

        generation_time_arr = utility.get_text_from_xpath(loaded_request, generation_time_xpath)
        data_time_arr = utility.get_text_from_xpath(loaded_request, data_time_xpath)

        generation_time = date_parser.parse(generation_time_arr[0])
        data_time = date_parser.parse(data_time_arr[0])
        now = datetime.now(pytz.utc)

        assert len(generation_time_arr) == 1
        assert len(data_time_arr) == 1
        assert (now - max_delay) < generation_time
        assert (now - max_delay) < data_time
<|endoftext|>


<commit_before>def _check_for_script(data):
    print(data)
    for key, value in data.items():
        if value:
            if isinstance(value, dict):
                if _check_for_script(value):
                    return True
            elif isinstance(value, str):
                if "<script>" in value:
                    return True
            elif isinstance(value, list):
                for x in value:
                    if isinstance(x, str):
                        if "<script>" in value:
                            return True
    return False
<commit_msg>Update common.py<commit_after>def _check_for_script(data):
    for key, value in data.items():
        if value:
            if isinstance(value, dict):
                if _check_for_script(value):
                    return True
            elif isinstance(value, str):
                if "<script>" in value:
                    return True
            elif isinstance(value, list):
                for x in value:
                    if isinstance(x, str):
                        if "<script>" in value:
                            return True
    return False
<|endoftext|>


<commit_before>import vx
from enum import Enum
from sys import argv
from functools import partial

vx.register_key = lambda x: None
vx.my_vx = lambda: None

_keys = {
    'langle':       '<', 'rangle':   '>',
    'lparen':       '(', 'rparen':   ')',
    'lbrace':       '{', 'rbrace':   '}',
    'lbracket':     '[', 'rbracket': ']',
    'grave':        '`',
    'backtick':     '`',
    'tilde':        '~',
    'bang':         '!',
    'exclamation':  '!',
    'at':           '@',
    'hash':         '#',
    'dollar':       '$',
    'percent':      '%',
    'carrot':       '^',
    'carat':        '^',
    'and':          '&',
    'ampersand':    '&',
    'star':         '*',
    'asterisk':     '*',
    'hyphen':       '-',
    'dash':         '-',
    'minus':        '-',
    'underscore':   '_',
    'equals':       '=',
    'equal':        '=',
    'plus':         '+',
    'pipe':         '|',
    'backslash':    '\\',
    'forwardslash': '/',
    'slash':        '/',
    'quote':        '"',
    'apostrophe':   '\'',
    'question':     '?',
    'dot':          '.',
    'period':       '.',
    'comma':        ',',
    'backspace':    chr(127),
    'enter':        chr(13),
    'return':       chr(13), # this one is funky
    'escape':       chr(27),
}

for x in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789':
    _keys[x] = x

Keys = Enum('Keys', _keys)

class _keybinding:
    def __init__(self, key, printable):
        if isinstance(key, Keys):
            self.key = key.value
        else:
            self.key = key
        self.printable = printable

    def __str__(self):
        return str(self.key)

    def __sub__(self, other):
        return self + _keyseparator() + other

class _keymodifier:
    def __init__(self, mod, printable):
        self.mod = mod
        self.printable = printable

    def __add__(self, other):
        if not isinstance(other, _keybinding):
            other = _keybinding(other, str(other))
        other.key = self.mod(other.key)
        other.printable = self.printable + '-' + other.printable
        return other

    __radd__ = __add__

_ctrl = _keymodifier(lambda c: chr(0x1f & ord(c)), 'C')
_alt =  _keymodifier(lambda c: chr(0x80 | ord(c)), 'M')

class _keyseparator:
    def __init__(self):
        self.left = None
        self.right = None

    def __add__(self, other):
        if isinstance(other, Keys):
            other = _keybinding(other, str(other))
        if type(other) is _keyseparator:
            other.left = self
            return other
        if self.right:
            self.right += other
        else:
            self.right = other
        return self

    def __radd__(self, other):
        if type(other) is str:
            other = _keybinding(other)
        if self.left:
            self.left += other
        else:
            self.left = other
        return self

    def __sub__(self, other):
        if isinstance(other, Keys):
            other = _keybinding(other, other)
        return self + _keyseparator() + other

    def __str__(self):
        ret = ''
        if self.left:
            ret += str(self.left)
        ret += ' '
        if self.right:
            ret += str(self.right)
        return ret

def _tobinding(s):
    '''Convert a key string (C-o) to a keycode.'''
    class donothing:
        def __add__(self, other): return other
    binding = donothing()
    for c,n in zip(s,s[1:]+' '):
        if c == '-':
            continue
        elif c == 'C' and n == '-':
            c = _ctrl
        elif c == 'M' and n == '-':
            c = _alt
        elif c == ' ':
            c = _keyseparator()
        binding = binding + c
    return binding

_keyscotch = {}
_current_square = _keyscotch

def _bind(keys, command=None):
    """Bind a key to a command. Can be used as a decorator"""
    if command is None:
        def wrapper(func):
            _bind(keys, func)
            return func
        return wrapper
    keys = str(keys)
    if type(keys) is str:
        squares = list(map(lambda x: str(_tobinding(x)), keys.split(' ')))
    else:
        squares = [keys]
    prehops = squares[0:-1]
    finalhop = squares[-1]
    cur = _keyscotch
    for h in prehops:
        if cur.get(h) is None:
            cur[h] = {}
        cur = cur[h]
    cur[finalhop] = command

def _quick_bind(key):
    '''Bind a keycode to insert itself as text.'''
    _bind(key, partial(vx.add_string, key))

# Quick-bind letter keys
for i in range(26):
    char = chr(ord('a') + i)
    _quick_bind(char)

    char = chr(ord('A') + i)
    _quick_bind(char)

# ...number keys
for i in range(10):
    _quick_bind(str(i))

# ...symbols
for char in ['?', '<', '>', '\'', '/', '"', ':',
             ';', '.', ',', '!', '@', '#', '$',
             '%', '^', '&', '*', '(', ')', '-',
             '_', '+', '=', '\\', '|', '`', '~',
             ' ']:
    _quick_bind(char)

# ...return/backspace
_bind(chr(13), partial(vx.add_string, '\n'))
_bind(chr(127), vx.backspace)

def _repeat(c, times=4):
    for _ in range(times):
        c()

class _window:
    def __init__(self, rows, columns, x, y):
        self._c_window = vx.new_window(rows, columns, x, y)

    def attach_file(self, filename):
        vx.attach_window(self._c_window, filename)

    def blank(self):
        vx.attach_window_blank(self._c_window)

    def focus(self):
        vx.focus_window(self._c_window)

    def update(self):
        vx.update_window(self._c_window)

_windows = []

_current_window = -1
def _next_window():
    global _current_window
    _current_window += 1
    if _current_window >= len(_windows):
        _current_window = 0
    _windows[_current_window].focus()

vx.next_window = _next_window
vx.windows = _windows
vx.files = argv[1:]
vx.window = _window
vx.keys = Keys
vx.repeat = _repeat
vx.tobinding = _tobinding
vx.bind = _bind
vx.ctrl = _ctrl
vx.alt = _alt

def _base_register_key(key):
    global _current_square
    _current_square = _current_square.get(key)
    if callable(_current_square):
        _current_square()
        _current_square = _keyscotch
    elif _current_square is None:
        _current_square = _keyscotch
        raise Exception('not found')
        return False
    return True

_key_callbacks = []
_key_callbacks.append(_base_register_key)
vx.key_callbacks = _key_callbacks

def _register_key(key):
    for c in _key_callbacks:
        if c(key): break
vx.register_key = _register_key
<commit_msg>Remove unnecessary __radd__<commit_after>import vx
from enum import Enum
from sys import argv
from functools import partial

vx.register_key = lambda x: None
vx.my_vx = lambda: None

_keys = {
    'langle':       '<', 'rangle':   '>',
    'lparen':       '(', 'rparen':   ')',
    'lbrace':       '{', 'rbrace':   '}',
    'lbracket':     '[', 'rbracket': ']',
    'grave':        '`',
    'backtick':     '`',
    'tilde':        '~',
    'bang':         '!',
    'exclamation':  '!',
    'at':           '@',
    'hash':         '#',
    'dollar':       '$',
    'percent':      '%',
    'carrot':       '^',
    'carat':        '^',
    'and':          '&',
    'ampersand':    '&',
    'star':         '*',
    'asterisk':     '*',
    'hyphen':       '-',
    'dash':         '-',
    'minus':        '-',
    'underscore':   '_',
    'equals':       '=',
    'equal':        '=',
    'plus':         '+',
    'pipe':         '|',
    'backslash':    '\\',
    'forwardslash': '/',
    'slash':        '/',
    'quote':        '"',
    'apostrophe':   '\'',
    'question':     '?',
    'dot':          '.',
    'period':       '.',
    'comma':        ',',
    'backspace':    chr(127),
    'enter':        chr(13),
    'return':       chr(13), # this one is funky
    'escape':       chr(27),
}

for x in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789':
    _keys[x] = x

Keys = Enum('Keys', _keys)

class _keybinding:
    def __init__(self, key, printable):
        if isinstance(key, Keys):
            self.key = key.value
        else:
            self.key = key
        self.printable = printable

    def __str__(self):
        return str(self.key)

    def __sub__(self, other):
        return self + _keyseparator() + other

class _keymodifier:
    def __init__(self, mod, printable):
        self.mod = mod
        self.printable = printable

    def __add__(self, other):
        if not isinstance(other, _keybinding):
            other = _keybinding(other, str(other))
        other.key = self.mod(other.key)
        other.printable = self.printable + '-' + other.printable
        return other

_ctrl = _keymodifier(lambda c: chr(0x1f & ord(c)), 'C')
_alt =  _keymodifier(lambda c: chr(0x80 | ord(c)), 'M')

class _keyseparator:
    def __init__(self):
        self.left = None
        self.right = None

    def __add__(self, other):
        if isinstance(other, Keys):
            other = _keybinding(other, str(other))
        if type(other) is _keyseparator:
            other.left = self
            return other
        if self.right:
            self.right += other
        else:
            self.right = other
        return self

    def __radd__(self, other):
        if type(other) is str:
            other = _keybinding(other)
        if self.left:
            self.left += other
        else:
            self.left = other
        return self

    def __sub__(self, other):
        if isinstance(other, Keys):
            other = _keybinding(other, other)
        return self + _keyseparator() + other

    def __str__(self):
        ret = ''
        if self.left:
            ret += str(self.left)
        ret += ' '
        if self.right:
            ret += str(self.right)
        return ret

def _tobinding(s):
    '''Convert a key string (C-o) to a keycode.'''
    class donothing:
        def __add__(self, other): return other
    binding = donothing()
    for c,n in zip(s,s[1:]+' '):
        if c == '-':
            continue
        elif c == 'C' and n == '-':
            c = _ctrl
        elif c == 'M' and n == '-':
            c = _alt
        elif c == ' ':
            c = _keyseparator()
        binding = binding + c
    return binding

_keyscotch = {}
_current_square = _keyscotch

def _bind(keys, command=None):
    """Bind a key to a command. Can be used as a decorator"""
    if command is None:
        def wrapper(func):
            _bind(keys, func)
            return func
        return wrapper
    keys = str(keys)
    if type(keys) is str:
        squares = list(map(lambda x: str(_tobinding(x)), keys.split(' ')))
    else:
        squares = [keys]
    prehops = squares[0:-1]
    finalhop = squares[-1]
    cur = _keyscotch
    for h in prehops:
        if cur.get(h) is None:
            cur[h] = {}
        cur = cur[h]
    cur[finalhop] = command

def _quick_bind(key):
    '''Bind a keycode to insert itself as text.'''
    _bind(key, partial(vx.add_string, key))

# Quick-bind letter keys
for i in range(26):
    char = chr(ord('a') + i)
    _quick_bind(char)

    char = chr(ord('A') + i)
    _quick_bind(char)

# ...number keys
for i in range(10):
    _quick_bind(str(i))

# ...symbols
for char in ['?', '<', '>', '\'', '/', '"', ':',
             ';', '.', ',', '!', '@', '#', '$',
             '%', '^', '&', '*', '(', ')', '-',
             '_', '+', '=', '\\', '|', '`', '~',
             ' ']:
    _quick_bind(char)

# ...return/backspace
_bind(chr(13), partial(vx.add_string, '\n'))
_bind(chr(127), vx.backspace)

def _repeat(c, times=4):
    for _ in range(times):
        c()

class _window:
    def __init__(self, rows, columns, x, y):
        self._c_window = vx.new_window(rows, columns, x, y)

    def attach_file(self, filename):
        vx.attach_window(self._c_window, filename)

    def blank(self):
        vx.attach_window_blank(self._c_window)

    def focus(self):
        vx.focus_window(self._c_window)

    def update(self):
        vx.update_window(self._c_window)

_windows = []

_current_window = -1
def _next_window():
    global _current_window
    _current_window += 1
    if _current_window >= len(_windows):
        _current_window = 0
    _windows[_current_window].focus()

vx.next_window = _next_window
vx.windows = _windows
vx.files = argv[1:]
vx.window = _window
vx.keys = Keys
vx.repeat = _repeat
vx.tobinding = _tobinding
vx.bind = _bind
vx.ctrl = _ctrl
vx.alt = _alt

def _base_register_key(key):
    global _current_square
    _current_square = _current_square.get(key)
    if callable(_current_square):
        _current_square()
        _current_square = _keyscotch
    elif _current_square is None:
        _current_square = _keyscotch
        raise Exception('not found')
        return False
    return True

_key_callbacks = []
_key_callbacks.append(_base_register_key)
vx.key_callbacks = _key_callbacks

def _register_key(key):
    for c in _key_callbacks:
        if c(key): break
vx.register_key = _register_key
<|endoftext|>


<commit_before>"""
Django settings for example project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import sys
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
sys.path.append(os.path.join(BASE_DIR, ".."))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '#^qtdm4ad9_44k+pf+2^ecrm(w9j@w(+s(^e$@s8l=zq%pqtwl'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
    'pimpmytheme',
    'example',
    'subapp',
    'compressor',
    'django_nose'
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'example.urls'

WSGI_APPLICATION = 'example.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_ROOT = os.path.join(BASE_DIR, "example", "static")
STATIC_URL = '/static/'

SITE_ID = 1

COMPRESS_PRECOMPILERS = (('text/less', 'lessc {infile} {outfile}'),)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'context_processors': [
                "django.contrib.auth.context_processors.auth",
                "django.template.context_processors.request",
                "django.template.context_processors.static",
                "pimpmytheme.context_processors.get_site",
            ],
            'builtins': [
                'django.templatetags.i18n',
                'django.templatetags.static',
                'django.templatetags.tz',
            ],
            'loaders': [
                'pimpmytheme.template_loader.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        }
    },
]

STATICFILES_FINDERS = (
    "pimpmytheme.static_finder.CustomFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
    "compressor.finders.CompressorFinder"
    )

NOSE_ARGS = [
    '--with-coverage',
    '--cover-package=pimpmytheme',
    '--verbosity=3',
    '--nocapture'
    ]

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CUSTOM_THEME_LOOKUP_OBJECT = "example.models.PimpSite"
CUSTOM_THEME_LOOKUP_ATTR = "name"
PIMPMYTHEME_FOLDER = os.path.join(BASE_DIR, "pimp_theme")

LOGGING = {
    'version': 1,
    'formatters': {
        'oneline': {
            'format': '%(asctime)s %(levelname)-8s %(name)s  %(message)s',
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'oneline',
        },
    },
    'root': {
        'level': 'DEBUG',
        'handlers': ['console'],
    },
}
<commit_msg>Update example settings<commit_after>"""
Django settings for example project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import sys
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
sys.path.append(os.path.join(BASE_DIR, ".."))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '#^qtdm4ad9_44k+pf+2^ecrm(w9j@w(+s(^e$@s8l=zq%pqtwl'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.sites',
    'pimpmytheme',
    'example',
    'subapp',
    'compressor',
    'django_nose'
)

MIDDLEWARE = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'example.urls'

WSGI_APPLICATION = 'example.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_ROOT = os.path.join(BASE_DIR, "example", "static")
STATIC_URL = '/static/'

SITE_ID = 1

COMPRESS_PRECOMPILERS = (('text/less', 'lessc {infile} {outfile}'),)

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'OPTIONS': {
            'context_processors': [
                "django.contrib.auth.context_processors.auth",
                "django.template.context_processors.request",
                "django.template.context_processors.static",
                "pimpmytheme.context_processors.get_site",
            ],
            'builtins': [
                'django.templatetags.i18n',
                'django.templatetags.static',
                'django.templatetags.tz',
            ],
            'loaders': [
                'pimpmytheme.template_loader.Loader',
                'django.template.loaders.app_directories.Loader',
            ],
        }
    },
]

STATICFILES_FINDERS = (
    "pimpmytheme.static_finder.CustomFinder",
    "django.contrib.staticfiles.finders.AppDirectoriesFinder",
    "compressor.finders.CompressorFinder"
    )

NOSE_ARGS = [
    '--with-coverage',
    '--cover-package=pimpmytheme',
    '--verbosity=3',
    '--nocapture'
    ]

TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
CUSTOM_THEME_LOOKUP_OBJECT = "example.models.PimpSite"
CUSTOM_THEME_LOOKUP_ATTR = "name"
PIMPMYTHEME_FOLDER = os.path.join(BASE_DIR, "pimp_theme")

LOGGING = {
    'version': 1,
    'formatters': {
        'oneline': {
            'format': '%(asctime)s %(levelname)-8s %(name)s  %(message)s',
        },
    },
    'handlers': {
        'console': {
            'class': 'logging.StreamHandler',
            'formatter': 'oneline',
        },
    },
    'root': {
        'level': 'DEBUG',
        'handlers': ['console'],
    },
}
<|endoftext|>


<commit_before>"""
Test Odin extensions to distarray.

To run these tests, you must have an ipcluster running
For example,

    $ ipcluster start -n <n> --engines=MPIEngineSetLauncher
"""

import unittest
import numpy as np

from distarray import odin
from distarray.client import DistArrayContext

@odin.local
def assert_allclose(da, db):
    assert np.allclose(da, db), "Arrays not equal within tolerance."

@odin.local
def local_sin(da):
    return np.sin(da)

@odin.local
def local_add50(da):
    return da + 50

@odin.local
def local_sum(da):
    return np.sum(da.get_localarray())

@odin.local
def local_add_num(da, num):
    return da + num

@odin.local
def local_add_nums(da, num1, num2, num3):
    return da + num1 + num2 + num3

@odin.local
def local_add_distarrayproxies(da, dg):
    return da + dg

@odin.local
def local_add_mixed(da, num1, dg, num2):
    return da + num1 + dg + num2

@odin.local
def local_add_ndarray(da, num, ndarr):
    return da + num + ndarr

@odin.local
def local_add_kwargs(da, num1, num2=55):
    return da + num1 + num2

@odin.local
def local_add_supermix(da, num1, db, num2, dc, num3=99, num4=66):
    return da + num1 + db + num2 + dc + num3 + num4

@odin.local
def local_none(da):
    return None

@odin.local
def call_local(da):
    db = local_add50(da)
    dc = local_add_num(db, 99)
    return dc

class TestLocal(unittest.TestCase):

    def setUp(self):
        odin.context._execute('import numpy as np')
        self.da = odin.context.empty((1024, 1024))
        self.da.fill(2 * np.pi)

    def test_local_sin(self):
        db = local_sin(self.da)
        assert_allclose(db, 0)

    def test_local_add(self):
        dc = local_add50(self.da)
        assert_allclose(dc, 2 * np.pi + 50)

    def test_local_sum(self):
        dd = local_sum(self.da)
        client_dd = np.array(odin.context._pull(dd.key))

        shape = self.da.get_localshapes()[0]
        sum_val = shape[0] * shape[1] * (2 * np.pi)

        assert_allclose(client_dd, sum_val)

    def test_local_add_num(self):
        de = local_add_num(self.da, 11)
        assert_allclose(de, 2 * np.pi + 11)

    def test_local_add_nums(self):
        df = local_add_nums(self.da, 11, 12, 13)
        assert_allclose(df, 2 * np.pi + 11 + 12 + 13)

    def test_local_add_distarrayproxies(self):
        dg = odin.context.empty((1024, 1024))
        dg.fill(33)
        dh = local_add_distarrayproxies(self.da, dg)
        assert_allclose(dh, 33 + 2 * np.pi)

    def test_local_add_mixed(self):
        di = odin.context.empty((1024, 1024))
        di.fill(33)
        dj = local_add_mixed(self.da, 11, di, 12)
        assert_allclose(dj, 2 * np.pi + 11 + 33 + 12)

    @unittest.skip('Locally adding ndarrays not supported.')
    def test_local_add_ndarray(self):
        shp = self.da.get_localshapes()[0]
        ndarr = np.empty(shp)
        ndarr.fill(33)
        dk = local_add_ndarray(self.da, 11, ndarr)
        assert_allclose(dk, 2 * np.pi + 11 + 33)

    def test_local_add_kwargs(self):
        dl = local_add_kwargs(self.da, 11, num2=12)
        assert_allclose(dl, 2 * np.pi + 11 + 12)

    def test_local_add_supermix(self):
        dm = odin.context.empty((1024, 1024))
        dm.fill(22)
        dn = odin.context.empty((1024, 1024))
        dn.fill(44)
        do = local_add_supermix(self.da, 11, dm, 33, dc=dn, num3=55)
        expected = 2 * np.pi + 11 + 22 + 33 + 44 + 55 + 66
        assert_allclose(do, expected)

    @unittest.skip("Doesn't know what to do with NoneType")
    def test_local_none(self):
        dp = local_none(self.da)
        client_dp = odin.context._pull(dp.key)
        print client_dp

    def test_call_local(self):
        dq = call_local(self.da)
        assert_allclose(dq, 2 * np.pi + 50 + 99)

    def test_subcontext(self):
        targets = [0, 2]
        subcontext = DistArrayContext(odin._global_view, targets=targets)
        da = subcontext.empty((1024, 1024))
        da.fill(11)
        de = local_add_num(da, 10)
        assert_allclose(de, 11 + 10)

class TestUtils(unittest.TestCase):

    def test_flatten(self):
        self.assertEqual(odin.flatten(zip([1, 2, 3], [4, 5, 6])),
                                      [1, 4, 2, 5, 3, 6])

    def test_all_equal_false(self):
        self.assertFalse(odin.all_equal([1, 2, 3, 4, 5]))

    def test_all_equal_true(self):
        self.assertTrue(odin.all_equal([7, 7, 7, 7, 7]))

    def test_all_equal_contexts(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        db = subcontext.empty((100,))
        dc = subcontext.ones((100,))
        contexts = (subcontext, db.context, dc.context)
        self.assertTrue(odin.all_equal(contexts))

class TestDetermineContext(unittest.TestCase):

    def test_global_context(self):
        da = odin.context.empty((100,))
        db = odin.context.empty((100,))
        self.assertEqual(odin.determine_context((da, db)), odin.context)

    def test_subcontext(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = subcontext.empty((100,))
        db = subcontext.empty((100,))
        self.assertEqual(odin.determine_context((da, db)), subcontext)

    def test_no_proxies(self):
        self.assertEqual(odin.determine_context((11, 12, 'abc')), odin.context)

    def test_mixed_types(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = subcontext.empty((100,))
        self.assertEqual(odin.determine_context((da, 12, 'abc')), da.context)

    def test_mixed_contexts(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = odin.context.empty((100,))
        db = subcontext.empty((100,))
        self.assertRaises(ValueError, odin.determine_context, (da, db))

if __name__ == '__main__':
    unittest.main(verbosity=2)
<commit_msg>Test MPI calls in a subcontext.<commit_after>"""
Test Odin extensions to distarray.

To run these tests, you must have an ipcluster running
For example,

    $ ipcluster start -n <n> --engines=MPIEngineSetLauncher
"""

import unittest
import numpy as np

from distarray import odin
from distarray.client import DistArrayContext

@odin.local
def assert_allclose(da, db):
    assert np.allclose(da, db), "Arrays not equal within tolerance."

@odin.local
def local_sin(da):
    return np.sin(da)

@odin.local
def local_add50(da):
    return da + 50

@odin.local
def local_sum(da):
    return np.sum(da.get_localarray())

@odin.local
def local_add_num(da, num):
    return da + num

@odin.local
def call_barrier(da):
    from mpi4py import MPI
    MPI.COMM_WORLD.Barrier()
    return da

@odin.local
def local_add_nums(da, num1, num2, num3):
    return da + num1 + num2 + num3

@odin.local
def local_add_distarrayproxies(da, dg):
    return da + dg

@odin.local
def local_add_mixed(da, num1, dg, num2):
    return da + num1 + dg + num2

@odin.local
def local_add_ndarray(da, num, ndarr):
    return da + num + ndarr

@odin.local
def local_add_kwargs(da, num1, num2=55):
    return da + num1 + num2

@odin.local
def local_add_supermix(da, num1, db, num2, dc, num3=99, num4=66):
    return da + num1 + db + num2 + dc + num3 + num4

@odin.local
def local_none(da):
    return None

@odin.local
def call_local(da):
    db = local_add50(da)
    dc = local_add_num(db, 99)
    return dc

class TestLocal(unittest.TestCase):

    def setUp(self):
        odin.context._execute('import numpy as np')
        self.da = odin.context.empty((1024, 1024))
        self.da.fill(2 * np.pi)

    def test_local_sin(self):
        db = local_sin(self.da)
        assert_allclose(db, 0)

    def test_local_add(self):
        dc = local_add50(self.da)
        assert_allclose(dc, 2 * np.pi + 50)

    def test_local_sum(self):
        dd = local_sum(self.da)
        client_dd = np.array(odin.context._pull(dd.key))

        shape = self.da.get_localshapes()[0]
        sum_val = shape[0] * shape[1] * (2 * np.pi)

        assert_allclose(client_dd, sum_val)

    def test_local_add_num(self):
        de = local_add_num(self.da, 11)
        assert_allclose(de, 2 * np.pi + 11)

    def test_local_add_nums(self):
        df = local_add_nums(self.da, 11, 12, 13)
        assert_allclose(df, 2 * np.pi + 11 + 12 + 13)

    def test_local_add_distarrayproxies(self):
        dg = odin.context.empty((1024, 1024))
        dg.fill(33)
        dh = local_add_distarrayproxies(self.da, dg)
        assert_allclose(dh, 33 + 2 * np.pi)

    def test_local_add_mixed(self):
        di = odin.context.empty((1024, 1024))
        di.fill(33)
        dj = local_add_mixed(self.da, 11, di, 12)
        assert_allclose(dj, 2 * np.pi + 11 + 33 + 12)

    @unittest.skip('Locally adding ndarrays not supported.')
    def test_local_add_ndarray(self):
        shp = self.da.get_localshapes()[0]
        ndarr = np.empty(shp)
        ndarr.fill(33)
        dk = local_add_ndarray(self.da, 11, ndarr)
        assert_allclose(dk, 2 * np.pi + 11 + 33)

    def test_local_add_kwargs(self):
        dl = local_add_kwargs(self.da, 11, num2=12)
        assert_allclose(dl, 2 * np.pi + 11 + 12)

    def test_local_add_supermix(self):
        dm = odin.context.empty((1024, 1024))
        dm.fill(22)
        dn = odin.context.empty((1024, 1024))
        dn.fill(44)
        do = local_add_supermix(self.da, 11, dm, 33, dc=dn, num3=55)
        expected = 2 * np.pi + 11 + 22 + 33 + 44 + 55 + 66
        assert_allclose(do, expected)

    @unittest.skip("Doesn't know what to do with NoneType")
    def test_local_none(self):
        dp = local_none(self.da)
        client_dp = odin.context._pull(dp.key)
        print client_dp

    def test_call_local(self):
        dq = call_local(self.da)
        assert_allclose(dq, 2 * np.pi + 50 + 99)

    def test_subcontext(self):
        targets = [0, 2]
        subcontext = DistArrayContext(odin._global_view, targets=targets)
        da = subcontext.empty((1024, 1024))
        da.fill(11)
        de = local_add_num(da, 10)
        assert_allclose(de, 11 + 10)

    def test_barrier(self):
        call_barrier(self.da)

    def test_barrier_with_subcontext(self):
        targets = [0, 2]
        subcontext = DistArrayContext(odin._global_view, targets=targets)
        da = subcontext.empty((1024, 1024))
        da.fill(11)
        call_barrier(da)

class TestUtils(unittest.TestCase):

    def test_flatten(self):
        self.assertEqual(odin.flatten(zip([1, 2, 3], [4, 5, 6])),
                                      [1, 4, 2, 5, 3, 6])

    def test_all_equal_false(self):
        self.assertFalse(odin.all_equal([1, 2, 3, 4, 5]))

    def test_all_equal_true(self):
        self.assertTrue(odin.all_equal([7, 7, 7, 7, 7]))

    def test_all_equal_contexts(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        db = subcontext.empty((100,))
        dc = subcontext.ones((100,))
        contexts = (subcontext, db.context, dc.context)
        self.assertTrue(odin.all_equal(contexts))

class TestDetermineContext(unittest.TestCase):

    def test_global_context(self):
        da = odin.context.empty((100,))
        db = odin.context.empty((100,))
        self.assertEqual(odin.determine_context((da, db)), odin.context)

    def test_subcontext(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = subcontext.empty((100,))
        db = subcontext.empty((100,))
        self.assertEqual(odin.determine_context((da, db)), subcontext)

    def test_no_proxies(self):
        self.assertEqual(odin.determine_context((11, 12, 'abc')), odin.context)

    def test_mixed_types(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = subcontext.empty((100,))
        self.assertEqual(odin.determine_context((da, 12, 'abc')), da.context)

    def test_mixed_contexts(self):
        subcontext = DistArrayContext(odin._global_view, targets=[0, 3])
        da = odin.context.empty((100,))
        db = subcontext.empty((100,))
        self.assertRaises(ValueError, odin.determine_context, (da, db))

if __name__ == '__main__':
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys

sys.path.insert(
    0,
    os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
try:
    import enasearch
except:
    raise

def cmp(la, lb):
    """Compare two lists"""
    return all(s in lb for s in la) and all(s in la for s in lb)

def test_get_results():
    """Test get_results function"""
    results = enasearch.get_results(verbose=False)
    exp_results = [
        'wgs_set', 'analysis_study', 'study', 'read_run', 'coding_release',
        'coding_update', 'analysis', 'environmental', 'tsa_set',
        'sequence_update', 'noncoding_update', 'noncoding_release', 'sample',
        'read_experiment', 'read_study', 'assembly', 'taxon',
        'sequence_release']
    assert len(results) == 18 and cmp(results.keys(), exp_results)

def test_get_search_result_number():
    """Test get_search_result_number function"""
    nb = enasearch.get_search_result_number(
        query="tax_eq(10090)",
        result="assembly",
        need_check_result=True)
    assert nb == 19

def test_get_filter_types():
    """Test get_search_result_number function"""
    filter_types = enasearch.get_filter_types(verbose=False)
    assert "Boolean" in filter_types
    assert "geo_box1" in filter_types["Geospatial"]

<commit_msg>Fix flake8<commit_after>#!/usr/bin/env python

import os
import sys

sys.path.insert(
    0,
    os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
try:
    import enasearch
except:
    raise

def cmp(la, lb):
    """Compare two lists"""
    return all(s in lb for s in la) and all(s in la for s in lb)

def test_get_results():
    """Test get_results function"""
    results = enasearch.get_results(verbose=False)
    exp_results = [
        'wgs_set', 'analysis_study', 'study', 'read_run', 'coding_release',
        'coding_update', 'analysis', 'environmental', 'tsa_set',
        'sequence_update', 'noncoding_update', 'noncoding_release', 'sample',
        'read_experiment', 'read_study', 'assembly', 'taxon',
        'sequence_release']
    assert len(results) == 18 and cmp(results.keys(), exp_results)

def test_get_search_result_number():
    """Test get_search_result_number function"""
    nb = enasearch.get_search_result_number(
        query="tax_eq(10090)",
        result="assembly",
        need_check_result=True)
    assert nb == 19

def test_get_filter_types():
    """Test get_search_result_number function"""
    filter_types = enasearch.get_filter_types(verbose=False)
    assert "Boolean" in filter_types
    assert "geo_box1" in filter_types["Geospatial"]
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
tollgate management command: repair script permissions
Copyright 2008-2012 Michael Farrell

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
import os, stat

class Command(BaseCommand):
	args = ''
	help = 'Repairs the execute permissions of tollgate binaries.'

	def handle(self, *args, **kwargs):
		executable_files = [
			'manage.py',
			os.path.join('backend', 'tollgate.py'),
			os.path.join('backend', 'tollgate.sh'),
		]

		for x in executable_files:
			# chmod the file such that all that have read access have execute as well.
			fp = os.path.join(settings.PROJECT_PATH, x)
			mode = os.stat(fp).st_mode

			if mode & stat.S_IRUSR:
				mode |= stat.S_IXUSR

			if mode & stat.S_IRGRP:
				mode |= stat.S_IXGRP

			if mode & stat.S_IROTH:
				mode |= stat.S_IXOTH

			os.chmod(fp, mode)

		print "Repaired execute permissions."
			<commit_msg>Fix more permissions.<commit_after>#!/usr/bin/env python
"""
tollgate management command: repair script permissions
Copyright 2008-2012 Michael Farrell

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
import os, stat

class Command(BaseCommand):
	args = ''
	help = 'Repairs the execute permissions of tollgate binaries.'

	def handle(self, *args, **kwargs):
		executable_files = [
			'manage.py',
			os.path.join('backend', 'tollgate.py'),
			os.path.join('backend', 'tollgate.sh'),
			os.path.join('debian', 'init.d', 'tollgate-backend'),
			os.path.join('debian', 'init.d', 'tollgate-captivity'),
		]

		for x in executable_files:
			# chmod the file such that all that have read access have execute as well.
			fp = os.path.join(settings.PROJECT_PATH, x)
			mode = os.stat(fp).st_mode

			if mode & stat.S_IRUSR:
				mode |= stat.S_IXUSR

			if mode & stat.S_IRGRP:
				mode |= stat.S_IXGRP

			if mode & stat.S_IROTH:
				mode |= stat.S_IXOTH

			os.chmod(fp, mode)

		print "Repaired execute permissions."
			<|endoftext|>


<commit_before>from __future__ import unicode_literals

import unittest

# from mopidy_ttsgpio import frontend as frontend_lib
from mopidy_ttsgpio import Extension

class ExtensionTest(unittest.TestCase):

    def test_get_default_config(self):
        ext = Extension()

        config = ext.get_default_config()

        self.assertIn('[ttsgpio]', config)
        self.assertIn('enabled = true', config)

    def test_get_config_schema(self):
        ext = Extension()
        schema = ext.get_config_schema()
        self.assertIn('pin_button_main', schema)

        # TODO Test the content of your config schema
        # self.assertIn('username', schema)
        # self.assertIn('password', schema)

    # TODO Write more tests
<commit_msg>Test changes<commit_after>from __future__ import unicode_literals

import mock
import unittest

from mopidy_ttsgpio import frontend
from mopidy_ttsgpio import Extension

class ExtensionTest(unittest.TestCase):

    def test_get_default_config(self):
        ext = Extension()
        config = ext.get_default_config()
        # ext = frontend.TtsGpio(config, mock.sentinel.core)
        self.assertIn('[ttsgpio]', config)
        self.assertIn('enabled = true', config)

    def test_get_config_schema(self):
        ext = Extension()
        schema = ext.get_config_schema()
        self.assertIn('pin_button_main', schema)
        self.assertIn('pin_button_next', schema)
        self.assertIn('pin_button_previous', schema)
        self.assertIn('pin_button_vol_up', schema)
        self.assertIn('pin_button_vol_up', schema)
        self.assertIn('pin_button_vol_down', schema)
<|endoftext|>