<commit_before># coding: utf-8
# Copyright 2009 Alexandre Fiori
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from StringIO import StringIO

from pymongo import objectid, timestamp
import txmongo
from txmongo import database
from txmongo import collection
from txmongo import gridfs
from txmongo import filter as qf
from txmongo._gridfs import GridIn
from twisted.trial import unittest
from twisted.trial import runner
from twisted.internet import base, defer

mongo_host="localhost"
mongo_port=27017
base.DelayedCall.debug = False

class TestMongoObjects(unittest.TestCase):
    @defer.inlineCallbacks
    def test_MongoObjects(self):
        """ Tests creating mongo objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        mydb = conn.mydb
        self.assertEqual(isinstance(mydb, database.Database), True)
        mycol = mydb.mycol
        self.assertEqual(isinstance(mycol, collection.Collection), True)
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_MongoOperations(self):
        """ Tests mongo operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test

        # insert
        doc = {"foo":"bar", "items":[1, 2, 3]}
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.has_key("_id"), True)
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # insert preserves object id
        doc.update({'_id': objectid.ObjectId()})
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.get('_id'), doc.get('_id'))
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # update
        yield test.update({"_id":result["_id"]}, {"$set":{"one":"two"}}, safe=True)
        result = yield test.find_one({"_id":result["_id"]})
        self.assertEqual(result["one"], "two")

        # delete
        yield test.remove(result["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Timestamps(self):
        """Tests mongo operations with Timestamps"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test_ts

        # insert with specific timestamp
        doc1 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(1, 2)}
        yield test.insert(doc1, safe=True)

        result = yield test.find_one(doc1)
        self.assertEqual(result.get('ts').time, 1)
        self.assertEqual(result.get('ts').inc, 2)

        # insert with specific timestamp
        doc2 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(2, 1)}
        yield test.insert(doc2, safe=True)

        # the objects come back sorted by ts correctly.
        # (test that we stored inc/time in the right fields)
        result = yield test.find(filter=qf.sort(qf.ASCENDING('ts')))
        self.assertEqual(result[0]['_id'], doc1['_id'])
        self.assertEqual(result[1]['_id'], doc2['_id'])

        # insert with null timestamp
        doc3 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(0, 0)}
        yield test.insert(doc3, safe=True)

        # time field loaded correctly
        result = yield test.find_one(doc3['_id'])
        now = time.time()
        self.assertTrue(now - 2 <= result['ts'].time <= now)

        # delete
        yield test.remove(doc1["_id"], safe=True)
        yield test.remove(doc2["_id"], safe=True)
        yield test.remove(doc3["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

class TestGridFsObjects(unittest.TestCase):
    """ Test the GridFS operations from txmongo._gridfs """
    @defer.inlineCallbacks
    def _disconnect(self, conn):
        """ Disconnect the connection """
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        collection = db.fs

        gfs = gridfs.GridFS(db) # Default collection

        gridin = GridIn(collection, filename='test', contentType="text/plain",
                        chunk_size=2**2**2**2)
        new_file = gfs.new_file(filename='test2', contentType="text/plain",
                        chunk_size=2**2**2**2)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsOperations(self):
        """ Tests gridfs operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        collection = db.fs

        # Don't forget to disconnect
        self.addCleanup(self._disconnect, conn)
        try:
            in_file = StringIO("Test input string")
            out_file = StringIO()
        except Exception, e:
            self.fail("Failed to create memory files for testing: %s" % e)

        try:
            # Tests writing to a new gridfs file
            gfs = gridfs.GridFS(db) # Default collection
            g_in = gfs.new_file(filename='optest', contentType="text/plain",
                            chunk_size=2**2**2**2) # non-default chunk size used
            # yielding to ensure writes complete before we close and close before we try to read
            yield g_in.write(in_file.read())
            yield g_in.close()

            # Tests reading from an existing gridfs file
            g_out = yield gfs.get_last_version('optest')
            data = yield g_out.read()
            out_file.write(data)
            _id = g_out._id
        except Exception,e:
            self.fail("Failed to communicate with the GridFS. " +
                      "Is MongoDB running? %s" % e)
        else:
            self.assertEqual(in_file.getvalue(), out_file.getvalue(),
                         "Could not read the value from writing an input")
        finally:
            in_file.close()
            out_file.close()
            g_out.close()

        listed_files = yield gfs.list()
        self.assertEqual(['optest'], listed_files,
                         "'optest' is the only expected file and we received %s" % listed_files)

        yield gfs.delete(_id)

if __name__ == "__main__":
    suite = runner.TrialSuite((TestMongoObjects, TestGridFsObjects))
    suite.run()
<commit_msg>tests: Use bson instead of pymongo module.<commit_after># coding: utf-8
# Copyright 2009 Alexandre Fiori
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from StringIO import StringIO

from bson import objectid, timestamp
import txmongo
from txmongo import database
from txmongo import collection
from txmongo import gridfs
from txmongo import filter as qf
from txmongo._gridfs import GridIn
from twisted.trial import unittest
from twisted.trial import runner
from twisted.internet import base, defer

mongo_host="localhost"
mongo_port=27017
base.DelayedCall.debug = False

class TestMongoObjects(unittest.TestCase):
    @defer.inlineCallbacks
    def test_MongoObjects(self):
        """ Tests creating mongo objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        mydb = conn.mydb
        self.assertEqual(isinstance(mydb, database.Database), True)
        mycol = mydb.mycol
        self.assertEqual(isinstance(mycol, collection.Collection), True)
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_MongoOperations(self):
        """ Tests mongo operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test

        # insert
        doc = {"foo":"bar", "items":[1, 2, 3]}
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.has_key("_id"), True)
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # insert preserves object id
        doc.update({'_id': objectid.ObjectId()})
        yield test.insert(doc, safe=True)
        result = yield test.find_one(doc)
        self.assertEqual(result.get('_id'), doc.get('_id'))
        self.assertEqual(result["foo"], "bar")
        self.assertEqual(result["items"], [1, 2, 3])

        # update
        yield test.update({"_id":result["_id"]}, {"$set":{"one":"two"}}, safe=True)
        result = yield test.find_one({"_id":result["_id"]})
        self.assertEqual(result["one"], "two")

        # delete
        yield test.remove(result["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_Timestamps(self):
        """Tests mongo operations with Timestamps"""
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        test = conn.foo.test_ts

        # insert with specific timestamp
        doc1 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(1, 2)}
        yield test.insert(doc1, safe=True)

        result = yield test.find_one(doc1)
        self.assertEqual(result.get('ts').time, 1)
        self.assertEqual(result.get('ts').inc, 2)

        # insert with specific timestamp
        doc2 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(2, 1)}
        yield test.insert(doc2, safe=True)

        # the objects come back sorted by ts correctly.
        # (test that we stored inc/time in the right fields)
        result = yield test.find(filter=qf.sort(qf.ASCENDING('ts')))
        self.assertEqual(result[0]['_id'], doc1['_id'])
        self.assertEqual(result[1]['_id'], doc2['_id'])

        # insert with null timestamp
        doc3 = {'_id':objectid.ObjectId(),
                'ts':timestamp.Timestamp(0, 0)}
        yield test.insert(doc3, safe=True)

        # time field loaded correctly
        result = yield test.find_one(doc3['_id'])
        now = time.time()
        self.assertTrue(now - 2 <= result['ts'].time <= now)

        # delete
        yield test.remove(doc1["_id"], safe=True)
        yield test.remove(doc2["_id"], safe=True)
        yield test.remove(doc3["_id"], safe=True)

        # disconnect
        yield conn.disconnect()

class TestGridFsObjects(unittest.TestCase):
    """ Test the GridFS operations from txmongo._gridfs """
    @defer.inlineCallbacks
    def _disconnect(self, conn):
        """ Disconnect the connection """
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsObjects(self):
        """ Tests gridfs objects """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        collection = db.fs

        gfs = gridfs.GridFS(db) # Default collection

        gridin = GridIn(collection, filename='test', contentType="text/plain",
                        chunk_size=2**2**2**2)
        new_file = gfs.new_file(filename='test2', contentType="text/plain",
                        chunk_size=2**2**2**2)

        # disconnect
        yield conn.disconnect()

    @defer.inlineCallbacks
    def test_GridFsOperations(self):
        """ Tests gridfs operations """
        conn = yield txmongo.MongoConnection(mongo_host, mongo_port)
        db = conn.test
        collection = db.fs

        # Don't forget to disconnect
        self.addCleanup(self._disconnect, conn)
        try:
            in_file = StringIO("Test input string")
            out_file = StringIO()
        except Exception, e:
            self.fail("Failed to create memory files for testing: %s" % e)

        try:
            # Tests writing to a new gridfs file
            gfs = gridfs.GridFS(db) # Default collection
            g_in = gfs.new_file(filename='optest', contentType="text/plain",
                            chunk_size=2**2**2**2) # non-default chunk size used
            # yielding to ensure writes complete before we close and close before we try to read
            yield g_in.write(in_file.read())
            yield g_in.close()

            # Tests reading from an existing gridfs file
            g_out = yield gfs.get_last_version('optest')
            data = yield g_out.read()
            out_file.write(data)
            _id = g_out._id
        except Exception,e:
            self.fail("Failed to communicate with the GridFS. " +
                      "Is MongoDB running? %s" % e)
        else:
            self.assertEqual(in_file.getvalue(), out_file.getvalue(),
                         "Could not read the value from writing an input")
        finally:
            in_file.close()
            out_file.close()
            g_out.close()

        listed_files = yield gfs.list()
        self.assertEqual(['optest'], listed_files,
                         "'optest' is the only expected file and we received %s" % listed_files)

        yield gfs.delete(_id)

if __name__ == "__main__":
    suite = runner.TrialSuite((TestMongoObjects, TestGridFsObjects))
    suite.run()
<|endoftext|>


<commit_before>import os

import networkx
from netdiff import OpenvpnParser
from netdiff.tests import TestCase

CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
links2 = open('{0}/static/openvpn-2-links.txt'.format(CURRENT_DIR)).read()
links2_tap = open('{0}/static/openvpn-5-links-tap.txt'.format(CURRENT_DIR)).read()

class TestOpenvpnParser(TestCase):

    def test_parse(self):
        p = OpenvpnParser(links2)
        self.assertIsInstance(p.graph, networkx.Graph)
        self.assertEqual(p.version, '1')
        self.assertEqual(p.metric, 'static')

    def test_json_dict(self):
        p = OpenvpnParser(links2)
        data = p.json(dict=True)
        self.assertIsInstance(data, dict)
        self.assertEqual(data['type'], 'NetworkGraph')
        self.assertEqual(data['protocol'], 'OpenVPN Status Log')
        self.assertEqual(data['version'], '1')
        self.assertEqual(data['revision'], None)
        self.assertEqual(data['metric'], 'static')
        self.assertIsInstance(data['nodes'], list)
        self.assertIsInstance(data['links'], list)
        self.assertEqual(len(data['nodes']), 3)
        self.assertEqual(len(data['links']), 2)
        # check presence of labels
        labels = []
        for node in data['nodes']:
            if 'label' in node:
                labels.append(node['label'])
        self.assertEqual(len(labels), 2)
        self.assertIn('nodeA', labels)
        self.assertIn('nodeB', labels)

    def test_json_dict_tap(self):
        p = OpenvpnParser(links2_tap)
        data = p.json(dict=True)
        self.assertIsInstance(data, dict)
        self.assertEqual(data['type'], 'NetworkGraph')
        self.assertEqual(data['protocol'], 'OpenVPN Status Log')
        self.assertEqual(data['version'], '1')
        self.assertEqual(data['revision'], None)
        self.assertEqual(data['metric'], 'static')
        self.assertIsInstance(data['nodes'], list)
        self.assertIsInstance(data['links'], list)
        self.assertEqual(len(data['nodes']), 6)
        self.assertEqual(len(data['links']), 5)
        labels = []
        for node in data['nodes']:
            if 'label' in node:
                labels.append(node['label'])
        self.assertEqual(len(labels), 5)
        self.assertIn('nodeA', labels)
        self.assertIn('nodeB', labels)
        self.assertIn('nodeC', labels)
        self.assertIn('nodeD', labels)
        self.assertIn('nodeE', labels)

    def test_empty_string(self):
        OpenvpnParser(data='{}')

    def test_empty_dict(self):
        OpenvpnParser({})

<commit_msg>[tests] Minor improvements to test_openvpn variable naming<commit_after>import os

import networkx
from netdiff import diff
from netdiff import OpenvpnParser
from netdiff.tests import TestCase

CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
links2 = open('{0}/static/openvpn-2-links.txt'.format(CURRENT_DIR)).read()
links5_tap = open('{0}/static/openvpn-5-links-tap.txt'.format(CURRENT_DIR)).read()

class TestOpenvpnParser(TestCase):

    def test_parse(self):
        p = OpenvpnParser(links2)
        self.assertIsInstance(p.graph, networkx.Graph)
        self.assertEqual(p.version, '1')
        self.assertEqual(p.metric, 'static')

    def test_json_dict(self):
        p = OpenvpnParser(links2)
        data = p.json(dict=True)
        self.assertIsInstance(data, dict)
        self.assertEqual(data['type'], 'NetworkGraph')
        self.assertEqual(data['protocol'], 'OpenVPN Status Log')
        self.assertEqual(data['version'], '1')
        self.assertEqual(data['revision'], None)
        self.assertEqual(data['metric'], 'static')
        self.assertIsInstance(data['nodes'], list)
        self.assertIsInstance(data['links'], list)
        self.assertEqual(len(data['nodes']), 3)
        self.assertEqual(len(data['links']), 2)
        # check presence of labels
        labels = []
        for node in data['nodes']:
            if 'label' in node:
                labels.append(node['label'])
        self.assertEqual(len(labels), 2)
        self.assertIn('nodeA', labels)
        self.assertIn('nodeB', labels)

    def test_json_dict_tap(self):
        p = OpenvpnParser(links5_tap)
        data = p.json(dict=True)
        self.assertIsInstance(data, dict)
        self.assertEqual(data['type'], 'NetworkGraph')
        self.assertEqual(data['protocol'], 'OpenVPN Status Log')
        self.assertEqual(data['version'], '1')
        self.assertEqual(data['revision'], None)
        self.assertEqual(data['metric'], 'static')
        self.assertIsInstance(data['nodes'], list)
        self.assertIsInstance(data['links'], list)
        self.assertEqual(len(data['nodes']), 6)
        self.assertEqual(len(data['links']), 5)
        labels = []
        for node in data['nodes']:
            if 'label' in node:
                labels.append(node['label'])
        self.assertEqual(len(labels), 5)
        self.assertIn('nodeA', labels)
        self.assertIn('nodeB', labels)
        self.assertIn('nodeC', labels)
        self.assertIn('nodeD', labels)
        self.assertIn('nodeE', labels)

    def test_empty_string(self):
        OpenvpnParser(data='{}')

    def test_empty_dict(self):
        OpenvpnParser({})

    def test_label_diff_added(self):
        old = OpenvpnParser({})
        new = OpenvpnParser(links5_tap)
        result = diff(old, new)
        labels = []
        for node in result['added']['nodes']:
            if 'label' in node:
                labels.append(node['label'])
        self.assertEqual(len(labels), 5)
        self.assertIn('nodeA', labels)
        self.assertIn('nodeB', labels)
        self.assertIn('nodeC', labels)
        self.assertIn('nodeD', labels)
        self.assertIn('nodeE', labels)
<|endoftext|>


<commit_before>import os
#from nltk.parse.stanford import StanfordDependencyParser

import kindred
import kindred.Dependencies
from kindred.Parser import Parser

from kindred.datageneration import generateData,generateTestData

def assertProcessedEntity(entity,expectedType,expectedLocs,expectedSourceEntityID):
	assert isinstance(entity,kindred.ProcessedEntity)
	assert entity.entityType == expectedType, "(%s) not as expected" % (entity.__str__())
	assert entity.entityLocs == expectedLocs, "(%s) not as expected" % (entity.__str__())
	assert entity.sourceEntityID == expectedSourceEntityID, "(%s) not as expected" % (entity.__str__())

def test_simpleSentenceParse():
	text = '<drug id="1">Erlotinib</drug> is a common treatment for <cancer id="2">lung</cancer> and unknown <cancer id="2">cancers</cancer>'
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == 1

	processedSentence = doc.processedSentences[0]
	assert isinstance(processedSentence,kindred.ProcessedSentence)

	expectedWords = "Erlotinib is a common treatment for lung and unknown cancers".split()
	assert isinstance(processedSentence.tokens,list)
	assert len(expectedWords) == len(processedSentence.tokens)
	for w,t in zip(expectedWords,processedSentence.tokens):
		assert isinstance(t,kindred.Token)
		assert len(t.lemma) > 0
		assert w == t.word

	assert isinstance(processedSentence.processedEntities,list)
	assert len(processedSentence.processedEntities) == 2
	assertProcessedEntity(processedSentence.processedEntities[0],'drug',[0],'1')
	assertProcessedEntity(processedSentence.processedEntities[1],'cancer',[6,9],'2')

	assert isinstance(processedSentence.dependencies,list)
	assert len(processedSentence.dependencies) > 0

def test_twoSentenceParse():
	text = '<drug id="1">Erlotinib</drug> is a common treatment for <cancer id="2">NSCLC</cancer>. <drug id="3">Aspirin</drug> is the main cause of <disease id="4">boneitis</disease>.'
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == 2

	# Check types
	for processedSentence in doc.processedSentences:
		assert isinstance(processedSentence,kindred.ProcessedSentence)
		assert isinstance(processedSentence.tokens,list)
		for t in processedSentence.tokens:
			assert isinstance(t,kindred.Token)
			assert len(t.lemma) > 0
		assert isinstance(processedSentence.processedEntities,list)
		assert isinstance(processedSentence.dependencies,list)
		assert len(processedSentence.dependencies) > 0

	# First sentence
	expectedWords = "Erlotinib is a common treatment for NSCLC .".split()
	processedSentence0 = doc.processedSentences[0]
	assert len(expectedWords) == len(processedSentence0.tokens)
	for w,t in zip(expectedWords,processedSentence0.tokens):
		assert w == t.word

	assert isinstance(processedSentence0.processedEntities,list)
	assert len(processedSentence0.processedEntities) == 2
	assertProcessedEntity(processedSentence0.processedEntities[0],'drug',[0],'1')
	assertProcessedEntity(processedSentence0.processedEntities[1],'cancer',[6],'2')

	# Second sentence
	expectedWords = "Aspirin is the main cause of boneitis .".split()
	processedSentence1 = doc.processedSentences[1]

	assert len(expectedWords) == len(processedSentence1.tokens)
	for w,t in zip(expectedWords,processedSentence1.tokens):
		assert w == t.word

	assert isinstance(processedSentence1.processedEntities,list)
	assert len(processedSentence1.processedEntities) == 2
	assertProcessedEntity(processedSentence1.processedEntities[0],'drug',[0],'3')
	assertProcessedEntity(processedSentence1.processedEntities[1],'disease',[6],'4')

def test_largeSentence():
	repeatCount = 1000
	singleSentence = 'Erlotinib is a common treatment for lung and unknown cancers.'
	text = " ".join( [ singleSentence for _ in xrange(repeatCount) ] )
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == repeatCount

if __name__ == '__main__':
	#test_stanfordDependencyParser()
	#test_maltParser()
	#text = "<drug id=1>Erlotinib</drug> is a common treatment for <cancer id=2>lung</cancer> and unknown <cancer id=2>cancers</cancer>"
	#data = [ kindred.TextAndEntityData(text) for _ in range(2) ]

	#parser = Parser()
	#processedSentences = parser.parse(data)
	#runPerfTest()
	#test_simpleSentenceParse()
	test_largeSentence()

<commit_msg>Fix xrange to range<commit_after>import os
#from nltk.parse.stanford import StanfordDependencyParser

import kindred
import kindred.Dependencies
from kindred.Parser import Parser

from kindred.datageneration import generateData,generateTestData

def assertProcessedEntity(entity,expectedType,expectedLocs,expectedSourceEntityID):
	assert isinstance(entity,kindred.ProcessedEntity)
	assert entity.entityType == expectedType, "(%s) not as expected" % (entity.__str__())
	assert entity.entityLocs == expectedLocs, "(%s) not as expected" % (entity.__str__())
	assert entity.sourceEntityID == expectedSourceEntityID, "(%s) not as expected" % (entity.__str__())

def test_simpleSentenceParse():
	text = '<drug id="1">Erlotinib</drug> is a common treatment for <cancer id="2">lung</cancer> and unknown <cancer id="2">cancers</cancer>'
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == 1

	processedSentence = doc.processedSentences[0]
	assert isinstance(processedSentence,kindred.ProcessedSentence)

	expectedWords = "Erlotinib is a common treatment for lung and unknown cancers".split()
	assert isinstance(processedSentence.tokens,list)
	assert len(expectedWords) == len(processedSentence.tokens)
	for w,t in zip(expectedWords,processedSentence.tokens):
		assert isinstance(t,kindred.Token)
		assert len(t.lemma) > 0
		assert w == t.word

	assert isinstance(processedSentence.processedEntities,list)
	assert len(processedSentence.processedEntities) == 2
	assertProcessedEntity(processedSentence.processedEntities[0],'drug',[0],'1')
	assertProcessedEntity(processedSentence.processedEntities[1],'cancer',[6,9],'2')

	assert isinstance(processedSentence.dependencies,list)
	assert len(processedSentence.dependencies) > 0

def test_twoSentenceParse():
	text = '<drug id="1">Erlotinib</drug> is a common treatment for <cancer id="2">NSCLC</cancer>. <drug id="3">Aspirin</drug> is the main cause of <disease id="4">boneitis</disease>.'
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == 2

	# Check types
	for processedSentence in doc.processedSentences:
		assert isinstance(processedSentence,kindred.ProcessedSentence)
		assert isinstance(processedSentence.tokens,list)
		for t in processedSentence.tokens:
			assert isinstance(t,kindred.Token)
			assert len(t.lemma) > 0
		assert isinstance(processedSentence.processedEntities,list)
		assert isinstance(processedSentence.dependencies,list)
		assert len(processedSentence.dependencies) > 0

	# First sentence
	expectedWords = "Erlotinib is a common treatment for NSCLC .".split()
	processedSentence0 = doc.processedSentences[0]
	assert len(expectedWords) == len(processedSentence0.tokens)
	for w,t in zip(expectedWords,processedSentence0.tokens):
		assert w == t.word

	assert isinstance(processedSentence0.processedEntities,list)
	assert len(processedSentence0.processedEntities) == 2
	assertProcessedEntity(processedSentence0.processedEntities[0],'drug',[0],'1')
	assertProcessedEntity(processedSentence0.processedEntities[1],'cancer',[6],'2')

	# Second sentence
	expectedWords = "Aspirin is the main cause of boneitis .".split()
	processedSentence1 = doc.processedSentences[1]

	assert len(expectedWords) == len(processedSentence1.tokens)
	for w,t in zip(expectedWords,processedSentence1.tokens):
		assert w == t.word

	assert isinstance(processedSentence1.processedEntities,list)
	assert len(processedSentence1.processedEntities) == 2
	assertProcessedEntity(processedSentence1.processedEntities[0],'drug',[0],'3')
	assertProcessedEntity(processedSentence1.processedEntities[1],'disease',[6],'4')

def test_largeSentence():
	repeatCount = 1000
	singleSentence = 'Erlotinib is a common treatment for lung and unknown cancers.'
	text = " ".join( [ singleSentence for _ in range(repeatCount) ] )
	corpus = kindred.Corpus(text)

	parser = Parser()
	parser.parse(corpus)

	assert len(corpus.documents) == 1
	doc = corpus.documents[0]
	assert isinstance(doc.processedSentences,list)
	assert len(doc.processedSentences) == repeatCount

if __name__ == '__main__':
	#test_stanfordDependencyParser()
	#test_maltParser()
	#text = "<drug id=1>Erlotinib</drug> is a common treatment for <cancer id=2>lung</cancer> and unknown <cancer id=2>cancers</cancer>"
	#data = [ kindred.TextAndEntityData(text) for _ in range(2) ]

	#parser = Parser()
	#processedSentences = parser.parse(data)
	#runPerfTest()
	#test_simpleSentenceParse()
	test_largeSentence()

<|endoftext|>


<commit_before># modified from https://github.com/goberoi/cloudy_vision

import cv2
from jinja2 import FileSystemLoader, Environment
import json
import numpy
import os
import pprint
import shutil
import time
import vendors.google
import vendors.microsoft
import vendors.clarifai_
import vendors.ibm
import vendors.cloudsight_

SETTINGS = None
def settings(name):
    """Fetch a settings parameter."""

    # Initialize settings if necessary.
    global SETTINGS
    if SETTINGS is None:

        # Change this dict to suit your taste.
        SETTINGS = {
            'api_keys_filepath' : './api_keys.json',
            'input_images_dir' : 'input_images',
            'output_dir' : 'output',
            'static_dir' : 'static',
            'output_image_height' : 200,
            'vendors' : {
                # 'google' : vendors.google,
                'msft' : vendors.microsoft,
                'clarifai' : vendors.clarifai_
                # 'ibm' : vendors.ibm,
                # 'cloudsight' : vendors.cloudsight_
            }
        }

        # Load API keys
        with open(SETTINGS['api_keys_filepath']) as data_file:
            SETTINGS['api_keys'] = json.load(data_file)

    return SETTINGS[name]

def log_status(filepath, vendor_name, msg):
    filename = os.path.basename(filepath)
    print("%s -> %s" % ((filename + ", " + vendor_name).ljust(40), msg))

def resize_and_save(input_image_filepath, output_image_filepath):
    image = cv2.imread(input_image_filepath)
    height = image.shape[0]
    width = image.shape[1]
    aspect_ratio = float(width) / float(height)

    new_height = settings('output_image_height')
    new_width = int(aspect_ratio * new_height)

    output_image = cv2.resize(image, (new_width, new_height))
    cv2.imwrite(output_image_filepath, output_image)

def render_from_template(directory, template_name, **kwargs):
    loader = FileSystemLoader(directory)
    env = Environment(loader=loader)
    template = env.get_template(template_name)
    return template.render(**kwargs)

def process_all_images():

    image_results = []

    # Create the output directory
    if not os.path.exists(settings('output_dir')):
        os.makedirs(settings('output_dir'))

    # Loop through all input images.
    for filename in os.listdir(settings('input_images_dir')):

        # Only process files that have these image extensions.
        if not filename.endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp')):
            continue

        # Create a full path so we can read these files.
        filepath = os.path.join(settings('input_images_dir'), filename)

        # Create an output object for the image
        image_result = {
            'input_image_filepath' : filepath,
            'output_image_filepath' : filename,
            'vendors' : []
        }
        image_results.append(image_result)

        # Walk through all vendor APIs to call.
        for vendor_name, vendor_module in sorted(settings('vendors').iteritems(), reverse=True):

            # Figure out filename to store and retrive cached JSON results.
            output_json_filename = filename + "." + vendor_name + ".json"
            output_json_path = os.path.join(settings('output_dir'), output_json_filename)

            # And where to store the output image
            output_image_filepath = os.path.join(settings('output_dir'), filename)

            # Check if the call is already cached.
            if os.path.isfile(output_json_path):

                # If so, read the result from the .json file stored in the output dir.
                log_status(filepath, vendor_name, "skipping API call, already cached")
                with open(output_json_path, 'r') as infile:
                    raw_api_result = infile.read()

            else:

                # If not, make the API call for this particular vendor.
                log_status(filepath, vendor_name, "calling API")
                raw_api_result = vendor_module.call_vision_api(filepath, settings('api_keys'))

                # And cache the result in a .json file
                log_status(filepath, vendor_name, "success, storing result in %s" % output_json_path)
                with open(output_json_path, 'w') as outfile:
                    outfile.write(raw_api_result)

                # Resize the original image and write to an output filename
                log_status(filepath, vendor_name, "writing output image in %s" % output_image_filepath)
                resize_and_save(filepath, output_image_filepath)

                # Sleep so we avoid hitting throttling limits
                time.sleep(1)

            # Parse the JSON result we fetched (via API call or from cache)
            api_result = json.loads(raw_api_result)

            standardized_result = vendor_module.get_standardized_result(api_result)
            image_result['vendors'].append({
                'api_result' : api_result,
                'vendor_name' : vendor_name,
                'standardized_result' : standardized_result,
                'output_json_filename' : output_json_filename
            })

    # Render HTML file with all results.
    output_html = render_from_template('.', os.path.join(settings('static_dir'), 'template.html'), image_results=image_results)

    # Write HTML output.
    output_html_filepath = os.path.join(settings('output_dir'), 'output.html')
    with open(output_html_filepath, 'w') as output_html_file:
        output_html_file.write(output_html)

#    print vendors.clarifai_.get_acces_token(settings('api_keys'))

def run():
    index=0
    video_capture = cv2.VideoCapture(0)
    # faceCascade = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
    while(True):
        # Capture frame-by-frame
        ret, frame = video_capture.read()

        # cv2.imshow('Webcam', frame)

        # if cv2.waitKey(1) & 0xFF == ord('p'):
        #     clickPhoto = True
#
        # if ((cv2.waitKey(1) & 0xFF )== ord('p')):
        #     print("ewqreqwr")
        #     cv2.imwrite('testImage %s.jpg' %(str(index)), crop_img)
        #     index+=1
        #     process_all_images()
        cv2.imwrite('input_images/testimage%s.jpg' %(str(index)), frame)
        index+=1
        if(index<4):
            time.sleep(3)
            continue
        break
        # cv2.waitKey broken systemwide locally
        # if (cv2.waitKey(1) & 0xFF) == ord('q'):
        #     break

    # When everything done, release the capture
    video_capture.release()
    cv2.destroyAllWindows()
    process_all_images()

if __name__ == "__main__":
    run()
<commit_msg>real time-ish analyzing and producing json files<commit_after># modified from https://github.com/goberoi/cloudy_vision

import cv2
from jinja2 import FileSystemLoader, Environment
import json
import numpy
import os
import pprint
import shutil
import time
import vendors.google
import vendors.microsoft
import vendors.clarifai_
import vendors.ibm
import vendors.cloudsight_

SETTINGS = None
def settings(name):
    """Fetch a settings parameter."""

    # Initialize settings if necessary.
    global SETTINGS
    if SETTINGS is None:

        # Change this dict to suit your taste.
        SETTINGS = {
            'api_keys_filepath' : './api_keys.json',
            'input_images_dir' : 'input_images',
            'output_dir' : 'output',
            'static_dir' : 'static',
            'output_image_height' : 200,
            'vendors' : {
                # 'google' : vendors.google,
                'msft' : vendors.microsoft,
                'clarifai' : vendors.clarifai_
                # 'ibm' : vendors.ibm,
                # 'cloudsight' : vendors.cloudsight_
            }
        }

        # Load API keys
        with open(SETTINGS['api_keys_filepath']) as data_file:
            SETTINGS['api_keys'] = json.load(data_file)

    return SETTINGS[name]

def log_status(filepath, vendor_name, msg):
    filename = os.path.basename(filepath)
    print("%s -> %s" % ((filename + ", " + vendor_name).ljust(40), msg))

def resize_and_save(input_image_filepath, output_image_filepath):
    image = cv2.imread(input_image_filepath)
    height = image.shape[0]
    width = image.shape[1]
    aspect_ratio = float(width) / float(height)

    new_height = settings('output_image_height')
    new_width = int(aspect_ratio * new_height)

    output_image = cv2.resize(image, (new_width, new_height))
    cv2.imwrite(output_image_filepath, output_image)

def render_from_template(directory, template_name, **kwargs):
    loader = FileSystemLoader(directory)
    env = Environment(loader=loader)
    template = env.get_template(template_name)
    return template.render(**kwargs)

def process_all_images():

    image_results = []

    # Create the output directory
    if not os.path.exists(settings('output_dir')):
        os.makedirs(settings('output_dir'))

    # Loop through all input images.
    for filename in os.listdir(settings('input_images_dir')):

        # Only process files that have these image extensions.
        if not filename.endswith(('.png', '.jpg', '.jpeg', '.gif', '.bmp')):
            continue

        # Create a full path so we can read these files.
        filepath = os.path.join(settings('input_images_dir'), filename)

        # Create an output object for the image
        image_result = {
            'input_image_filepath' : filepath,
            'output_image_filepath' : filename,
            'vendors' : []
        }
        image_results.append(image_result)

        # Walk through all vendor APIs to call.
        for vendor_name, vendor_module in sorted(settings('vendors').iteritems(), reverse=True):

            # Figure out filename to store and retrive cached JSON results.
            output_json_filename = filename + "." + vendor_name + ".json"
            output_json_path = os.path.join(settings('output_dir'), output_json_filename)

            # And where to store the output image
            output_image_filepath = os.path.join(settings('output_dir'), filename)

            # Check if the call is already cached.
            # if os.path.isfile(output_json_path):

            #     # If so, read the result from the .json file stored in the output dir.
            #     log_status(filepath, vendor_name, "skipping API call, already cached")
            #     with open(output_json_path, 'r') as infile:
            #         raw_api_result = infile.read()

            # else:

            # If not, make the API call for this particular vendor.
            log_status(filepath, vendor_name, "calling API")
            raw_api_result = vendor_module.call_vision_api(filepath, settings('api_keys'))

            # And cache the result in a .json file
            log_status(filepath, vendor_name, "success, storing result in %s" % output_json_path)
            with open(output_json_path, 'w') as outfile:
                outfile.write(raw_api_result)

            # Resize the original image and write to an output filename
            log_status(filepath, vendor_name, "writing output image in %s" % output_image_filepath)
            resize_and_save(filepath, output_image_filepath)

            # Sleep so we avoid hitting throttling limits
            time.sleep(1)

            # Parse the JSON result we fetched (via API call or from cache)
            api_result = json.loads(raw_api_result)

            standardized_result = vendor_module.get_standardized_result(api_result)
            image_result['vendors'].append({
                'api_result' : api_result,
                'vendor_name' : vendor_name,
                'standardized_result' : standardized_result,
                'output_json_filename' : output_json_filename
            })

    # Render HTML file with all results.
    output_html = render_from_template('.', os.path.join(settings('static_dir'), 'template.html'), image_results=image_results)

    # Write HTML output.
    output_html_filepath = os.path.join(settings('output_dir'), 'output.html')
    with open(output_html_filepath, 'w') as output_html_file:
        output_html_file.write(output_html)

#    print vendors.clarifai_.get_acces_token(settings('api_keys'))

def run():
    index=0
    video_capture = cv2.VideoCapture(0)
    # faceCascade = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
    while(True):
        # Capture frame-by-frame
        ret, frame = video_capture.read()

        # cv2.imshow('Webcam', frame)

        # if cv2.waitKey(1) & 0xFF == ord('p'):
        #     clickPhoto = True
#
        # if ((cv2.waitKey(1) & 0xFF )== ord('p')):
        #     print("ewqreqwr")
        #     cv2.imwrite('testImage %s.jpg' %(str(index)), crop_img)
        #     index+=1
        #     process_all_images()
        cv2.imwrite('input_images/testimage.jpg', frame)
        process_all_images()
        index+=1
        if(index<4):
            time.sleep(10)
            continue
        break
        # cv2.waitKey broken systemwide locally
        # if (cv2.waitKey(1) & 0xFF) == ord('q'):
        #     break

    # When everything done, release the capture
    video_capture.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>#!/home/cc/library/bin/cctools_python
# CCTOOLS_PYTHON_VERSION 2.7 2.6

from datetime import datetime
from work_queue import *

import os
import socket
import sys
import tarfile

def get_port():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    ip = s.getsockname()[0]
    s.close()
    return ip

def print_info(port, ip):
    wkrstr = "work_queue_worker -d all --cores 0 %s %s" % (ip, port)
    print("Listening for workers @ %s on port %s" % (ip, port))
    print("(this is a best guess IP, depending on your computing environment you may need to adjust.)")
    print("\nHINT: To start a worker, you can probably use this command: \n%s\n" % wkrstr)

def compress_remove(filelist, targzname):
    tar = tarfile.open(targzname, "w:gz")
    for f in filelist:
        tar.add(f)
    tar.close()
    for f in filelist:
        os.remove(f)

def write_worker_report(queue):
    print("Worker Report (%s): connected(%s), busy(%s), idle(%s), lost(%s)" % (str(datetime.now()),
                                                                               str(queue.stats.total_workers_connected),
                                                                               str(queue.stats.workers_busy),
                                                                               str(queue.stats.workers_idle),
                                                                               str(queue.stats.total_workers_removed)))

def write_task_report(task, queue):
    r = task.resources_measured
    s = queue.stats
    rl = [task.id, r.command, r.start, r.end, r.exit_status,
          r.cpu_time, r.wall_time, r.cores, r.virtual_memory, r.swap_memory,
          r.total_processes, r.max_concurrent_processes, r.bytes_read, r.bytes_written,
          s.total_workers_connected, s.workers_busy, s.workers_idle, s.total_workers_removed,
          s.tasks_complete, s.tasks_running, s.tasks_waiting, s.total_execute_time]
    return "\t".join([str(l) for l in rl]) + "\n"

def spawn_queue(port, logPrefix):
    queue = WorkQueue(port)
    queue.specify_log(logPrefix + "_wq.log")
    monitor_status = queue.enable_monitoring(logPrefix + "_monitors")
    if not monitor_status:
        print("NOTICE: Monitoring failed to initialize")
    # queue.specify_password_file()  # TODO: Give workers a password file
    print("Workqueue launched on port (%s)" % str(port))
    return queue, monitor_status

def runKlipReduce(klipReduce="klipReduce", logPrefix="run", configList=None, resume=False, resumeLogPrefix=None):
    """
    runKlipReduce: Distribute klipReduce tasks across resources using WorkQueue.

    ## Example Usages ##
    # New Run #
    runKlipReduce(klipReduce="klipReduce", logPrefix="try1", "configList")
    # Resume Previous Run #
    runKlipReduce(klipReduce="klipReduce", logPrefix="try2", resume=True, resumeLogPrefix="try1")

    :param klipReduce: command or path to klipReduce.
    :param logPrefix: prefix to attach to run logs.
    :param configList: list of input files & expected output files.
    :param resume: OPTIONAL, specify true if want to resume from a previous run.
    :param resumeLogPrefix: OPTIONAL, prefix of a previous run logs from which to resume.
    :return:
    """

    # Set number of files completed that will trigger compression, the root and starting iterator of batch names.
    compress_threshold = 100
    batch_root = "batch"
    batch_count = 0

    # Check to make sure specified options are valid.
    if configList == None and resumeLogPrefix == None:
        print("ERROR (runKlipReduce): Specify a configList OR resume=True and a resumeLogPrefix to run")
        exit()
    if configList != None and resumeLogPrefix != None:
        print("ERROR (runKlipReduce): configList and resumeLogPrefix specified. Use one OR the other.")
        exit()

    # Create log filename objects.
    alltasklog = logPrefix + "_alltasks.log"
    completetasklog = logPrefix + "_completetasks.log"
    failedtasklog = logPrefix + "_failedtasks.log"
    usagelog = logPrefix + "_usage.log"

    # Create the tasks queue using the default port. If this port is already
    # been used by another program, you can try setting port = 0 to use an
    # available port, or specify a port directly.
    port = WORK_QUEUE_DEFAULT_PORT
    # Launch work queue
    try:
        q, monitoring = spawn_queue(port, logPrefix)
    except:
        print("Failed to launch on default WorkQueue port. Trying to find an available port...")
        try:
            port = 0
            q, monitoring = spawn_queue(port, logPrefix)
        except Exception as e:
            print("Instantiation of Work Queue failed!")
            print(e)
            sys.exit(1)

    # Build task list, either from scratch or using resume logs.
    all_tasks = []
    done = []
    if not resume:
        with open(configList, 'U') as cfgin:
            for line in cfgin:
                contents = line.rstrip().split()
                all_tasks.append({"cfg": contents[0], "outf": contents[1]})
    elif resume:
        if resumeLogPrefix == None:
            print("ERROR (runKlipReduce): Must specify a resumeLogPrefix when resuming...")
            exit()

        # Generate a list of files already in directory (for clean up into tar.gz in case of crash).
        currents = [f for f in os.listdir('.')]
        current_batches = [f for f in currents if batch_root in f]
        if len(current_batches) > 0:
            for batch in current_batches:
                count = int(batch.split('batch')[1].split('.tar.gz')[0])
                if count >= batch_count:
                    batch_count = count+1

        resume_all = resumeLogPrefix + "_alltasks.log"
        resume_complete = resumeLogPrefix + "_completetasks.log"
        resume_failed = resumeLogPrefix + "_failedtasks.log"
        # Find incomplete tasks and store as dictionaries in remaining_tasks
        # remaining_tasks = [{"cmd": "fill_in_command", "cfg": "fill_in_cfg", "outf": "fill_in_output_file"}, ...]
        with open(resume_all, 'r') as a, open(resume_complete, 'r') as c, open(resume_failed, 'r') as f:
            comp = []
            fail = []
            for line in c:
                comp.append(line.rstrip())
            for line in f:
                fail.append(line.rstrip())
            for line in a:
                contents = line.rstrip().split('\t')
                cmd = contents[0]
                if cmd not in comp and cmd not in fail:
                    cfg = contents[1]
                    outf = contents[2]
                    all_tasks.append({"cmd": cmd, "cfg": cfg, "outf": outf})
                if contents[2] in currents:
                    done.append(contents[2])
    else:
        print("WARNING: runKlipReduce is confused and did not run. Please troubleshoot!")

    # Generate tasks & submit to queue, record dictionary of taskid:expected output.
    submit_count = 0
    expect_out = {}
    with open(alltasklog, 'w') as alltasks, open(completetasklog, 'w') as completetasks, open(failedtasklog, 'w') as failedtasks:
        for entry in all_tasks:
            # Specify command and expected output file.
            cfg = entry["cfg"]
            outf = entry["outf"]
            command = "%s -c %s" % (klipReduce, os.path.basename(cfg))
            # Add job to all tasks log.
            alltasks.write("%s\t%s\t%s\n" % (command, cfg, outf))
            # Build task.
            t = Task(command)
            t.specify_file(cfg, os.path.basename(cfg), WORK_QUEUE_INPUT, cache=False)
            t.specify_file(outf, os.path.basename(outf), WORK_QUEUE_OUTPUT, cache=False)
            # Add other file specifications as needed here (ALSO BELOW).

            # Submit task to queue.
            taskid = q.submit(t)
            expect_out[taskid] = outf
            submit_count += 1

        # Determine listening IP
        ip = get_port()

        print("WorkQueue Launched Successfully!")
        print_info(str(q.port), str(ip))
        print("...waiting for tasks to complete...")

        # Monitor queue, alert user to status, compress and remove files at specified threshold.
        if monitoring:
            use_log = open(usagelog, 'w')
            use_log.write("TaskID\tCommand\tStart\tEnd\tExitStatus\t"
                          "CPUTime\tWallTime\tCores\tVirtualMemory\tSwapMemory\t"
                          "TotalProcesses\tMaxConcurrentProcesses\tBytesRead\tBytesWritten\t"
                          "WorkersConnected\tWorkersBusy\tWorkersIdle\tWorkersRemoved\t"
                          "TasksComplete\tTasksRunning\tTasksWaiting\tTotalExecuteTime\n")

        write_worker_report(q)
        tries = 0
        while not q.empty():
            tries += 1
            t = q.wait(6)
            # Write report of worker conditions
            if not tries % 10:
                write_worker_report(q)
            # If tasks have returned.
            if t:
                # Print return message.
                print("%s - task (id# %d) complete: %s (return code %d)" % (str(datetime.now()), t.id, t.command, t.return_status))
                if monitoring:
                    use_log.write(write_task_report(t, q))
                # Check that task is actually complete.
                if t.return_status != 0:
                    # Task failed. Write to failed task log.
                    failedtasks.write("%s\n" % t.command)
                else:
                    # Task succeeded. Check for valid output.
                    expect = expect_out[t.id]
                    if os.path.exists(expect):
                        # Task succeeded & output exists. Write to complete task log.
                        completetasks.write("%s\n" % t.command)
                        done.append(expect)
                    else:
                        # Output is missing, alert user and write to failed tasks.
                        print("NOTICE: Missing output - " + str(t.command))
                        failedtasks.write("%s\n" % t.command)

                # Check if compression threshold is met, if true gzip & tar then remove uncompressed versions.
                if len(done) >= compress_threshold:
                    compress_remove(done, "%s%s.tar.gz" % (batch_root, str(batch_count)))
                    batch_count += 1
                    done = []
        # Compress any remaining outputs.
        if len(done) > 0:
            compress_remove(done, "%s%s.tar.gz" % (batch_root, str(batch_count)))

    if monitoring:
        use_log.close()
    print("MASTER: all tasks complete!")
    return 1

if __name__ == "__main__":
    stime = datetime.now()
    program_args = sys.argv[1:]
    if len(program_args) < 3 or len(program_args) > 4:
        print("ERROR (arguments)")
        print("Please specify your klipReduce path "
              "(usually just 'klipReduce'), a prefix for the run, and a configList or resume options.")
        print("Syntax:")
        print("-- Launch a new run: python findr_reduce.py <klipReduce path> <run_prefix> <config-and-outputs_list>")
        print("-- Resume a previous run: python findr_reduce.py resume <klipReduce_path> <run_prefix> <resume_prefix>")
        print("Examples:")
        print("-- Launch a new run: python findr_reduce.py klipReduce try1 configList.list")
        print("-- Resume a previous run: python findr_reduce.py resume klipReduce try2 try1")
        exit()
    try:
        runKlipReduce(klipReduce=program_args[1], logPrefix=program_args[2],
                      resume=True, resumeLogPrefix=program_args[3])
    except:
        runKlipReduce(klipReduce=program_args[0], logPrefix=program_args[1], configList=program_args[2])

    print("Reductions Complete")
    ftime = datetime.now()
    print("Total Run Time: %s" % str(ftime - stime))
<commit_msg>fixed error in resuming resulting from resource monitoring<commit_after>#!/home/cc/library/bin/cctools_python
# CCTOOLS_PYTHON_VERSION 2.7 2.6

from datetime import datetime
from work_queue import *

import os
import socket
import sys
import tarfile

def get_port():
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.connect(("8.8.8.8", 80))
    ip = s.getsockname()[0]
    s.close()
    return ip

def print_info(port, ip):
    wkrstr = "work_queue_worker -d all --cores 0 %s %s" % (ip, port)
    print("Listening for workers @ %s on port %s" % (ip, port))
    print("(this is a best guess IP, depending on your computing environment you may need to adjust.)")
    print("\nHINT: To start a worker, you can probably use this command: \n%s\n" % wkrstr)

def compress_remove(filelist, targzname):
    tar = tarfile.open(targzname, "w:gz")
    for f in filelist:
        tar.add(f)
    tar.close()
    for f in filelist:
        os.remove(f)

def write_worker_report(queue):
    print("Status Report: %s (time elapsed %s)" % (str(datetime.now()), str(datetime.now()-stime)))
    print("- Workers: connected(%s), busy(%s), idle(%s), lost(%s)" % (str(queue.stats.total_workers_connected),
                                                                      str(queue.stats.workers_busy),
                                                                      str(queue.stats.workers_idle),
                                                                      str(queue.stats.total_workers_removed)))
    print("- Tasks: complete(%s), running(%s), waiting (%s)" % (str(queue.stats.tasks_complete),
                                                                str(queue.stats.tasks_running),
                                                                str(queue.stats.tasks_waiting)))

def write_task_report(task, queue):
    r = task.resources_measured
    s = queue.stats
    rl = [task.id, r.command, r.start, r.end, r.exit_status,
          r.cpu_time, r.wall_time, r.cores, r.virtual_memory, r.swap_memory,
          r.total_processes, r.max_concurrent_processes, r.bytes_read, r.bytes_written,
          s.total_workers_connected, s.workers_busy, s.workers_idle, s.total_workers_removed,
          s.tasks_complete, s.tasks_running, s.tasks_waiting, s.total_execute_time]
    return "\t".join([str(l) for l in rl]) + "\n"

def spawn_queue(port, logPrefix):
    queue = WorkQueue(port)
    queue.specify_log(logPrefix + "_wq.log")
    monitor_status = queue.enable_monitoring(logPrefix + "_monitors")
    if not monitor_status:
        print("NOTICE: Monitoring failed to initialize")
    # queue.specify_password_file()  # TODO: Give workers a password file
    print("Workqueue launched on port (%s)" % str(port))
    return queue, monitor_status

def runKlipReduce(klipReduce="klipReduce", logPrefix="run", configList=None, resume=False, resumeLogPrefix=None):
    """
    runKlipReduce: Distribute klipReduce tasks across resources using WorkQueue.

    ## Example Usages ##
    # New Run #
    runKlipReduce(klipReduce="klipReduce", logPrefix="try1", "configList")
    # Resume Previous Run #
    runKlipReduce(klipReduce="klipReduce", logPrefix="try2", resume=True, resumeLogPrefix="try1")

    :param klipReduce: command or path to klipReduce.
    :param logPrefix: prefix to attach to run logs.
    :param configList: list of input files & expected output files.
    :param resume: OPTIONAL, specify true if want to resume from a previous run.
    :param resumeLogPrefix: OPTIONAL, prefix of a previous run logs from which to resume.
    :return:
    """
    # Set number of files completed that will trigger compression, the root and starting iterator of batch names.
    compress_threshold = 100
    batch_root = "batch"
    batch_count = 0

    # Check to make sure specified options are valid.
    if configList == None and resumeLogPrefix == None:
        print("ERROR (runKlipReduce): Specify a configList OR resume=True and a resumeLogPrefix to run")
        exit()
    if configList != None and resumeLogPrefix != None:
        print("ERROR (runKlipReduce): configList and resumeLogPrefix specified. Use one OR the other.")
        exit()

    # Create log filename objects.
    alltasklog = logPrefix + "_alltasks.log"
    completetasklog = logPrefix + "_completetasks.log"
    failedtasklog = logPrefix + "_failedtasks.log"
    usagelog = logPrefix + "_usage.log"

    # Create the tasks queue using the default port. If this port is already
    # been used by another program, you can try setting port = 0 to use an
    # available port, or specify a port directly.
    port = WORK_QUEUE_DEFAULT_PORT
    # Launch work queue
    try:
        q, monitoring = spawn_queue(port, logPrefix)
    except:
        print("Failed to launch on default WorkQueue port. Trying to find an available port...")
        try:
            port = 0
            q, monitoring = spawn_queue(port, logPrefix)
        except Exception as e:
            print("Instantiation of Work Queue failed!")
            print(e)
            sys.exit(1)

    # Build task list, either from scratch or using resume logs.
    all_tasks = []
    done = []
    if not resume:
        with open(configList, 'U') as cfgin:
            for line in cfgin:
                contents = line.rstrip().split()
                all_tasks.append({"cfg": contents[0], "outf": contents[1]})
    elif resume:
        if resumeLogPrefix == None:
            print("ERROR (runKlipReduce): Must specify a resumeLogPrefix when resuming...")
            exit()

        # Generate a list of files already in directory (for clean up into tar.gz in case of crash).
        currents = [f for f in os.listdir('.')]
        current_batches = [f for f in currents if batch_root in f]
        if len(current_batches) > 0:
            for batch in current_batches:
                count = int(batch.split('batch')[1].split('.tar.gz')[0])
                if count >= batch_count:
                    batch_count = count + 1

        resume_all = resumeLogPrefix + "_alltasks.log"
        resume_complete = resumeLogPrefix + "_completetasks.log"
        resume_failed = resumeLogPrefix + "_failedtasks.log"
        # Find incomplete tasks and store as dictionaries in remaining_tasks
        with open(resume_all, 'r') as a, open(resume_complete, 'r') as c, open(resume_failed, 'r') as f:
            comp = []
            fail = []
            for line in c:
                comp.append(line.rstrip())
            for line in f:
                fail.append(line.rstrip())
            for line in a:
                contents = line.rstrip().split('\t')
                cmd = contents[0]
                if cmd not in comp and cmd not in fail:
                    cfg = contents[1]
                    outf = contents[2]
                    all_tasks.append({"cfg": cfg, "outf": outf})
                if contents[2] in currents:
                    done.append(contents[2])
    else:
        print("WARNING: runKlipReduce is confused and did not run. Please troubleshoot!")
        exit()

    # Generate tasks & submit to queue, record dictionary of taskid:expected output.
    submit_count = 0
    expect_out = {}
    with open(alltasklog, 'w') as alltasks, open(completetasklog, 'w') as completetasks, open(failedtasklog,
                                                                                'w') as failedtasks:
        for entry in all_tasks:
            # Specify command and expected output file.
            cfg = entry["cfg"]
            outf = entry["outf"]
            command = "%s -c %s" % (klipReduce, os.path.basename(cfg))

            # Build task.
            t = Task(command)
            t.specify_tag(command)
            t.specify_file(cfg, os.path.basename(cfg), WORK_QUEUE_INPUT, cache=False)
            t.specify_file(outf, os.path.basename(outf), WORK_QUEUE_OUTPUT, cache=False)
            # Add other file specifications as needed here (ALSO BELOW).

            # Submit task to queue.
            taskid = q.submit(t)

            # Add job to all tasks log.
            alltasks.write("%s\t%s\t%s\n" % (t.tag, cfg, outf))
            expect_out[taskid] = outf
            submit_count += 1

        # Determine listening IP
        ip = get_port()

        print("WorkQueue Launched Successfully!")
        print_info(str(q.port), str(ip))
        print("...waiting for %s tasks to complete..." % str(submit_count))

        # Monitor queue, alert user to status, compress and remove files at specified threshold.
        if monitoring:
            use_log = open(usagelog, 'w')
            use_log.write("TaskID\tCommand\tStart\tEnd\tExitStatus\t"
                          "CPUTime\tWallTime\tCores\tVirtualMemory\tSwapMemory\t"
                          "TotalProcesses\tMaxConcurrentProcesses\tBytesRead\tBytesWritten\t"
                          "WorkersConnected\tWorkersBusy\tWorkersIdle\tWorkersRemoved\t"
                          "TasksComplete\tTasksRunning\tTasksWaiting\tTotalExecuteTime\n")

        write_worker_report(q)
        tries = 0
        while not q.empty():
            tries += 1
            t = q.wait(6)
            # Write report of worker conditions
            if not tries % 10:
                write_worker_report(q)
            # If tasks have returned.
            if t:
                # Print return message.
                print("%s - task (id# %d) complete: %s (return code %d)" % (
                str(datetime.now()), t.id, t.command, t.return_status))
                if monitoring:
                    use_log.write(write_task_report(t, q))
                # Check that task is actually complete.
                if t.return_status != 0:
                    # Task failed. Write to failed task log.
                    failedtasks.write("%s\n" % t.tag)
                else:
                    # Task succeeded. Check for valid output.
                    expect = expect_out[t.id]
                    if os.path.exists(expect):
                        # Task succeeded & output exists. Write to complete task log.
                        completetasks.write("%s\n" % t.tag)
                        done.append(expect)
                    else:
                        # Output is missing, alert user and write to failed tasks.
                        print("NOTICE: Missing output - " + str(t.command))
                        failedtasks.write("%s\n" % t.tag)

                # Check if compression threshold is met, if true gzip & tar then remove uncompressed versions.
                if len(done) >= compress_threshold:
                    compress_remove(done, "%s%s.tar.gz" % (batch_root, str(batch_count)))
                    batch_count += 1
                    done = []
        # Compress any remaining outputs.
        if len(done) > 0:
            compress_remove(done, "%s%s.tar.gz" % (batch_root, str(batch_count)))

    if monitoring:
        use_log.close()
    print("MASTER: all tasks complete!")
    return 1

if __name__ == "__main__":
    stime = datetime.now()
    program_args = sys.argv[1:]
    if len(program_args) < 3 or len(program_args) > 4:
        print("ERROR (arguments)")
        print("Please specify your klipReduce path "
              "(usually just 'klipReduce'), a prefix for the run, and a configList or resume options.")
        print("Syntax:")
        print("-- Launch a new run: python findr_reduce.py <klipReduce path> <run_prefix> <config-and-outputs_list>")
        print("-- Resume a previous run: python findr_reduce.py resume <klipReduce_path> <run_prefix> <resume_prefix>")
        print("Examples:")
        print("-- Launch a new run: python findr_reduce.py klipReduce try1 configList.list")
        print("-- Resume a previous run: python findr_reduce.py resume klipReduce try2 try1")
        exit()
    if "resume" in program_args:
        print("Resuming previous from %s_*.log" % program_args[3])
        runKlipReduce(klipReduce=program_args[1], logPrefix=program_args[2],
                      resume=True, resumeLogPrefix=program_args[3])
    else:
        runKlipReduce(klipReduce=program_args[0], logPrefix=program_args[1], configList=program_args[2])

    print("Reductions Complete")
    ftime = datetime.now()
    print("Total Run Time: %s" % str(ftime - stime))
<|endoftext|>


<commit_before>import argparse
from configparser import ConfigParser
from datetime import datetime
from itertools import chain
from pathlib import Path
import re
import sys

from attr import attrib, attrs

import firefed.__version__ as version

LINUX_PROFILES_PATH = '~/.mozilla/firefox'
OSX_PROFILES_PATH = '~/Library/Application Support/Firefox/Profiles'
PROFILES_INI = 'profiles.ini'

@attrs
class Profile:
    name = attrib()
    path = attrib()
    default = attrib(default=False, converter=lambda x: bool(int(x)))

class ProfileNotFoundError(Exception):

    def __init__(self, name):
        if name is None:
            text = 'No default profile found. Use --profile to specify one.'
        else:
            text = 'Profile "%s" not found.' % name
        super().__init__(text)

class FatalError(Exception):
    """Raised when an unrecoverable error in a feature occurs."""
    pass

def fatal(text):
    raise FatalError(text)

def mozilla_dir():
    if sys.platform == 'darwin':
        profiles_path = OSX_PROFILES_PATH
    else:
        # For now, we assume that other OSes follow the Linux convention
        profiles_path = LINUX_PROFILES_PATH
    return Path(profiles_path).expanduser()

def read_profiles():
    config = ConfigParser()
    config.read(str(mozilla_dir() / PROFILES_INI))
    for section, profile in config.items():
        if not section.startswith('Profile'):
            continue
        path = Path(profile.get('Path'))
        if int(profile['IsRelative']):
            path = Path(mozilla_dir() / path)
        yield Profile(profile.get('Name'), path, profile.get('Default', 0))

def profile_dir(name):
    """Return path to FF profile for a given profile name or path."""
    if name:
        possible_path = Path(name)
        if possible_path.exists():
            return possible_path
    profiles = list(read_profiles())
    try:
        if name:
            profile = next(p for p in profiles if p.name == name)
        else:
            profile = next(p for p in profiles if p.default)
    except StopIteration:
        raise ProfileNotFoundError(name)
    return profile.path

def moz_datetime(ts):
    """Convert Mozilla timestamp to datetime."""
    return datetime.fromtimestamp(moz_to_unix_timestamp(ts))

def moz_to_unix_timestamp(ts):
    """Convert Mozilla timestamp to Unix timestamp."""
    try:
        return ts // 1000000
    except TypeError:
        return 0

def make_parser():
    from firefed.feature import Feature
    parser = argparse.ArgumentParser(
        'firefed',
        description=version.__description__,
    )
    parser.add_argument(
        '-P',
        '--profiles',
        help='show all local profiles',
        action='store_true',
        dest='show_profiles',
    )
    parser.add_argument(
        '-p',
        '--profile',
        help='profile name or directory to be used when running a feature',
    )
    parser.add_argument(
        '-v',
        '--verbose',
        help='verbose output (can be used multiple times)',
        action='count',
        dest='verbosity',
        default=0,
    )
    parser.add_argument(
        '-f',
        '--force',
        help='treat target as a profile directory even if it doesn\'t'
             ' look like one',
        action='store_true',
        default=False,
    )
    subparsers = parser.add_subparsers(
        title='features',
        metavar='FEATURE',
        description='Set the feature you want to run as positional argument. '
        'Each feature has its own sub arguments which can be listed with '
        '`firefed <feature> -h`.',
        dest='feature',
    )
    for name, feature in Feature.feature_map().items():
        feature_parser = subparsers.add_parser(name,
                                               help=feature.description())
        for args, kwargs in feature.cli_args():
            feature_parser.add_argument(*args, **kwargs)
    return parser

@attrs
class Tabulate:

    rows = attrib(converter=list)
    headers = attrib()
    maximums = attrib(init=False)

    def __attrs_post_init__(self):
        maxs = [0] * len(self.rows[0])
        for row in chain([self.headers], self.rows):
            for i, column in enumerate(row):
                maxs[i] = max(maxs[i], len(self.strip_invisible(column)))
        self.maximums = maxs

    def __call__(self):
        self.print_row(self.headers)
        self.print_row([''] * len(self.maximums), ch='-')
        for row in self.rows:
            self.print_row(row)

    def pad(self, s, num, ch):
        missing = num - len(self.strip_invisible(s))
        return s + missing * ch

    def print_row(self, row, ch=' '):
        print('  '.join([self.pad(c, self.maximums[i], ch) for i, c in
                         enumerate(row)]))

    @staticmethod
    def strip_invisible(s):
        invisibles = re.compile(r'\x1b\[\d*m')
        return re.sub(invisibles, '', s)

def tabulate(*args, **kwargs):
    Tabulate(*args, **kwargs)()
<commit_msg>Add version info<commit_after>import argparse
from configparser import ConfigParser
from datetime import datetime
from itertools import chain
from pathlib import Path
import re
import sys

from attr import attrib, attrs

import firefed.__version__ as version

LINUX_PROFILES_PATH = '~/.mozilla/firefox'
OSX_PROFILES_PATH = '~/Library/Application Support/Firefox/Profiles'
PROFILES_INI = 'profiles.ini'

@attrs
class Profile:
    name = attrib()
    path = attrib()
    default = attrib(default=False, converter=lambda x: bool(int(x)))

class ProfileNotFoundError(Exception):

    def __init__(self, name):
        if name is None:
            text = 'No default profile found. Use --profile to specify one.'
        else:
            text = 'Profile "%s" not found.' % name
        super().__init__(text)

class FatalError(Exception):
    """Raised when an unrecoverable error in a feature occurs."""
    pass

def fatal(text):
    raise FatalError(text)

def mozilla_dir():
    if sys.platform == 'darwin':
        profiles_path = OSX_PROFILES_PATH
    else:
        # For now, we assume that other OSes follow the Linux convention
        profiles_path = LINUX_PROFILES_PATH
    return Path(profiles_path).expanduser()

def read_profiles():
    config = ConfigParser()
    config.read(str(mozilla_dir() / PROFILES_INI))
    for section, profile in config.items():
        if not section.startswith('Profile'):
            continue
        path = Path(profile.get('Path'))
        if int(profile['IsRelative']):
            path = Path(mozilla_dir() / path)
        yield Profile(profile.get('Name'), path, profile.get('Default', 0))

def profile_dir(name):
    """Return path to FF profile for a given profile name or path."""
    if name:
        possible_path = Path(name)
        if possible_path.exists():
            return possible_path
    profiles = list(read_profiles())
    try:
        if name:
            profile = next(p for p in profiles if p.name == name)
        else:
            profile = next(p for p in profiles if p.default)
    except StopIteration:
        raise ProfileNotFoundError(name)
    return profile.path

def moz_datetime(ts):
    """Convert Mozilla timestamp to datetime."""
    return datetime.fromtimestamp(moz_to_unix_timestamp(ts))

def moz_to_unix_timestamp(ts):
    """Convert Mozilla timestamp to Unix timestamp."""
    try:
        return ts // 1000000
    except TypeError:
        return 0

def make_parser():
    from firefed.feature import Feature
    parser = argparse.ArgumentParser(
        'firefed',
        description=version.__description__,
    )
    parser.add_argument(
        '-V',
        '--version',
        action='version',
        version='%(prog)s v' + version.__version__,
    )
    parser.add_argument(
        '-P',
        '--profiles',
        help='show all local profiles',
        action='store_true',
        dest='show_profiles',
    )
    parser.add_argument(
        '-p',
        '--profile',
        help='profile name or directory to be used when running a feature',
    )
    parser.add_argument(
        '-v',
        '--verbose',
        help='verbose output (can be used multiple times)',
        action='count',
        dest='verbosity',
        default=0,
    )
    parser.add_argument(
        '-f',
        '--force',
        help='treat target as a profile directory even if it doesn\'t'
             ' look like one',
        action='store_true',
        default=False,
    )
    subparsers = parser.add_subparsers(
        title='features',
        metavar='FEATURE',
        description='Set the feature you want to run as positional argument. '
        'Each feature has its own sub arguments which can be listed with '
        '`firefed <feature> -h`.',
        dest='feature',
    )
    for name, feature in Feature.feature_map().items():
        feature_parser = subparsers.add_parser(name,
                                               help=feature.description())
        for args, kwargs in feature.cli_args():
            feature_parser.add_argument(*args, **kwargs)
    return parser

@attrs
class Tabulate:

    rows = attrib(converter=list)
    headers = attrib()
    maximums = attrib(init=False)

    def __attrs_post_init__(self):
        maxs = [0] * len(self.rows[0])
        for row in chain([self.headers], self.rows):
            for i, column in enumerate(row):
                maxs[i] = max(maxs[i], len(self.strip_invisible(column)))
        self.maximums = maxs

    def __call__(self):
        self.print_row(self.headers)
        self.print_row([''] * len(self.maximums), ch='-')
        for row in self.rows:
            self.print_row(row)

    def pad(self, s, num, ch):
        missing = num - len(self.strip_invisible(s))
        return s + missing * ch

    def print_row(self, row, ch=' '):
        print('  '.join([self.pad(c, self.maximums[i], ch) for i, c in
                         enumerate(row)]))

    @staticmethod
    def strip_invisible(s):
        invisibles = re.compile(r'\x1b\[\d*m')
        return re.sub(invisibles, '', s)

def tabulate(*args, **kwargs):
    Tabulate(*args, **kwargs)()
<|endoftext|>


<commit_before>from __future__ import print_function
from __future__ import absolute_import
from sfepy.base.testing import TestCommon

class Test(TestCommon):

    def from_conf(conf, options):
        return Test(conf=conf, options=options)
    from_conf = staticmethod(from_conf)

    def test_parse_equations(self):
        from sfepy.discrete.parse_equations import create_bnf

        test_strs = [
            """- ev_volume.i1.Omega(uc)""",
            """- 2 * dw_term.i1.Omega(uc)
             = - 3.0 * dw_term2.i1.Omega2(uc)""",
            """2 * dw_term.i1.Omega(uc)
             = 3.0 * dw_term2.i1.Omega2(uc)""",
            """- (2 + 1j - 3) * dw_term.i1.Omega(uc)
               = - (1j - 3.0 + 2.0j) * dw_term2.i1.Omega2(uc)""",
            """(2 + 1j) * dw_term.i1.Omega(uc)
               = (3.0 + 2.0j) * dw_term2.i1.Omega2(uc)""",
            """d_term1.Y(fluid, u, w, Nu, dcf, mode)
                 + d_term2.Omega(u, w, Nu, dcf, mode)
                 - d_another_term.Elsewhere(w, p, Nu, dcf, mode)
                 = - dw_rhs.a.Y3(u, q, Nu, dcf, mode)""",
            """no_args() = 0""",
            """+ something(a, b, c) = + something_else(c, a, d[-1])""",
            """term_.a.a(u)""",
            """term.i1.Omega(v, du/dt) + term2.i2.Gamma(v, dphi/dt)""",
            """dw_jump.isurf.Gamma12_1(jump1.val, q1, p1, tr(p2))""",
            """term_with_trace.2.Gamma1(v, tr(Gamma2, u))""",
        ]

        n_fail = 0
        term_descs = []
        for test_str in test_strs:
            term_descs[:] = []
            try:
                bnf = create_bnf(term_descs)
                bnf.parseString(test_str)
            except:
                self.report('failed: %s' % test_str)
                if self.options.debug:
                    raise
                n_fail += 1
            for td in term_descs:
                print(td)
        self.report('%d failure(s)' % n_fail)

        if n_fail:
            raise AssertionError
        return True

    def test_parse_regions(self):
        from sfepy.discrete.parse_regions import create_bnf

        test_strs = ['vertices of surface -v r.Omega',
                     'r.Y_2 +v copy r.Y_1',
                     'vertices in (y <= 0.00001) & (x < 0.11)',
                     'vertices in ((y <= 0.00001) & (x < 0.11))',
                     'vertices in (((y <= 0.00001) & (x < 0.11)))',
                     'vertices in (((0.00001 < y) & (x < 0.11)))',
                     'vertices of group 0',
                     """vertices of group 0 +v vertices of group 1
                     +c cells by afun""",
                     'all -v vertices in (y == 0.00001)',
                     'all -v vertices of surface',
                     'all -c r.DOmega_100',
                     """r.Y_1 -v vertices of surface *c r.Z_8
                        *v vertices in (y > 0)""",
                     'vertices of surface +v vertices by pokus',
                     'cells of group 6 +c vertices by fn2_3c',
                     """r.Y_1 *v (r.Y_2 +c (vertices in (y > 0) *v r.Y_32))
                        -v vertices of surface -c r.Y_5""",
                     'copy r.ab2-b-c +v r.d12-23',
                     'vertices by afun',
                     'vertex in r.Gamma_3',
                     'vertex 10',
                     'vertex 10, 20, 30',
                     'cell 10',
                     'cell 10, 20, 30',
                     'vertex 10, 20 +v cell 30, 40',
                     '(vertex 10, 20) +v (cell 30, 40)',
                     'cell 10, 20, 30 +v vertex 10',
                     'cells by afun']

        stack = []
        bnf = create_bnf(stack)

        n_fail = 0
        for test_str in test_strs:
            stack[:] = []

            try:
                out = bnf.parseString(test_str)
                print(out)

            except:
                self.report('failed: %s' % test_str)
                n_fail += 1

        self.report('%d failures' % n_fail)

        if n_fail:
            raise AssertionError
        return True
<commit_msg>update test_parsing.py for pytest<commit_after>import sfepy.base.testing as tst

def test_parse_equations():
    from sfepy.discrete.parse_equations import create_bnf

    test_strs = [
        """- ev_volume.i1.Omega(uc)""",
        """- 2 * dw_term.i1.Omega(uc)
         = - 3.0 * dw_term2.i1.Omega2(uc)""",
        """2 * dw_term.i1.Omega(uc)
         = 3.0 * dw_term2.i1.Omega2(uc)""",
        """- (2 + 1j - 3) * dw_term.i1.Omega(uc)
           = - (1j - 3.0 + 2.0j) * dw_term2.i1.Omega2(uc)""",
        """(2 + 1j) * dw_term.i1.Omega(uc)
           = (3.0 + 2.0j) * dw_term2.i1.Omega2(uc)""",
        """d_term1.Y(fluid, u, w, Nu, dcf, mode)
             + d_term2.Omega(u, w, Nu, dcf, mode)
             - d_another_term.Elsewhere(w, p, Nu, dcf, mode)
             = - dw_rhs.a.Y3(u, q, Nu, dcf, mode)""",
        """no_args() = 0""",
        """+ something(a, b, c) = + something_else(c, a, d[-1])""",
        """term_.a.a(u)""",
        """term.i1.Omega(v, du/dt) + term2.i2.Gamma(v, dphi/dt)""",
        """dw_jump.isurf.Gamma12_1(jump1.val, q1, p1, tr(p2))""",
        """term_with_trace.2.Gamma1(v, tr(Gamma2, u))""",
    ]

    n_fail = 0
    term_descs = []
    for test_str in test_strs:
        term_descs[:] = []
        try:
            bnf = create_bnf(term_descs)
            bnf.parseString(test_str)
        except:
            tst.report('failed: %s' % test_str)
            n_fail += 1

    tst.report('%d failure(s)' % n_fail)

    assert n_fail == 0

def test_parse_regions():
    from sfepy.discrete.parse_regions import create_bnf

    test_strs = ['vertices of surface -v r.Omega',
                 'r.Y_2 +v copy r.Y_1',
                 'vertices in (y <= 0.00001) & (x < 0.11)',
                 'vertices in ((y <= 0.00001) & (x < 0.11))',
                 'vertices in (((y <= 0.00001) & (x < 0.11)))',
                 'vertices in (((0.00001 < y) & (x < 0.11)))',
                 'vertices of group 0',
                 """vertices of group 0 +v vertices of group 1
                 +c cells by afun""",
                 'all -v vertices in (y == 0.00001)',
                 'all -v vertices of surface',
                 'all -c r.DOmega_100',
                 """r.Y_1 -v vertices of surface *c r.Z_8
                    *v vertices in (y > 0)""",
                 'vertices of surface +v vertices by pokus',
                 'cells of group 6 +c vertices by fn2_3c',
                 """r.Y_1 *v (r.Y_2 +c (vertices in (y > 0) *v r.Y_32))
                    -v vertices of surface -c r.Y_5""",
                 'copy r.ab2-b-c +v r.d12-23',
                 'vertices by afun',
                 'vertex in r.Gamma_3',
                 'vertex 10',
                 'vertex 10, 20, 30',
                 'cell 10',
                 'cell 10, 20, 30',
                 'vertex 10, 20 +v cell 30, 40',
                 '(vertex 10, 20) +v (cell 30, 40)',
                 'cell 10, 20, 30 +v vertex 10',
                 'cells by afun']

    stack = []
    bnf = create_bnf(stack)

    n_fail = 0
    for test_str in test_strs:
        stack[:] = []

        try:
            bnf.parseString(test_str)

        except:
            tst.report('failed: %s' % test_str)
            n_fail += 1

    tst.report('%d failures' % n_fail)

    assert n_fail == 0
<|endoftext|>


<commit_before>import argparse
import logging
import os
import shutil
import subprocess
import tempfile
import time

import boto3
from ebcli.lib import aws as ebaws
from ebcli.lib import s3 as ebs3
from ebcli.lib import elasticbeanstalk
from ebcli.objects.exceptions import NotFoundError

DOCKERRUN_NAME = 'Dockerrun.aws.json'
DOCKEREXT_NAME = '.ebextensions/'

logger = logging.getLogger('ebi')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

def make_version(version_label: str,
                 dockerrun:str=DOCKERRUN_NAME, ebext:str=DOCKEREXT_NAME) -> str:
    """ Making zip file to upload for ElasticBeanstalk

    :param version_label: will be name of the created zip file

    * Including :param dockerrun: file as Dockerrun.aws.json
    * Including :param exext: directory as .ebextensions/

    :return: File path to created zip file (current directory).
    """
    with tempfile.TemporaryDirectory() as tempd:
        deploy_dockerrun = os.path.join(tempd, DOCKERRUN_NAME)
        deploy_ebext = os.path.join(tempd, DOCKEREXT_NAME)
        shutil.copyfile(dockerrun, deploy_dockerrun)
        shutil.copytree(ebext, deploy_ebext)
        return shutil.make_archive(version_label, 'zip', root_dir=tempd)

def upload_app_version(app_name: str, bundled_zip: str) -> (str, str):
    """ Uploading zip file of app version to S3
    :param app_name: application name to deploy
    :param bundled_zip: String path to zip file
    :return: bucket name and key for file (as tuple).
    """
    bucket = elasticbeanstalk.get_storage_location()
    key = app_name + '/' + os.path.basename(bundled_zip)
    try:
        ebs3.get_object_info(bucket, key)
        logger.info('S3 Object already exists. Skipping upload.')
    except NotFoundError:
        logger.info('Uploading archive to s3 location: ' + key)
        ebs3.upload_application_version(bucket, key, bundled_zip)
    return bucket, key

def main():
    """ Main function called from console_scripts
    """
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_create = subparsers.add_parser('create')
    parser_deploy = subparsers.add_parser('deploy')

    parser_create.add_argument('app_name', help='Application name to create')
    parser_create.add_argument('env_name', help='Environ name to deploy')
    parser_create.add_argument('cname', help='cname for created server')
    parser_create.add_argument('--version', help='Version label you want to specify')
    parser_create.add_argument('--profile', help='AWS account')
    parser_create.add_argument('--dockerrun', default=DOCKERRUN_NAME,
                               help='Path to file used as Dockerrun.aws.json')
    parser_create.add_argument('--ebext', default=DOCKEREXT_NAME,
                               help='Path to directory used as .ebextensions/')
    parser_create.add_argument('--cfg', help='Configuration template name to eb create')
    parser_create.set_defaults(sub='create')

    parser_deploy.add_argument('app_name', help='Application name to deploy')
    parser_deploy.add_argument('env_name', help='Environ name to deploy')
    parser_deploy.add_argument('--version', help='Version label you want to specify')
    parser_deploy.add_argument('--profile', help='AWS account')
    parser_deploy.add_argument('--dockerrun', default=DOCKERRUN_NAME,
                               help='Path to file used as Dockerrun.aws.json')
    parser_deploy.add_argument('--ebext', default=DOCKEREXT_NAME,
                               help='Path to directory used as .ebextensions/')
    parser_deploy.set_defaults(sub='deploy')
    parsed = parser.parse_args()

    if not hasattr(parsed, 'sub'):
        parser.print_help()
        return

    if parsed.version:
        version = parsed.version
    else:
        version = str(int(time.time()))

    if parsed.profile:
        session = boto3.session.Session(profile_name=parsed.profile)
    else:
        session = boto3.session.Session()
    eb = session.client('elasticbeanstalk')

    ebaws.set_region(session._session.get_config_variable('region'))
    ebaws.set_profile(session.profile_name)

    bundled_zip = make_version(version,
                               dockerrun=parsed.dockerrun,
                               ebext=parsed.ebext)
    bucket, key = upload_app_version(parsed.app_name, bundled_zip)

    logger.info('Creating application version')

    eb.create_application_version(
        ApplicationName=parsed.app_name,
        VersionLabel=version,
        SourceBundle={
            'S3Bucket': bucket,
            'S3Key': key,
        }
    )

    if parsed.sub == 'deploy':
        logger.info('Ok, now deploying the version %s for %s', version, parsed.env_name)
        subprocess.call(['eb', 'deploy', parsed.env_name, '--version=' + version,
                         '--profile=' + session.profile_name])
    elif parsed.sub == 'create':
        logger.info('Ok, now creating version %s or environment %s', version, parsed.env_name)
        payload = ['eb', 'create', parsed.env_name,
                   '--version=' + version,
                   '--cname=' + parsed.cname,
                   '--profile=' + session.profile_name]
        if parsed.cfg:
            payload.append('--cfg=' + parsed.cfg)
        subprocess.call(payload)
<commit_msg>Fixed to return code from subprocess calling directly<commit_after>import argparse
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import time

import boto3
from ebcli.lib import aws as ebaws
from ebcli.lib import s3 as ebs3
from ebcli.lib import elasticbeanstalk
from ebcli.objects.exceptions import NotFoundError

DOCKERRUN_NAME = 'Dockerrun.aws.json'
DOCKEREXT_NAME = '.ebextensions/'

logger = logging.getLogger('ebi')
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

def make_version(version_label: str,
                 dockerrun:str=DOCKERRUN_NAME, ebext:str=DOCKEREXT_NAME) -> str:
    """ Making zip file to upload for ElasticBeanstalk

    :param version_label: will be name of the created zip file

    * Including :param dockerrun: file as Dockerrun.aws.json
    * Including :param exext: directory as .ebextensions/

    :return: File path to created zip file (current directory).
    """
    with tempfile.TemporaryDirectory() as tempd:
        deploy_dockerrun = os.path.join(tempd, DOCKERRUN_NAME)
        deploy_ebext = os.path.join(tempd, DOCKEREXT_NAME)
        shutil.copyfile(dockerrun, deploy_dockerrun)
        shutil.copytree(ebext, deploy_ebext)
        return shutil.make_archive(version_label, 'zip', root_dir=tempd)

def upload_app_version(app_name: str, bundled_zip: str) -> (str, str):
    """ Uploading zip file of app version to S3
    :param app_name: application name to deploy
    :param bundled_zip: String path to zip file
    :return: bucket name and key for file (as tuple).
    """
    bucket = elasticbeanstalk.get_storage_location()
    key = app_name + '/' + os.path.basename(bundled_zip)
    try:
        ebs3.get_object_info(bucket, key)
        logger.info('S3 Object already exists. Skipping upload.')
    except NotFoundError:
        logger.info('Uploading archive to s3 location: ' + key)
        ebs3.upload_application_version(bucket, key, bundled_zip)
    return bucket, key

def main():
    """ Main function called from console_scripts
    """
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_create = subparsers.add_parser('create')
    parser_deploy = subparsers.add_parser('deploy')

    parser_create.add_argument('app_name', help='Application name to create')
    parser_create.add_argument('env_name', help='Environ name to deploy')
    parser_create.add_argument('cname', help='cname for created server')
    parser_create.add_argument('--version', help='Version label you want to specify')
    parser_create.add_argument('--profile', help='AWS account')
    parser_create.add_argument('--dockerrun', default=DOCKERRUN_NAME,
                               help='Path to file used as Dockerrun.aws.json')
    parser_create.add_argument('--ebext', default=DOCKEREXT_NAME,
                               help='Path to directory used as .ebextensions/')
    parser_create.add_argument('--cfg', help='Configuration template name to eb create')
    parser_create.set_defaults(sub='create')

    parser_deploy.add_argument('app_name', help='Application name to deploy')
    parser_deploy.add_argument('env_name', help='Environ name to deploy')
    parser_deploy.add_argument('--version', help='Version label you want to specify')
    parser_deploy.add_argument('--profile', help='AWS account')
    parser_deploy.add_argument('--dockerrun', default=DOCKERRUN_NAME,
                               help='Path to file used as Dockerrun.aws.json')
    parser_deploy.add_argument('--ebext', default=DOCKEREXT_NAME,
                               help='Path to directory used as .ebextensions/')
    parser_deploy.set_defaults(sub='deploy')
    parsed = parser.parse_args()

    if not hasattr(parsed, 'sub'):
        parser.print_help()
        return

    if parsed.version:
        version = parsed.version
    else:
        version = str(int(time.time()))

    if parsed.profile:
        session = boto3.session.Session(profile_name=parsed.profile)
    else:
        session = boto3.session.Session()
    eb = session.client('elasticbeanstalk')

    ebaws.set_region(session._session.get_config_variable('region'))
    ebaws.set_profile(session.profile_name)

    bundled_zip = make_version(version,
                               dockerrun=parsed.dockerrun,
                               ebext=parsed.ebext)
    bucket, key = upload_app_version(parsed.app_name, bundled_zip)

    logger.info('Creating application version')

    eb.create_application_version(
        ApplicationName=parsed.app_name,
        VersionLabel=version,
        SourceBundle={
            'S3Bucket': bucket,
            'S3Key': key,
        }
    )

    if parsed.sub == 'deploy':
        logger.info('Ok, now deploying the version %s for %s', version, parsed.env_name)
        sys.exit(subprocess.call(['eb', 'deploy', parsed.env_name, '--version=' + version,
                                  '--profile=' + session.profile_name]))
    elif parsed.sub == 'create':
        logger.info('Ok, now creating version %s or environment %s', version, parsed.env_name)
        payload = ['eb', 'create', parsed.env_name,
                   '--version=' + version,
                   '--cname=' + parsed.cname,
                   '--profile=' + session.profile_name]
        if parsed.cfg:
            payload.append('--cfg=' + parsed.cfg)
        sys.exit(subprocess.call(payload))
<|endoftext|>