<commit_before>APPLICATION_LABEL = 'cog'

PURPOSE_TYPES = (
                 'Overall Project Coordination',
                 'Steering Committee',
                 'Design',
                 'Design and Implementation Review',
                 'Task Prioritization',
                 'Requirements Identification',
                 'Strategic Direction',
                 'External Review',
                 'Implementation',
                 'Meeting Planning',
                 'Testing',
                 'Knowledge Transfer',
                 'Grant Writing',
                 'Other',
                )
PURPOSE_CV = [ (x,x) for x in PURPOSE_TYPES ]

COMMUNICATION_TYPES = (
                       'Telco',
                       'Face-to-face',
                       'Webinar',
                       'Video Conference',
                       'Internet chat',
                       'Wiki',
                       'Mailing list'
                     )
COMMUNICATION_CV = [ (x,x) for x in COMMUNICATION_TYPES ]

# list of tuples containing (role value, role display order)
LEAD_ORGANIZATIONAL_ROLES = (
                              ('Principal Investigator', 1),
                              ('Co-Investigator', 2),
                              ('Program Manager', 3),
                              ('Project Manager', 4),
                              ('Software Architect', 5),
                              ('Lead', 6),
                              ('Other', 7),
                            )

# list of tuples containing (role value, role display order)
MEMBER_ORGANIZATIONAL_ROLES = (
                               ('Administrative Assistant', 1),
                               ('Data Manager', 2),
                               ('Outreach Coordinator', 3),
                               ('Researcher', 4),
                               ('Software Developer', 5),
                               ('Webmaster', 6),
                               ('Other', 7),
                              )

ORGANIZATIONAL_ROLE_CV = [ (x[0]," %s (Lead Role)" % x[0]) for x in LEAD_ORGANIZATIONAL_ROLES ] + [ (x[0]," %s (Member Role)" % x[0]) for x in MEMBER_ORGANIZATIONAL_ROLES ]
# create and merge a combined dictionary of organizational roles
LEAD_ORGANIZATIONAL_ROLES_DICT = dict(LEAD_ORGANIZATIONAL_ROLES)
MEMBER_ORGANIZATIONAL_ROLES_DICT = dict(MEMBER_ORGANIZATIONAL_ROLES)
ORGANIZATIONAL_ROLES_DICT = dict( LEAD_ORGANIZATIONAL_ROLES_DICT.items() + MEMBER_ORGANIZATIONAL_ROLES_DICT.items() )

ROLE_CATEGORY_LEAD = 'Lead'
ROLE_CATEGORY_MEMBER = 'Member'
ORGANIZATIONAL_ROLE_CATEGORIES = (ROLE_CATEGORY_LEAD, ROLE_CATEGORY_MEMBER)
ORGANIZATIONAL_ROLE_CATEGORIES_CV = [ (x,x) for x in ORGANIZATIONAL_ROLE_CATEGORIES ]

MANAGEMENT_BODY_CATEGORY_STRATEGIC = 'Strategic'
MANAGEMENT_BODY_CATEGORY_OPERATIONAL = 'Operational'
MANAGEMENT_BODY_CATEGORIES = (MANAGEMENT_BODY_CATEGORY_STRATEGIC, MANAGEMENT_BODY_CATEGORY_OPERATIONAL)
MANAGEMENT_BODY_CATEGORIES_CV = [ (x,x) for x in MANAGEMENT_BODY_CATEGORIES ]

# list of tuples containing (management body value, management body display order)
STRATEGIC_MANAGEMENT_BODIES = (
                               ('Strategic Direction', 1),
                               ('Advice or Guidance', 2),
                               ('Program Direction', 3),
                               ('Review', 4),
                              )

# list of tuples containing (role value, role display order)
OPERATIONAL_MANAGEMENT_BODIES = (
                                 ('Research', 1),
                                 ('Development', 2),
                                 ('Requirements Identification', 3),
                                 ('Task Prioritization', 4),
                                 ('Testing', 5),
                                 ('Web Review', 6),
                                 ('Meeting and Event Planning', 7),
                                 ('Administration', 8),
                                )

MANAGEMENT_BODY_CV = [ (x[0]," %s (Strategic)" % x[0]) for x in STRATEGIC_MANAGEMENT_BODIES ] + [ (x[0]," %s (Operational)" % x[0]) for x in OPERATIONAL_MANAGEMENT_BODIES ]
# create and merge a combined dictionary of management bodies
STRATEGIC_MANAGEMENT_BODY_DICT = dict(STRATEGIC_MANAGEMENT_BODIES)
OPERATIONAL_MANAGEMENT_BODY_DICT = dict(OPERATIONAL_MANAGEMENT_BODIES)
MANAGEMENT_BODY_DICT = dict( STRATEGIC_MANAGEMENT_BODY_DICT.items() + OPERATIONAL_MANAGEMENT_BODY_DICT.items() )

MEMBERSHIP_TYPES = ('Open','Closed','By Invitation')

MEMBERSHIP_CV = [ (x,x) for x in MEMBERSHIP_TYPES ]

ROLE_ADMIN = 'admin'
ROLE_USER = 'user'
ROLES = [ROLE_ADMIN, ROLE_USER]

DOCUMENT_TYPE_ALL = 'All'
DOCUMENT_TYPE_IMAGE = 'Image'
DOCUMENT_TYPE_TEXT = 'Text'
DOCUMENT_TYPE_PRESENTATION = 'Presentation'
DOCUMENT_TYPE_PROGRAM = 'Program'

DOCUMENT_TYPES = {
                   DOCUMENT_TYPE_IMAGE: ['.gif', '.png', 'jpg,', '.jpeg'],
                   DOCUMENT_TYPE_TEXT:  ['.txt', '.pdf', '.doc', '.docx'],
                   DOCUMENT_TYPE_PRESENTATION: ['.ppt','.pptx','.key'],
                   DOCUMENT_TYPE_PROGRAM: ['.java', '.py', '.sh']
                 }

# path of default logo relative to MEDIA_ROOT
# use a location outside of "logos/" so that the default logo can
#DEFAULT_LOGO = "img/admin/logo_1109_cog.JPG"
DEFAULT_LOGO = "cog/img/cog_web_beta.png"
FOOTER_LOGO = "cog/img/logo_1310_cogfootershrunk.PNG"

UPLOAD_DIR_PHOTOS = "photos/"
UPLOAD_DIR_LOGOS = "logos/"

DEFAULT_IMAGES = { 'User':'/%s/unknown.jpeg' % UPLOAD_DIR_PHOTOS,
                   'Collaborator':'/%s/unknown.jpeg' % UPLOAD_DIR_PHOTOS,
                   'Organization':'/%s/notfound.jpeg' % UPLOAD_DIR_LOGOS,
                   'FundingSource':'/%s/notfound.jpeg' % UPLOAD_DIR_LOGOS }

# legacy media sub-directories of 'projects/'
SYSTEM_DOCS = 'system_docs'
SYSTEM_IMAGES = 'system_images'

# 1MB - 1048576
# 2.5MB - 2621440
# 5MB - 5242880
# 10MB - 10485760
# 20MB - 20971520
# 50MB - 52428800
# 100MB 104857600
# 250MB - 214958080
# 500MB - 429916160
MAX_UPLOADES_BYTES = 52428800

RESEARCH_KEYWORDS_MAX_CHARS = 60
RESEARCH_INTERESTS_MAX_CHARS = 1000

# signals
SIGNAL_OBJECT_CREATED = 'object_created'
SIGNAL_OBJECT_UPDATED = 'object_updated'
SIGNAL_OBJECT_DELETED = 'object_deleted'<commit_msg>roles.<commit_after>APPLICATION_LABEL = 'cog'

PURPOSE_TYPES = (
                 'Overall Project Coordination',
                 'Steering Committee',
                 'Design',
                 'Design and Implementation Review',
                 'Task Prioritization',
                 'Requirements Identification',
                 'Strategic Direction',
                 'External Review',
                 'Implementation',
                 'Meeting Planning',
                 'Testing',
                 'Knowledge Transfer',
                 'Grant Writing',
                 'Other',
                )
PURPOSE_CV = [ (x,x) for x in PURPOSE_TYPES ]

COMMUNICATION_TYPES = (
                       'Telco',
                       'Face-to-face',
                       'Webinar',
                       'Video Conference',
                       'Internet chat',
                       'Wiki',
                       'Mailing list'
                     )
COMMUNICATION_CV = [ (x,x) for x in COMMUNICATION_TYPES ]

# list of tuples containing (role value, role display order)
LEAD_ORGANIZATIONAL_ROLES = (
                              ('Principal Investigator', 1),
                              ('Co-Investigator', 2),
                              ('Program Manager', 3),
                              ('Project Manager', 4),
                              ('Software Architect', 5),
                              ('Lead', 6),
                              ('Other', 7),
                            )

# list of tuples containing (role value, role display order)
MEMBER_ORGANIZATIONAL_ROLES = (
                               ('Administrative Assistant', 1),
                               ('Data Manager', 2),
                               ('Outreach Coordinator', 3),
                               ('Researcher', 4),
                               ('Software Developer', 5),
                               ('Webmaster', 6),
                               ('Other', 7),
                              )

ORGANIZATIONAL_ROLE_CV = [ (x[0]," %s (Lead Role)" % x[0]) for x in LEAD_ORGANIZATIONAL_ROLES ] + [('','--------------')] + [ (x[0]," %s (Member Role)" % x[0]) for x in MEMBER_ORGANIZATIONAL_ROLES ]
# create and merge a combined dictionary of organizational roles
LEAD_ORGANIZATIONAL_ROLES_DICT = dict(LEAD_ORGANIZATIONAL_ROLES)
MEMBER_ORGANIZATIONAL_ROLES_DICT = dict(MEMBER_ORGANIZATIONAL_ROLES)
ORGANIZATIONAL_ROLES_DICT = dict( LEAD_ORGANIZATIONAL_ROLES_DICT.items() + MEMBER_ORGANIZATIONAL_ROLES_DICT.items() )

ROLE_CATEGORY_LEAD = 'Lead'
ROLE_CATEGORY_MEMBER = 'Member'
ORGANIZATIONAL_ROLE_CATEGORIES = (ROLE_CATEGORY_LEAD, ROLE_CATEGORY_MEMBER)
ORGANIZATIONAL_ROLE_CATEGORIES_CV = [ (x,x) for x in ORGANIZATIONAL_ROLE_CATEGORIES ]

MANAGEMENT_BODY_CATEGORY_STRATEGIC = 'Strategic'
MANAGEMENT_BODY_CATEGORY_OPERATIONAL = 'Operational'
MANAGEMENT_BODY_CATEGORIES = (MANAGEMENT_BODY_CATEGORY_STRATEGIC, MANAGEMENT_BODY_CATEGORY_OPERATIONAL)
MANAGEMENT_BODY_CATEGORIES_CV = [ (x,x) for x in MANAGEMENT_BODY_CATEGORIES ]

# list of tuples containing (management body value, management body display order)
STRATEGIC_MANAGEMENT_BODIES = (
                               ('Strategic Direction', 1),
                               ('Advice or Guidance', 2),
                               ('Program Direction', 3),
                               ('Review', 4),
                              )

# list of tuples containing (role value, role display order)
OPERATIONAL_MANAGEMENT_BODIES = (
                                 ('Research', 1),
                                 ('Development', 2),
                                 ('Requirements Identification', 3),
                                 ('Task Prioritization', 4),
                                 ('Testing', 5),
                                 ('Web Review', 6),
                                 ('Meeting and Event Planning', 7),
                                 ('Administration', 8),
                                )

MANAGEMENT_BODY_CV = [ (x[0]," %s (Strategic)" % x[0]) for x in STRATEGIC_MANAGEMENT_BODIES ] + [ (x[0]," %s (Operational)" % x[0]) for x in OPERATIONAL_MANAGEMENT_BODIES ]
# create and merge a combined dictionary of management bodies
STRATEGIC_MANAGEMENT_BODY_DICT = dict(STRATEGIC_MANAGEMENT_BODIES)
OPERATIONAL_MANAGEMENT_BODY_DICT = dict(OPERATIONAL_MANAGEMENT_BODIES)
MANAGEMENT_BODY_DICT = dict( STRATEGIC_MANAGEMENT_BODY_DICT.items() + OPERATIONAL_MANAGEMENT_BODY_DICT.items() )

MEMBERSHIP_TYPES = ('Open','Closed','By Invitation')

MEMBERSHIP_CV = [ (x,x) for x in MEMBERSHIP_TYPES ]

ROLE_ADMIN = 'admin'
ROLE_USER = 'user'
ROLES = [ROLE_ADMIN, ROLE_USER]

DOCUMENT_TYPE_ALL = 'All'
DOCUMENT_TYPE_IMAGE = 'Image'
DOCUMENT_TYPE_TEXT = 'Text'
DOCUMENT_TYPE_PRESENTATION = 'Presentation'
DOCUMENT_TYPE_PROGRAM = 'Program'

DOCUMENT_TYPES = {
                   DOCUMENT_TYPE_IMAGE: ['.gif', '.png', 'jpg,', '.jpeg'],
                   DOCUMENT_TYPE_TEXT:  ['.txt', '.pdf', '.doc', '.docx'],
                   DOCUMENT_TYPE_PRESENTATION: ['.ppt','.pptx','.key'],
                   DOCUMENT_TYPE_PROGRAM: ['.java', '.py', '.sh']
                 }

# path of default logo relative to MEDIA_ROOT
# use a location outside of "logos/" so that the default logo can
#DEFAULT_LOGO = "img/admin/logo_1109_cog.JPG"
DEFAULT_LOGO = "cog/img/cog_web_beta.png"
FOOTER_LOGO = "cog/img/logo_1310_cogfootershrunk.PNG"

UPLOAD_DIR_PHOTOS = "photos/"
UPLOAD_DIR_LOGOS = "logos/"

DEFAULT_IMAGES = { 'User':'/%s/unknown.jpeg' % UPLOAD_DIR_PHOTOS,
                   'Collaborator':'/%s/unknown.jpeg' % UPLOAD_DIR_PHOTOS,
                   'Organization':'/%s/notfound.jpeg' % UPLOAD_DIR_LOGOS,
                   'FundingSource':'/%s/notfound.jpeg' % UPLOAD_DIR_LOGOS }

# legacy media sub-directories of 'projects/'
SYSTEM_DOCS = 'system_docs'
SYSTEM_IMAGES = 'system_images'

# 1MB - 1048576
# 2.5MB - 2621440
# 5MB - 5242880
# 10MB - 10485760
# 20MB - 20971520
# 50MB - 52428800
# 100MB 104857600
# 250MB - 214958080
# 500MB - 429916160
MAX_UPLOADES_BYTES = 52428800

RESEARCH_KEYWORDS_MAX_CHARS = 60
RESEARCH_INTERESTS_MAX_CHARS = 1000

# signals
SIGNAL_OBJECT_CREATED = 'object_created'
SIGNAL_OBJECT_UPDATED = 'object_updated'
SIGNAL_OBJECT_DELETED = 'object_deleted'<|endoftext|>


<commit_before># -*- coding: iso-8859-1 -*-
# Maintainer: joaander

from hoomd_script import *
import unittest
import os

# unit tests for init.read_xml
class init_read_xml_tests (unittest.TestCase):
    def setUp(self):
        print
        if (comm.get_rank()==0):
            f = open("test.xml", "w");
            f.write('''<?xml version="1.0" encoding="UTF-8"?>
<hoomd_xml version="1.0">
<configuration time_step="0">
<box lx="8" ly="8" lz="8"/>
<position>
-1 2 3
2 1 -3
3 -2 1
</position>
<type>
A B C
</type>
</configuration>
</hoomd_xml>
''');
            f = open("test_out_of_box.xml", "w");
            f.write('''<?xml version="1.0" encoding="UTF-8"?>
<hoomd_xml version="1.0">
<configuration time_step="0">
<box lx="4" ly="4" lz="4"/>
<position>
-1 2 3
2 1 -3
3 -2 1
</position>
<type>
A B C
</type>
</configuration>
</hoomd_xml>
''');

    # tests basic creation of the random initializer
    def test(self):
        init.read_xml('test.xml');
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);

    # tests creation with a few more arugments specified
    def test_moreargs(self):
        init.read_xml('test.xml', time_step=100);
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);

    # tests creation with out of box particles
    def test_out_of_box_1(self):
        sys=init.read_xml('test_out_of_box.xml')
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);
        self.assertAlmostEqual(sys.particles[0].position[2],3,5)

    # tests creation with out of box particles
    def test_out_of_box_2(self):
        sys=init.read_xml('test_out_of_box.xml',wrap_coordinates=True)
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);
        self.assertAlmostEqual(sys.particles[0].position[2],-1,5)

    # checks for an error if initialized twice
    def test_inittwice(self):
        init.read_xml('test.xml');
        self.assertRaises(RuntimeError, init.read_xml, 'test.xml');

    def tearDown(self):
        if (comm.get_rank()==0):
            os.remove("test.xml");
        init.reset();

if __name__ == '__main__':
    unittest.main(argv = ['test.py', '-v'])
<commit_msg>Forgot to remove file in unit test<commit_after># -*- coding: iso-8859-1 -*-
# Maintainer: joaander

from hoomd_script import *
import unittest
import os

# unit tests for init.read_xml
class init_read_xml_tests (unittest.TestCase):
    def setUp(self):
        print
        if (comm.get_rank()==0):
            f = open("test.xml", "w");
            f.write('''<?xml version="1.0" encoding="UTF-8"?>
<hoomd_xml version="1.0">
<configuration time_step="0">
<box lx="8" ly="8" lz="8"/>
<position>
-1 2 3
2 1 -3
3 -2 1
</position>
<type>
A B C
</type>
</configuration>
</hoomd_xml>
''');
            f = open("test_out_of_box.xml", "w");
            f.write('''<?xml version="1.0" encoding="UTF-8"?>
<hoomd_xml version="1.0">
<configuration time_step="0">
<box lx="4" ly="4" lz="4"/>
<position>
-1 2 3
2 1 -3
3 -2 1
</position>
<type>
A B C
</type>
</configuration>
</hoomd_xml>
''');

    # tests basic creation of the random initializer
    def test(self):
        init.read_xml('test.xml');
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);

    # tests creation with a few more arugments specified
    def test_moreargs(self):
        init.read_xml('test.xml', time_step=100);
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);

    # tests creation with out of box particles
    def test_out_of_box_1(self):
        sys=init.read_xml('test_out_of_box.xml')
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);
        self.assertAlmostEqual(sys.particles[0].position[2],3,5)

    # tests creation with out of box particles
    def test_out_of_box_2(self):
        sys=init.read_xml('test_out_of_box.xml',wrap_coordinates=True)
        self.assert_(globals.system_definition);
        self.assert_(globals.system);
        self.assertEqual(globals.system_definition.getParticleData().getNGlobal(), 3);
        self.assertAlmostEqual(sys.particles[0].position[2],-1,5)

    # checks for an error if initialized twice
    def test_inittwice(self):
        init.read_xml('test.xml');
        self.assertRaises(RuntimeError, init.read_xml, 'test.xml');

    def tearDown(self):
        if (comm.get_rank()==0):
            os.remove("test.xml");
            os.remove("test_out_of_box.xml");
        init.reset();

if __name__ == '__main__':
    unittest.main(argv = ['test.py', '-v'])
<|endoftext|>


<commit_before>#coding: latin-1

import numpy as np
import cv2

import serial

import time
from struct import *

import sys, os, select

import socket

import Proprioceptive as prop
import PicameraStreamer as pcs

obj = pcs.VideoStreamer()

import thread

thread.start_new_thread( obj.connect, () )

# Initialize UDP Controller Server on port 10001
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_address = ('0.0.0.0', 10001)
print >> sys.stderr, 'Starting up Controller Server on %s port %s', server_address
sock.bind(server_address)

hidraw = prop.setupsensor()

[ssmr, mtrn] = prop.serialcomm()

# Instruct the Sensorimotor Cortex to stop wandering.
ssmr.write('C')

tgt = -1000

wristpos=48

while(True):
    try:
        data, address = sock.recvfrom(1)

        if (data == 'Y'):
            mtrn.write('A3250')
            time.sleep(0.8)
            mtrn.write('A5000')
        elif (data=='J'):
            # mtrn.write('A6180')
            wristpos = wristpos + 1
            mtrn.write('A6'+'{:3d}'.format(wristpos))
        elif (data=='j'):
            # mtrn.write('A6090')
            wristpos = wristpos - 1
            mtrn.write('A6'+'{:3d}'.format(wristpos))
        elif (data == 'H'):
            mtrn.write('A4250')
            time.sleep(0.2)
            mtrn.write('A5000')
        elif (data=='G'):
            mtrn.write('A1000')
            time.sleep(2)
        elif (data=='R'):
            mtrn.write('A1200')
            time.sleep(2)
        elif (data==' '):
            ssmr.write('1')
        elif (data=='W'):
            ssmr.write('2')
        elif (data=='S'):
            ssmr.write('3')
        elif (data=='D'):
            ssmr.write('5')
        elif (data=='A'):
            ssmr.write('4')
        elif (data=='.'):
            ssmr.write('-')
        elif (data==','):
            ssmr.write('+')
        elif (data=='L'):
            ssmr.write('L')
        elif (data=='l'):
            ssmr.write('l')
        elif (data=='+'):
            tgt = tgt + 100
        elif (data=='-'):
            tgt = tgt - 100
        elif (data=='T'):
            prop.moveto(mtrn, hidraw, tgt)
        elif (data=='X'):
            break
    except:
        print "Waiting for serial connection to reestablish..."
        time.sleep(2)
        ssmr.close()
        mtrn.close()
        [ssmr, mtrn] = prop.serialcomm()

        # Instruct the Sensorimotor Cortex to stop wandering.
        ssmr.write('C')

obj.keeprunning = False
time.sleep(2)

#When everything done, release the capture
mtrn.close()
ssmr.close()
sock.close()
<commit_msg>Now NeoCortex is filled with sensorimotor information<commit_after>#coding: latin-1

import numpy as np
import cv2

import serial

import time
from struct import *

import sys, os, select

import socket

import Proprioceptive as prop
import PicameraStreamer as pcs
import SensorimotorLogger as senso

obj = pcs.VideoStreamer()

import thread

thread.start_new_thread( obj.connect, () )

# Initialize UDP Controller Server on port 10001
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_address = ('0.0.0.0', 10001)
print >> sys.stderr, 'Starting up Controller Server on %s port %s', server_address
sock.bind(server_address)

hidraw = prop.setupsensor()

[ssmr, mtrn] = prop.serialcomm()

# Instruct the Sensorimotor Cortex to stop wandering.
ssmr.write('C')

tgt = -1000

wristpos=48

sensor = 0

sensorimotor = senso.Sensorimotor()
sensorimotor.start()
sensorimotor.cleanbuffer(smnr,mtrn)

while(True):
    try:
        data, address = sock.recvfrom(1)

        if (sensor == 1):
            sensorimotor.sendsensorsample(smnr,mtrn)

        if (data == '%'):
            if (sensor == 1):
                sensor = 0
            else
                sensor = 1
        elif (data == 'Y'):
            mtrn.write('A3250')
            time.sleep(0.8)
            mtrn.write('A5000')
        elif (data=='J'):
            # mtrn.write('A6180')
            wristpos = wristpos + 1
            mtrn.write('A6'+'{:3d}'.format(wristpos))
        elif (data=='j'):
            # mtrn.write('A6090')
            wristpos = wristpos - 1
            mtrn.write('A6'+'{:3d}'.format(wristpos))
        elif (data == 'H'):
            mtrn.write('A4250')
            time.sleep(0.2)
            mtrn.write('A5000')
        elif (data=='G'):
            mtrn.write('A1000')
            time.sleep(2)
        elif (data=='R'):
            mtrn.write('A1200')
            time.sleep(2)
        elif (data==' '):
            ssmr.write('1')
        elif (data=='W'):
            ssmr.write('2')
        elif (data=='S'):
            ssmr.write('3')
        elif (data=='D'):
            ssmr.write('5')
        elif (data=='A'):
            ssmr.write('4')
        elif (data=='.'):
            ssmr.write('-')
        elif (data==','):
            ssmr.write('+')
        elif (data=='L'):
            ssmr.write('L')
        elif (data=='l'):
            ssmr.write('l')
        elif (data=='+'):
            tgt = tgt + 100
        elif (data=='-'):
            tgt = tgt - 100
        elif (data=='T'):
            prop.moveto(mtrn, hidraw, tgt)
        elif (data=='X'):
            break
    except:
        print "Waiting for serial connection to reestablish..."
        time.sleep(2)
        ssmr.close()
        mtrn.close()
        [ssmr, mtrn] = prop.serialcomm()

        # Instruct the Sensorimotor Cortex to stop wandering.
        ssmr.write('C')

obj.keeprunning = False
time.sleep(2)

#When everything done, release the capture
mtrn.close()
ssmr.close()
sock.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-
#  Copyright (c) 2014 unfoldingWord
#  http://creativecommons.org/licenses/MIT/
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
#
# Author: Joel Neutrino <joel@neutrinographics.com>
# Revised: Jesse Griffin <jesse@distantshores.org>

"""
This script splits the USFM files from T4T (http://ebible.org/t4t/) into
separate chapters and does some basic text to footnote conversions.
"""

import os
import re
import sys
import argparse
import zipfile
import os.path
import shutil
import glob
import codecs

# To Do:
# Alternatives marked with:
#  -> ◄ option1/option2 ► (e.g. Ps 100)
#    --> Alternative within above is marked by < suboption1/suboption2 > (Not found)

footnote = ur'\\f + \\ft {0} \\f*'
orp = re.compile(r'([(]OR.*[)])', flags=16) # Matches: (OR alternative text)
slashp = re.compile(r'(\w*)/(\w*)', flags=32) # Matches: option1/option2
abbvtable = {
  u'[APO]': u'apostrophe',
  u'[CHI]': u'chiasmus',
  u'[DOU]': u'doublet',
  u'[EUP]': u'euphemism',
  u'[HEN]': u'hendiadys',
  u'[HYP]': u'hyperbole',
  u'[IDM]': u'idiom',
  u'[IRO]': u'irony',
  u'[LIT]': u'litotes',
  u'[MET]': u'metaphor',
  u'[MTY]': u'metonymy',
  u'[PRS]': u'personification',
  u'[RHQ]': u'rhetorical question',
  u'[SIM]': u'simile',
  u'[SYM]': u'symbol',
  u'[SAR]': u'sarcasm',
  u'[SYN]': u'synecdoche',
  u'[TRI]': u'triple',
}

def main(arguments):

  parser = argparse.ArgumentParser(description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-o', '--output', dest="outputDir", default=False,
    required=True, help = "Output Directory")
  parser.add_argument('infile', help = "Input file",
    type = argparse.FileType('r'))
  parser.add_argument('-c', '--clean', dest="clean", default=False, action="store_const", const="sum", help = "Clean workspace")

  args = parser.parse_args(arguments)

  sourceDir = 'usfm_t4t_source/'
  if args.outputDir:
    outputDir = args.outputDir
    if not os.path.isdir(outputDir):
      os.mkdir(outputDir)

  bookTags = ['\id', '\ide', '\h', '\\toc1', '\\toc2', '\\toc3', '\mt']

  # clean the workspace
  if args.clean:
    print('cleaning workspace..')
    if os.path.isdir(outputDir) or os.path.isdir(args.infile.name):
      if os.path.isdir(outputDir):
        shutil.rmtree(outputDir)

      if os.path.isdir(args.infile.name):
        shutil.rmtree(args.infile.name)
    print('done')
    return

  # extract the archive
  unzip(args.infile.name, sourceDir)

  # list books
  fileList = []
  files = glob.glob(sourceDir + '*.usfm')
  for path in files:
    # open a new book
    f = codecs.open(path, encoding='utf-8')
    bookHeader = []
    bookID = None
    chapterNum = None
    chapterLines = [] # emptied after each chapter

    lines = f.readlines()
    f.close()
    for l in lines:
      tag = l.split(None,1)[0]
        # load the book header
      if tag in bookTags:
        bookHeader.append(l)
        #if tag == '\h':
          #print('exploding '+l.split(None,1)[1])
        if tag == '\id':
          bookID = l.split(None,2)[1]
      else:
        if tag == '\c':
          # write finished chapter
          if chapterNum is not None:
            book = str(outputDir+bookID).lower()
            if not os.path.isdir(book):
              os.mkdir(book)
            outputFile = book+'/'+chapterNum+'.usfm.txt'
            writeFile(outputFile, convert(''.join(chapterLines)))
            fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))

          # begin reading new chapter
          chapterNum = l.split(None,2)[1]
          chapterLines = []
          chapterLines.append(l)
        else:
          # load chapter lines
          chapterLines.append(l)
    # write the last chapter
    if chapterNum is not None:
      book = str(outputDir+bookID).lower()
      if not os.path.isdir(book):
        os.mkdir(book)
      outputFile = book+'/'+chapterNum+'.usfm.txt'
      writeFile(outputFile, convert(''.join(chapterLines)))
      fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))
    # write book information
    book = str(outputDir+bookID).lower()
    if not os.path.isdir(book):
      os.mkdir(book)
    outputFile = book+'/0.usfm.txt'
    writeFile(outputFile, ''.join(bookHeader))
    fileList.append('{0}:0.usfm'.format(bookID))

  # clean up after ourselves
  shutil.rmtree(sourceDir)

  f = codecs.open('{0}/uwb.txt'.format(outputDir), encoding='utf-8', mode='w')
  for e in fileList:
    f.write('  * [[playground:uwb:{0}|{1}]]\n'.format(e.lower(),
      e.strip('.usfm').replace(':', ' ')))
  f.close()

def writeFile(f, content):
  out = codecs.open(f, encoding='utf-8', mode='w')
  out.write(content)
  out.close()

def convert(f):
  '''
  Converts T4T features into footnotes.
  '''
  f = orp.sub(footnote.format(r'\1'), f)
  f = slashp.sub(r'\1{0}'.format(footnote.format(r'Or: \2')), f)
  for k,v in abbvtable.iteritems():
    f = f.replace(k, footnote.format(v))
  return f

def unzip(source, dest):
  with zipfile.ZipFile(source) as zf:
    for member in zf.infolist():
      # Path traversal defense copied from
      # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
      words = member.filename.split('/')
      path = dest
      for word in words[:-1]:
        drive, word = os.path.splitdrive(word)
        head, word = os.path.split(word)
        if word in (os.curdir, os.pardir, ''): continue
        path = os.path.join(path, word)
      zf.extract(member, path)

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>Added more alternative replacements<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
#  Copyright (c) 2014 unfoldingWord
#  http://creativecommons.org/licenses/MIT/
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.
#
# Author: Joel Neutrino <joel@neutrinographics.com>
# Revised: Jesse Griffin <jesse@distantshores.org>

"""
This script splits the USFM files from T4T (http://ebible.org/t4t/) into
separate chapters and does some basic text to footnote conversions.
"""

import os
import re
import sys
import argparse
import zipfile
import os.path
import shutil
import glob
import codecs

footnote = u'\\f + \\ft {0} \\f*'
refootnote = ur'\\f + \\ft {0} \\f*'
arrowp = re.compile(ur'◄([^/]*)/(.*)►', flags=32) # Matches: ◄option1/option2►
slashinft = re.compile(ur'(\\ft .*)/', flags=32) # Matches: \ft option2/option3
orp = re.compile(ur'([(]OR.*[)])', flags=16) # Matches: (OR alternative text)
slashp = re.compile(ur'(\w*)/(\w*)', flags=32) # Matches: option1/option2
abbvtable = {
  u'[APO]': u'apostrophe',
  u'[CHI]': u'chiasmus',
  u'[DOU]': u'doublet',
  u'[EUP]': u'euphemism',
  u'[HEN]': u'hendiadys',
  u'[HYP]': u'hyperbole',
  u'[IDM]': u'idiom',
  u'[IRO]': u'irony',
  u'[LIT]': u'litotes',
  u'[MET]': u'metaphor',
  u'[MTY]': u'metonymy',
  u'[PRS]': u'personification',
  u'[RHQ]': u'rhetorical question',
  u'[SIM]': u'simile',
  u'[SYM]': u'symbol',
  u'[SAR]': u'sarcasm',
  u'[SYN]': u'synecdoche',
  u'[TRI]': u'triple',
}

def main(arguments):

  parser = argparse.ArgumentParser(description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-o', '--output', dest="outputDir", default=False,
    required=True, help = "Output Directory")
  parser.add_argument('infile', help = "Input file",
    type = argparse.FileType('r'))
  parser.add_argument('-c', '--clean', dest="clean", default=False, action="store_const", const="sum", help = "Clean workspace")

  args = parser.parse_args(arguments)

  sourceDir = 'usfm_t4t_source/'
  if args.outputDir:
    outputDir = args.outputDir
    if not os.path.isdir(outputDir):
      os.mkdir(outputDir)

  bookTags = ['\id', '\ide', '\h', '\\toc1', '\\toc2', '\\toc3', '\mt']

  # clean the workspace
  if args.clean:
    print('cleaning workspace..')
    if os.path.isdir(outputDir) or os.path.isdir(args.infile.name):
      if os.path.isdir(outputDir):
        shutil.rmtree(outputDir)

      if os.path.isdir(args.infile.name):
        shutil.rmtree(args.infile.name)
    print('done')
    return

  # extract the archive
  unzip(args.infile.name, sourceDir)

  # list books
  fileList = []
  files = glob.glob(sourceDir + '*.usfm')
  for path in files:
    # open a new book
    f = codecs.open(path, encoding='utf-8')
    bookHeader = []
    bookID = None
    chapterNum = None
    chapterLines = [] # emptied after each chapter

    lines = f.readlines()
    f.close()
    for l in lines:
      tag = l.split(None,1)[0]
        # load the book header
      if tag in bookTags:
        bookHeader.append(l)
        #if tag == '\h':
          #print('exploding '+l.split(None,1)[1])
        if tag == '\id':
          bookID = l.split(None,2)[1]
      else:
        if tag == '\c':
          # write finished chapter
          if chapterNum is not None:
            book = str(outputDir+bookID).lower()
            if not os.path.isdir(book):
              os.mkdir(book)
            outputFile = book+'/'+chapterNum+'.usfm.txt'
            writeFile(outputFile, convert(''.join(chapterLines)))
            fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))

          # begin reading new chapter
          chapterNum = l.split(None,2)[1]
          chapterLines = []
          chapterLines.append(l)
        else:
          # load chapter lines
          chapterLines.append(l)
    # write the last chapter
    if chapterNum is not None:
      book = str(outputDir+bookID).lower()
      if not os.path.isdir(book):
        os.mkdir(book)
      outputFile = book+'/'+chapterNum+'.usfm.txt'
      writeFile(outputFile, convert(''.join(chapterLines)))
      fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))
    # write book information
    book = str(outputDir+bookID).lower()
    if not os.path.isdir(book):
      os.mkdir(book)
    outputFile = book+'/0.usfm.txt'
    writeFile(outputFile, ''.join(bookHeader))
    fileList.append('{0}:0.usfm'.format(bookID))

  # clean up after ourselves
  shutil.rmtree(sourceDir)

  f = codecs.open('{0}/uwb.txt'.format(outputDir), encoding='utf-8', mode='w')
  for e in fileList:
    f.write('  * [[playground:uwb:{0}|{1}]]\n'.format(e.lower(),
      e.strip('.usfm').replace(':', ' ')))
  f.close()

def writeFile(f, content):
  out = codecs.open(f, encoding='utf-8', mode='w')
  out.write(content)
  out.close()

def convert(f):
  '''
  Converts T4T features into footnotes.
  '''
  f = arrowp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  f = slashinft.sub(r'\1, Or: ', f)
  f = orp.sub(refootnote.format(ur'\1'), f)
  f = slashp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  for k,v in abbvtable.iteritems():
    f = f.replace(k, footnote.format(v))
  return f

def unzip(source, dest):
  with zipfile.ZipFile(source) as zf:
    for member in zf.infolist():
      # Path traversal defense copied from
      # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
      words = member.filename.split('/')
      path = dest
      for word in words[:-1]:
        drive, word = os.path.splitdrive(word)
        head, word = os.path.split(word)
        if word in (os.curdir, os.pardir, ''): continue
        path = os.path.join(path, word)
      zf.extract(member, path)

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before><commit_msg>Working on image processing<commit_after>import sqlite3
import logging
from PIL import Image

class SqliteManager:

    """
    Import pictures taken by users.
    The database will be filled with entries like :
        "rgb color in hex format" : "directory where a picture with this average color can be found"
    eg. "33CC66" : "folder1"

    For now, the color code will be only one byte ("80"), from "00" to "FF"
    as we'll work with black&white pictures only
    """

    def __init__(self, database):
        self.connection = sqlite3.connect(database)

    def close_connection(self):
        self.connection.close()

    def create_event(self, event_name):
        cursor = self.connection.cursor()
        cursor.execute('CREATE TABLE ' + event_name + ' (color text, path text)')  # ohh la belle injection des familles
        self.connection.commit()

    '''
    @eventName is the name of the event where the picture was taken. It's a table name in the database
    @raw is a list ('color', 'path')
    '''
    def insert_picture(self, event_name, row):
        # Try opening the image and analyse it :

        # Insert picture into database
        cursor = self.connection.cursor()
        logging.info("Row for insertion : ")
        logging.info(row)
        cursor.execute('INSERT INTO ' + event_name + ' VALUES (?,?)', row)
        self.connection.commit()
        logging.info('New picture inserted (color = ' + row[0] + ', path = ' + row[1] + ')')<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Scene.SceneNode import SceneNode
from UM.Resources import Resources
from UM.Math.Color import Color
from UM.Mesh.MeshBuilder import MeshBuilder  # To create a mesh to display the convex hull with.

from UM.View.GL.OpenGL import OpenGL

class ConvexHullNode(SceneNode):
    ##  Convex hull node is a special type of scene node that is used to display an area, to indicate the
    #   location an object uses on the buildplate. This area (or area's in case of one at a time printing) is
    #   then displayed as a transparent shadow. If the adhesion type is set to raft, the area is extruded
    #   to represent the raft as well.
    def __init__(self, node, hull, thickness, parent = None):
        super().__init__(parent)

        self.setCalculateBoundingBox(False)

        self._shader = None

        self._original_parent = parent

        # Color of the drawn convex hull
        self._color = Color(0.4, 0.4, 0.4, 1.0)

        # The y-coordinate of the convex hull mesh. Must not be 0, to prevent z-fighting.
        self._mesh_height = 0.1

        self._thickness = thickness

        # The node this mesh is "watching"
        self._node = node
        self._convex_hull_head_mesh = None

        self._node.decoratorsChanged.connect(self._onNodeDecoratorsChanged)
        self._onNodeDecoratorsChanged(self._node)

        self._hull = hull
        if self._hull:
            hull_mesh_builder = MeshBuilder()

            if hull_mesh_builder.addConvexPolygonExtrusion(
                self._hull.getPoints()[::-1],  # bottom layer is reversed
                self._mesh_height-thickness, self._mesh_height, color=self._color):

                hull_mesh = hull_mesh_builder.build()
                self.setMeshData(hull_mesh)

    def getHull(self):
        return self._hull

    def getThickness(self):
        return self._thickness

    def getWatchedNode(self):
        return self._node

    def render(self, renderer):
        if not self._shader:
            self._shader = OpenGL.getInstance().createShaderProgram(Resources.getPath(Resources.Shaders, "transparent_object.shader"))
            self._shader.setUniformValue("u_diffuseColor", self._color)
            self._shader.setUniformValue("u_opacity", 0.6)

        if self.getParent():
            if self.getMeshData():
                renderer.queueNode(self, transparent = True, shader = self._shader, backface_cull = True, sort = -8)
                if self._convex_hull_head_mesh:
                    renderer.queueNode(self, shader = self._shader, transparent = True, mesh = self._convex_hull_head_mesh, backface_cull = True, sort = -8)

        return True

    def _onNodeDecoratorsChanged(self, node):
        self._color = Color(35, 35, 35, 0.5)

        convex_hull_head = self._node.callDecoration("getConvexHullHead")
        if convex_hull_head:
            convex_hull_head_builder = MeshBuilder()
            convex_hull_head_builder.addConvexPolygon(convex_hull_head.getPoints(), self._mesh_height-self._thickness)
            self._convex_hull_head_mesh = convex_hull_head_builder.build()

        if not node:
            return

<commit_msg>Layout. CURA-1707<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Scene.SceneNode import SceneNode
from UM.Resources import Resources
from UM.Math.Color import Color
from UM.Mesh.MeshBuilder import MeshBuilder  # To create a mesh to display the convex hull with.

from UM.View.GL.OpenGL import OpenGL

class ConvexHullNode(SceneNode):
    ##  Convex hull node is a special type of scene node that is used to display an area, to indicate the
    #   location an object uses on the buildplate. This area (or area's in case of one at a time printing) is
    #   then displayed as a transparent shadow. If the adhesion type is set to raft, the area is extruded
    #   to represent the raft as well.
    def __init__(self, node, hull, thickness, parent = None):
        super().__init__(parent)

        self.setCalculateBoundingBox(False)

        self._shader = None

        self._original_parent = parent

        # Color of the drawn convex hull
        self._color = Color(0.4, 0.4, 0.4, 1.0)

        # The y-coordinate of the convex hull mesh. Must not be 0, to prevent z-fighting.
        self._mesh_height = 0.1

        self._thickness = thickness

        # The node this mesh is "watching"
        self._node = node
        self._convex_hull_head_mesh = None

        self._node.decoratorsChanged.connect(self._onNodeDecoratorsChanged)
        self._onNodeDecoratorsChanged(self._node)

        self._hull = hull
        if self._hull:
            hull_mesh_builder = MeshBuilder()

            if hull_mesh_builder.addConvexPolygonExtrusion(
                self._hull.getPoints()[::-1],  # bottom layer is reversed
                self._mesh_height-thickness, self._mesh_height, color=self._color):

                hull_mesh = hull_mesh_builder.build()
                self.setMeshData(hull_mesh)

    def getHull(self):
        return self._hull

    def getThickness(self):
        return self._thickness

    def getWatchedNode(self):
        return self._node

    def render(self, renderer):
        if not self._shader:
            self._shader = OpenGL.getInstance().createShaderProgram(Resources.getPath(Resources.Shaders, "transparent_object.shader"))
            self._shader.setUniformValue("u_diffuseColor", self._color)
            self._shader.setUniformValue("u_opacity", 0.6)

        if self.getParent():
            if self.getMeshData():
                renderer.queueNode(self, transparent = True, shader = self._shader, backface_cull = True, sort = -8)
                if self._convex_hull_head_mesh:
                    renderer.queueNode(self, shader = self._shader, transparent = True, mesh = self._convex_hull_head_mesh, backface_cull = True, sort = -8)

        return True

    def _onNodeDecoratorsChanged(self, node):
        self._color = Color(35, 35, 35, 0.5)

        convex_hull_head = self._node.callDecoration("getConvexHullHead")
        if convex_hull_head:
            convex_hull_head_builder = MeshBuilder()
            convex_hull_head_builder.addConvexPolygon(convex_hull_head.getPoints(), self._mesh_height - self._thickness)
            self._convex_hull_head_mesh = convex_hull_head_builder.build()

        if not node:
            return

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  Copyright (c) 2014 unfoldingWord
#  http://creativecommons.org/licenses/MIT/
#  See LICENSE file for details.
#
#  Contributors:
#  Joel Neutrino <joel@neutrinographics.com>
#  Jesse Griffin <jesse@distantshores.org>

"""
This script splits the USFM files from T4T (http://ebible.org/t4t/) into
separate chapters and does some basic text to footnote conversions.
"""

import os
import re
import sys
import glob
import codecs
import shutil
import urllib2
import zipfile
import argparse
import datetime

# USFM footnote syntax strings
footnote = u'\\f + \\ft {0} \\f*'
refootnote = ur'\\f + \\ft {0} \\f*'

# Pattern matching, used in convert()
# Matches arrow pattern: ◄option1/option2►
arrowp = re.compile(ur'◄([^/]*)/([^►]*)►', re.UNICODE)

# Matches alternative in footnote pattern: \ft option2/option3
slashinft = re.compile(ur'(\\ft .*)/', re.UNICODE)

# Matches alternative text pattern: (OR alternative text)
orp = re.compile(ur'[(]OR,([^)]*)[)]', re.DOTALL | re.UNICODE)

# Matches alternative pattern: option1/option2
slashp = re.compile(ur'(\w*)/(\w*)', re.UNICODE)

# Matches brace pattern: {possible added}
bracep = re.compile(ur'{([^}]*)}', re.UNICODE)

# Matches footnote in footnote
ftinft = re.compile(ur'(\\f \+ \\ft)([^*]*)\\f \+ \\ft([^\\]*)\\f\*',
  re.DOTALL | re.UNICODE)

# Matches Israeli or Israelis
israelisp = re.compile(ur'Israelis', re.UNICODE)
israelip = re.compile(ur'Israeli([^t])', re.UNICODE)

# Pronoun notes abbreviation table
prntable = {
  u'(inc)': u'inclusive',
  u'(exc)': u'exclusive',
  u'(sg)': u'singular',
  u'(pl)': u'plural',
}
# Figures abbreviation table
abbvtable = {
  u'APO': u'apostrophe',
  u'CHI': u'chiasmus',
  u'DOU': u'doublet',
  u'EUP': u'euphemism',
  u'HEN': u'hendiadys',
  u'HYP': u'hyperbole',
  u'IDM': u'idiom',
  u'IRO': u'irony',
  u'LIT': u'litotes',
  u'MET': u'metaphor',
  u'MTY': u'metonymy',
  u'PRS': u'personification',
  u'RHQ': u'rhetorical question',
  u'SIM': u'simile',
  u'SYM': u'symbol',
  u'SAR': u'sarcasm',
  u'SYN': u'synecdoche',
  u'TRI': u'triple',
}

def main(arguments):

  parser = argparse.ArgumentParser(description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-o', '--output', dest="outputDir", default=False,
    required=True, help = "Output Directory")
  parser.add_argument('-c', '--clean', dest="clean", default=False, action="store_const", const="sum", help = "Clean workspace")

  args = parser.parse_args(arguments)

  sourceDir = 'usfm_t4t_source/'
  if args.outputDir:
    outputDir = args.outputDir
    if not os.path.isdir(outputDir):
      os.mkdir(outputDir)

  bookTags = ['\id', '\ide', '\h', '\\toc1', '\\toc2', '\\toc3', '\mt']
  t4tzip = '/tmp/eng-t4t_usfm.zip_{0}'.format(str(datetime.date.today()))

  # clean the workspace
  if args.clean:
    print('cleaning workspace..')
    if os.path.isdir(outputDir):
      shutil.rmtree(outputDir)
    os.remove(t4tzip)
    print('done')
    return

  # Download T4T zip
  if not os.path.isfile(t4tzip):
    getURL('http://ebible.org/t4t/eng-t4t_usfm.zip', t4tzip)

  # extract the archive
  unzip(t4tzip, sourceDir)

  # list books
  fileList = []
  files = glob.glob(sourceDir + '*.usfm')
  for path in files:
    # open a new book
    f = codecs.open(path, encoding='utf-8')
    bookHeader = []
    bookID = None
    chapterNum = None
    chapterLines = [] # emptied after each chapter

    lines = f.readlines()
    f.close()
    for l in lines:
      tag = l.split(None,1)[0]
        # load the book header
      if tag in bookTags:
        bookHeader.append(l)
        #if tag == '\h':
          #print('exploding '+l.split(None,1)[1])
        if tag == '\id':
          bookID = l.split(None,2)[1]
      else:
        if tag == '\c':
          # write finished chapter
          if chapterNum is not None:
            book = str(outputDir+bookID).lower()
            if not os.path.isdir(book):
              os.mkdir(book)
            outputFile = book+'/'+chapterNum+'.usfm.txt'
            writeFile(outputFile, convert(''.join(chapterLines)))
            fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))

          # begin reading new chapter
          chapterNum = str(l.split(None,2)[1]).zfill(3)
          chapterLines = []
          chapterLines.append(l)
        else:
          # load chapter lines
          chapterLines.append(l)
    # write the last chapter
    if chapterNum is not None:
      book = str(outputDir+bookID).lower()
      if not os.path.isdir(book):
        os.mkdir(book)
      outputFile = book+'/'+chapterNum+'.usfm.txt'
      writeFile(outputFile, convert(''.join(chapterLines)))
      fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))
    # write book information
    book = str(outputDir+bookID).lower()
    if not os.path.isdir(book):
      os.mkdir(book)
    outputFile = book+'/000.usfm.txt'
    writeFile(outputFile, ''.join(bookHeader))
    fileList.append('{0}:000.usfm'.format(bookID))

  # clean up after ourselves
  shutil.rmtree(sourceDir)

  f = codecs.open('{0}/uwb.txt'.format(outputDir), encoding='utf-8', mode='w')
  for e in fileList:
    f.write('  * [[en:utb:v1:{0}|{1}]]\n'.format(e.lower(),
      e.strip('.usfm').replace(':', ' ')))
  f.close()

def writeFile(f, content):
  out = codecs.open(f, encoding='utf-8', mode='w')
  out.write(content)
  out.close()

def convert(f):
  '''
  Converts T4T features into footnotes.
  '''
  f = (f.replace(u'“', '"').replace(u'”', '"').replace(u'’', "'")
        .replace(u'‘', "'"))
  f = arrowp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  f = bracep.sub(refootnote.format(ur'Or:\1'), f)
  f = slashinft.sub(ur'\1, Or: ', f)
  f = israelisp.sub(ur'Israelites', f)
  f = israelip.sub(ur'Israelite\1', f)
  for k,v in prntable.iteritems():
    f = f.replace(u'{0}'.format(k), footnote.format(v))
  f = orp.sub(refootnote.format(ur'Or:\1'), f)
  f = slashp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  for k,v in abbvtable.iteritems():
    f = f.replace(u'[{0}]'.format(k), footnote.format(v))
    for k2,v2 in abbvtable.iteritems():
      f = f.replace(u'[{0}, {1}]'.format(k, k2), footnote.format(u'{0}, {1}'
        .format(v, v2)))
  f = ftinft.sub(ur'\1\2(\3)', f)
  return f

def unzip(source, dest):
  with zipfile.ZipFile(source) as zf:
    for member in zf.infolist():
      # Path traversal defense copied from
      # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
      words = member.filename.split('/')
      path = dest
      for word in words[:-1]:
        drive, word = os.path.splitdrive(word)
        head, word = os.path.split(word)
        if word in (os.curdir, os.pardir, ''): continue
        path = os.path.join(path, word)
      zf.extract(member, path)

def getURL(url, outfile):
  print "Getting %s" % url
  try:
    request = urllib2.urlopen(url)
  except:
    print "  => ERROR retrieving %s\nCheck the URL" % url
    sys.exit(1)
  print "  => Writing to %s" % outfile
  with open(outfile, 'wb') as fp:
    shutil.copyfileobj(request, fp)

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>Added license to t4t2dw.py import<commit_after>#!/usr/bin/env python
# -*- coding: utf8 -*-
#
#  Copyright (c) 2014 unfoldingWord
#  http://creativecommons.org/licenses/MIT/
#  See LICENSE file for details.
#
#  Contributors:
#  Joel Neutrino <joel@neutrinographics.com>
#  Jesse Griffin <jesse@distantshores.org>

"""
This script splits the USFM files from T4T (http://ebible.org/t4t/) into
separate chapters and does some basic text to footnote conversions.
"""

import os
import re
import sys
import glob
import codecs
import shutil
import urllib2
import zipfile
import argparse
import datetime

LICENSE = u'''~~NOCACHE~~
\mt unfoldingWord | Translator's Bible

\p \\bd an unrestricted Bible intended for translation into any language \\bd*

\p \em http://unfoldingWord.org/Bible \em*

\p unfoldingWord Translator's Bible, v. 0.1

This work is based on \em The Translation for Translators \em* by Ellis Deibler, available under the terms of a Creative Commons Attribution-ShareAlike 4.0 International License (http://creativecommons.org/licenses/by-sa/4.0/).

\p License:

\p This work is made available under a Creative Commons Attribution-ShareAlike 4.0 International License (http://creativecommons.org/licenses/by-sa/4.0/).

\p You are free:

\p \\bd Share \\bd* — copy and redistribute the material in any medium or format
\p \\bd Adapt \\bd* — remix, transform, and build upon the material for any purpose, even commercially.

\p Under the following conditions:

\p \\bd Attribution \\bd* — You must attribute the work as follows: "Original work available at http://openbiblestories.com." Attribution statements in derivative works should not in any way suggest that we endorse you or your use of this work.
\p \\bd ShareAlike \\bd* — If you remix, transform, or build upon the material, you must distribute your contributions under the same license as the original.

\p Use of trademarks: \\bd unfoldingWord \\bd* is a trademark of Distant Shores Media and may not be included on any derivative works created from this content.  Unaltered content from http://unfoldingWord.org must include the \\bd unfoldingWord \\bd* logo when distributed to others. But if you alter the content in any way, you must remove the \\bd unfoldingWord \\bd* logo before distributing your work.
'''

# USFM footnote syntax strings
footnote = u'\\f + \\ft {0} \\f*'
refootnote = ur'\\f + \\ft {0} \\f*'

# Pattern matching, used in convert()
# Matches arrow pattern: ◄option1/option2►
arrowp = re.compile(ur'◄([^/]*)/([^►]*)►', re.UNICODE)

# Matches alternative in footnote pattern: \ft option2/option3
slashinft = re.compile(ur'(\\ft .*)/', re.UNICODE)

# Matches alternative text pattern: (OR alternative text)
orp = re.compile(ur'[(]OR,([^)]*)[)]', re.DOTALL | re.UNICODE)

# Matches alternative pattern: option1/option2
slashp = re.compile(ur'(\w*)/(\w*)', re.UNICODE)

# Matches brace pattern: {possible added}
bracep = re.compile(ur'{([^}]*)}', re.UNICODE)

# Matches footnote in footnote
ftinft = re.compile(ur'(\\f \+ \\ft)([^*]*)\\f \+ \\ft([^\\]*)\\f\*',
  re.DOTALL | re.UNICODE)

# Matches Israeli or Israelis
israelisp = re.compile(ur'Israelis', re.UNICODE)
israelip = re.compile(ur'Israeli([^t])', re.UNICODE)

# Pronoun notes abbreviation table
prntable = {
  u'(inc)': u'inclusive',
  u'(exc)': u'exclusive',
  u'(sg)': u'singular',
  u'(pl)': u'plural',
}
# Figures abbreviation table
abbvtable = {
  u'APO': u'apostrophe',
  u'CHI': u'chiasmus',
  u'DOU': u'doublet',
  u'EUP': u'euphemism',
  u'HEN': u'hendiadys',
  u'HYP': u'hyperbole',
  u'IDM': u'idiom',
  u'IRO': u'irony',
  u'LIT': u'litotes',
  u'MET': u'metaphor',
  u'MTY': u'metonymy',
  u'PRS': u'personification',
  u'RHQ': u'rhetorical question',
  u'SIM': u'simile',
  u'SYM': u'symbol',
  u'SAR': u'sarcasm',
  u'SYN': u'synecdoche',
  u'TRI': u'triple',
}

def main(arguments):

  parser = argparse.ArgumentParser(description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-o', '--output', dest="outputDir", default=False,
    required=True, help = "Output Directory")
  parser.add_argument('-c', '--clean', dest="clean", default=False, action="store_const", const="sum", help = "Clean workspace")

  args = parser.parse_args(arguments)

  sourceDir = 'usfm_t4t_source/'
  if args.outputDir:
    outputDir = args.outputDir
    if not os.path.isdir(outputDir):
      os.mkdir(outputDir)

  bookTags = ['\id', '\ide', '\h', '\\toc1', '\\toc2', '\\toc3', '\mt']
  t4tzip = '/tmp/eng-t4t_usfm.zip_{0}'.format(str(datetime.date.today()))

  # clean the workspace
  if args.clean:
    print('cleaning workspace..')
    if os.path.isdir(outputDir):
      shutil.rmtree(outputDir)
    os.remove(t4tzip)
    print('done')
    return

  # Download T4T zip
  if not os.path.isfile(t4tzip):
    getURL('http://ebible.org/t4t/eng-t4t_usfm.zip', t4tzip)

  # extract the archive
  unzip(t4tzip, sourceDir)

  # list books
  fileList = []
  files = glob.glob(sourceDir + '*.usfm')
  for path in files:
    # open a new book
    f = codecs.open(path, encoding='utf-8')
    bookHeader = []
    bookID = None
    chapterNum = None
    chapterLines = [] # emptied after each chapter

    lines = f.readlines()
    f.close()
    for l in lines:
      tag = l.split(None,1)[0]
        # load the book header
      if tag in bookTags:
        bookHeader.append(l)
        #if tag == '\h':
          #print('exploding '+l.split(None,1)[1])
        if tag == '\id':
          bookID = l.split(None,2)[1]
      else:
        if tag == '\c':
          # write finished chapter
          if chapterNum is not None:
            book = str(outputDir+bookID).lower()
            if not os.path.isdir(book):
              os.mkdir(book)
            outputFile = book+'/'+chapterNum+'.usfm.txt'
            writeFile(outputFile, convert(''.join(chapterLines)))
            fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))

          # begin reading new chapter
          chapterNum = str(l.split(None,2)[1]).zfill(3)
          chapterLines = []
          chapterLines.append(l)
        else:
          # load chapter lines
          chapterLines.append(l)
    # write the last chapter
    if chapterNum is not None:
      book = str(outputDir+bookID).lower()
      if not os.path.isdir(book):
        os.mkdir(book)
      outputFile = book+'/'+chapterNum+'.usfm.txt'
      writeFile(outputFile, convert(''.join(chapterLines)))
      fileList.append('{0}:{1}.usfm'.format(bookID, chapterNum))
    # write book information
    book = str(outputDir+bookID).lower()
    if not os.path.isdir(book):
      os.mkdir(book)
    outputFile = book+'/000.usfm.txt'
    writeFile(outputFile, ''.join(bookHeader))
    fileList.append('{0}:000.usfm'.format(bookID))

  # clean up after ourselves
  shutil.rmtree(sourceDir)

  f = codecs.open('{0}/uwb.txt'.format(outputDir), encoding='utf-8', mode='w')
  for e in fileList:
    f.write('  * [[en:utb:v1:{0}|{1}]]\n'.format(e.lower(),
      e.strip('.usfm').replace(':', ' ')))
  f.close()
  writeFile('{0}/frt/000.usfm.txt'.format(outputDir), LICENSE)

def writeFile(f, content):
  out = codecs.open(f, encoding='utf-8', mode='w')
  out.write(content)
  out.close()

def convert(f):
  '''
  Converts T4T features into footnotes.
  '''
  f = (f.replace(u'“', '"').replace(u'”', '"').replace(u'’', "'")
        .replace(u'‘', "'"))
  f = arrowp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  f = bracep.sub(refootnote.format(ur'Or:\1'), f)
  f = slashinft.sub(ur'\1, Or: ', f)
  f = israelisp.sub(ur'Israelites', f)
  f = israelip.sub(ur'Israelite\1', f)
  for k,v in prntable.iteritems():
    f = f.replace(u'{0}'.format(k), footnote.format(v))
  f = orp.sub(refootnote.format(ur'Or:\1'), f)
  f = slashp.sub(ur'\1{0}'.format(refootnote.format(ur'Or: \2')), f)
  for k,v in abbvtable.iteritems():
    f = f.replace(u'[{0}]'.format(k), footnote.format(v))
    for k2,v2 in abbvtable.iteritems():
      f = f.replace(u'[{0}, {1}]'.format(k, k2), footnote.format(u'{0}, {1}'
        .format(v, v2)))
  f = ftinft.sub(ur'\1\2(\3)', f)
  return f

def unzip(source, dest):
  with zipfile.ZipFile(source) as zf:
    for member in zf.infolist():
      # Path traversal defense copied from
      # http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789
      words = member.filename.split('/')
      path = dest
      for word in words[:-1]:
        drive, word = os.path.splitdrive(word)
        head, word = os.path.split(word)
        if word in (os.curdir, os.pardir, ''): continue
        path = os.path.join(path, word)
      zf.extract(member, path)

def getURL(url, outfile):
  try:
    request = urllib2.urlopen(url)
  except:
    print "  => ERROR retrieving %s\nCheck the URL" % url
    sys.exit(1)
  with open(outfile, 'wb') as fp:
    shutil.copyfileobj(request, fp)

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before>"""
intervaltree: A mutable, self-balancing interval tree for Python 2 and 3.
Queries may be by point, by range overlap, or by range envelopment.

Test module: IntervalTree, Basic query methods (read-only)

Copyright 2013-2015 Chaim-Leib Halbert

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import absolute_import
from intervaltree import Interval, IntervalTree
import pytest
from test.intervaltrees import trees, sdata
try:
    import cPickle as pickle
except ImportError:
    import pickle

def test_empty_queries():
    t = IntervalTree()
    e = set()

    assert len(t) == 0
    assert t.is_empty()
    assert t[3] == e
    assert t[4:6] == e
    assert t.begin() == 0
    assert t.end() == 0
    assert t[t.begin():t.end()] == e
    assert t.items() == e
    assert set(t) == e
    assert set(t.copy()) == e
    assert t.find_nested() == {}
    assert t.range().is_null()
    assert t.range().length() == 0
    t.verify()

def test_queries():
    t = trees['ivs1']()

    assert sdata(t[4]) == set(['[4,7)'])
    assert sdata(t[4:5]) == set(['[4,7)'])
    assert sdata(t[4:6]) == set(['[4,7)', '[5,9)'])
    assert sdata(t[9]) == set(['[6,10)', '[8,10)', '[8,15)'])
    assert sdata(t[15]) == set()
    assert sdata(t.search(5)) == set(['[4,7)', '[5,9)'])
    assert sdata(t.search(6, 11, strict=True)) == set(['[6,10)', '[8,10)'])

def test_partial_slice_query():
    def assert_chop(t, limit):
        s = set(t)
        assert t[:] == s

        s = set(iv for iv in t if iv.begin < limit)
        assert t[:limit] == s

        s = set(iv for iv in t if iv.end > limit)
        assert t[limit:] == s

    assert_chop(trees['ivs1'](), 7)
    assert_chop(trees['ivs2'](), -3)

def test_tree_bounds():
    def assert_tree_bounds(t):
        begin, end, _ = set(t).pop()
        for iv in t:
            if iv.begin < begin: begin = iv.begin
            if iv.end > end: end = iv.end
        assert t.begin() == begin
        assert t.end() == end

    assert_tree_bounds(trees['ivs1']())
    assert_tree_bounds(trees['ivs2']())

def test_membership():
    t = trees['ivs1']()
    assert Interval(1, 2, '[1,2)') in t
    assert Interval(1, 3, '[1,3)') not in t
    assert t.overlaps(4)
    assert t.overlaps(9)
    assert not t.overlaps(15)
    assert t.overlaps(0, 4)
    assert t.overlaps(1, 2)
    assert t.overlaps(1, 3)
    assert t.overlaps(8, 15)
    assert not t.overlaps(15, 16)
    assert not t.overlaps(-1, 0)
    assert not t.overlaps(2, 4)

def test_overlaps_empty():
    # Empty tree
    t = IntervalTree()
    assert not t.overlaps(-1)
    assert not t.overlaps(0)

    assert not t.overlaps(-1, 1)
    assert not t.overlaps(-1, 0)
    assert not t.overlaps(0, 0)
    assert not t.overlaps(0, 1)
    assert not t.overlaps(1, 0)
    assert not t.overlaps(1, -1)
    assert not t.overlaps(0, -1)

    assert not t.overlaps(Interval(-1, 1))
    assert not t.overlaps(Interval(-1, 0))
    assert not t.overlaps(Interval(0, 0))
    assert not t.overlaps(Interval(0, 1))
    assert not t.overlaps(Interval(1, 0))
    assert not t.overlaps(Interval(1, -1))
    assert not t.overlaps(Interval(0, -1))

def test_overlaps():
    t = trees['ivs1']()
    assert not t.overlaps(-3.2)
    assert t.overlaps(1)
    assert t.overlaps(1.5)
    assert t.overlaps(0, 3)
    assert not t.overlaps(0, 1)
    assert not t.overlaps(2, 4)
    assert not t.overlaps(4, 2)
    assert not t.overlaps(3, 0)

def test_span():
    e = IntervalTree()
    assert e.span() == 0

    t = trees['ivs1']()
    assert t.span() == t.end() - t.begin()
    assert t.span() == 14

if __name__ == "__main__":
    pytest.main([__file__, '-v'])
<commit_msg>add: containsi() tests<commit_after>"""
intervaltree: A mutable, self-balancing interval tree for Python 2 and 3.
Queries may be by point, by range overlap, or by range envelopment.

Test module: IntervalTree, Basic query methods (read-only)

Copyright 2013-2015 Chaim-Leib Halbert

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import absolute_import
from intervaltree import Interval, IntervalTree
import pytest
from test.intervaltrees import trees, sdata
try:
    import cPickle as pickle
except ImportError:
    import pickle

def test_empty_queries():
    t = IntervalTree()
    e = set()

    assert len(t) == 0
    assert t.is_empty()
    assert t[3] == e
    assert t[4:6] == e
    assert t.begin() == 0
    assert t.end() == 0
    assert t[t.begin():t.end()] == e
    assert t.items() == e
    assert set(t) == e
    assert set(t.copy()) == e
    assert t.find_nested() == {}
    assert t.range().is_null()
    assert t.range().length() == 0
    t.verify()

def test_queries():
    t = trees['ivs1']()

    assert sdata(t[4]) == set(['[4,7)'])
    assert sdata(t[4:5]) == set(['[4,7)'])
    assert sdata(t[4:6]) == set(['[4,7)', '[5,9)'])
    assert sdata(t[9]) == set(['[6,10)', '[8,10)', '[8,15)'])
    assert sdata(t[15]) == set()
    assert sdata(t.search(5)) == set(['[4,7)', '[5,9)'])
    assert sdata(t.search(6, 11, strict=True)) == set(['[6,10)', '[8,10)'])

def test_partial_slice_query():
    def assert_chop(t, limit):
        s = set(t)
        assert t[:] == s

        s = set(iv for iv in t if iv.begin < limit)
        assert t[:limit] == s

        s = set(iv for iv in t if iv.end > limit)
        assert t[limit:] == s

    assert_chop(trees['ivs1'](), 7)
    assert_chop(trees['ivs2'](), -3)

def test_tree_bounds():
    def assert_tree_bounds(t):
        begin, end, _ = set(t).pop()
        for iv in t:
            if iv.begin < begin: begin = iv.begin
            if iv.end > end: end = iv.end
        assert t.begin() == begin
        assert t.end() == end

    assert_tree_bounds(trees['ivs1']())
    assert_tree_bounds(trees['ivs2']())

def test_membership():
    t = trees['ivs1']()
    assert Interval(1, 2, '[1,2)') in t
    assert t.containsi(1, 2, '[1,2)')
    assert Interval(1, 3, '[1,3)') not in t
    assert not t.containsi(1, 3, '[1,3)')
    assert t.overlaps(4)
    assert t.overlaps(9)
    assert not t.overlaps(15)
    assert t.overlaps(0, 4)
    assert t.overlaps(1, 2)
    assert t.overlaps(1, 3)
    assert t.overlaps(8, 15)
    assert not t.overlaps(15, 16)
    assert not t.overlaps(-1, 0)
    assert not t.overlaps(2, 4)

def test_overlaps_empty():
    # Empty tree
    t = IntervalTree()
    assert not t.overlaps(-1)
    assert not t.overlaps(0)

    assert not t.overlaps(-1, 1)
    assert not t.overlaps(-1, 0)
    assert not t.overlaps(0, 0)
    assert not t.overlaps(0, 1)
    assert not t.overlaps(1, 0)
    assert not t.overlaps(1, -1)
    assert not t.overlaps(0, -1)

    assert not t.overlaps(Interval(-1, 1))
    assert not t.overlaps(Interval(-1, 0))
    assert not t.overlaps(Interval(0, 0))
    assert not t.overlaps(Interval(0, 1))
    assert not t.overlaps(Interval(1, 0))
    assert not t.overlaps(Interval(1, -1))
    assert not t.overlaps(Interval(0, -1))

def test_overlaps():
    t = trees['ivs1']()
    assert not t.overlaps(-3.2)
    assert t.overlaps(1)
    assert t.overlaps(1.5)
    assert t.overlaps(0, 3)
    assert not t.overlaps(0, 1)
    assert not t.overlaps(2, 4)
    assert not t.overlaps(4, 2)
    assert not t.overlaps(3, 0)

def test_span():
    e = IntervalTree()
    assert e.span() == 0

    t = trees['ivs1']()
    assert t.span() == t.end() - t.begin()
    assert t.span() == 14

if __name__ == "__main__":
    pytest.main([__file__, '-v'])
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This module implements the main Custodian class, which manages a list of jobs
given a set of error handlers, and the abstract base classes for the
ErrorHandlers and Jobs.
"""

from __future__ import division

__author__ = "Shyue Ping Ong, William Davidson Richards"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "May 3, 2013"

import logging
import subprocess
import time
from abc import ABCMeta, abstractmethod, abstractproperty
import json

class Custodian(object):
    """
    The Custodian class is the manager for a list of jobs given a list of
    error handlers. The way it works is as follows:

    1. Let's say you have defined a list of jobs as [job1, job2, job3, ...] and
       you have defined a list of possible error handlers as [err1, err2, ...]
    2. Custodian will run the jobs in the order of job1, job2, ... During each
       job, custodian will monitor for errors using the handlers that have
       is_monitor == True. If an error is detected, corrective measures are
       taken and the particular job is rerun.
    3. At the end of each individual job, Custodian will run through the list
       error handlers that have is_monitor == False. If an error is detected,
       corrective measures are taken and the particular job is rerun.

    .. attribute: max_errors

        Maximum number of errors allowed.

    .. attribute: handlers

        All error handlers (including monitors). All error handlers are used
        to check for errors at the end of a run.

    .. attribute: monitors

        Error handlers that are Monitors, i.e., handlers that monitors a job
        as it is being run.

    .. attribute: polling_time_step

        The length of time in seconds between steps in which a job is
        checked for completion.

    .. attribute: monitor_freq

        The number of polling steps before monitoring occurs. For example,
        if you have a polling_time_step of 10seconds and a monitor_freq of
        30, this means that Custodian uses the monitors to check for errors
        every 30 x 10 = 300 seconds, i.e., 5 minutes.
    """

    def __init__(self, handlers, jobs, max_errors=1, polling_time_step=10,
                 monitor_freq=30, log_file="custodian.json"):
        """
        Args:
            handlers:
                Error handlers. In order of priority of fixing.
            jobs:
                List of Jobs to be run sequentially.
            max_errors:
                Maximum number of errors allowed before exiting.
            polling_time_step:
                The length of time in seconds between steps in which a
                job is checked for completion. Defaults to 10 seconds.
            monitor_freq:
                The number of polling steps before monitoring occurs. For
                example, if you have a polling_time_step of 10seconds and a
                monitor_freq of 30, this means that Custodian uses the
                monitors to check for errors every 30 x 10 = 300 seconds,
                i.e., 5 minutes.
            log_file:
                Log file to log all jobs and corrections to. Defaults
                to custodian.json. Set to None for no logging.
        """
        self.max_errors = max_errors
        self.jobs = jobs
        self.handlers = handlers
        self.monitors = filter(lambda x: x.is_monitor, handlers)
        self.polling_time_step = polling_time_step
        self.monitor_freq = monitor_freq
        self.log_file = log_file

    def run(self):
        """
        Runs the job.

        Returns:
            All errors encountered as a list of list.
            [[error_dicts for job 1], [error_dicts for job 2], ....]
        """
        run_log = []
        total_errors = 0
        unrecoverable = False
        for i, job in enumerate(self.jobs):
            run_log.append({"job": job.to_dict, "corrections": []})
            for attempt in xrange(self.max_errors):
                logging.info(
                    "Starting job no. {} ({}) attempt no. {}. Errors thus far"
                    " = {}.".format(i + 1, job.name, attempt + 1,
                                    total_errors))

                # If this is the start of the job, do the setup.
                if not run_log[-1]["corrections"]:
                    job.setup()

                p = job.run()
                # Check for errors using the error handlers and perform
                # corrections.
                error = False

                # While the job is running, we use the handlers that are
                # monitors to monitor the job.
                if isinstance(p, subprocess.Popen):
                    if self.monitors:
                        n = 0
                        while True:
                            n += 1
                            time.sleep(self.polling_time_step)
                            if p.poll() is not None:
                                break
                            if n % self.monitor_freq == 0:
                                for h in self.monitors:
                                    if h.check():
                                        p.terminate()
                                        total_errors += 1
                                        d = h.correct()
                                        logging.error(str(d))
                                        run_log[-1]["corrections"].append(d)
                                        error = True
                    else:
                        p.wait()

                # Check for errors again, since in some cases non-monitor
                # handlers fix the problems detected by monitors
                # if an error has been found, not all handlers need to run
                if error:
                    remaining_handlers = filter(lambda x: not x.is_monitor,
                                                handlers)
                else:
                    remaining_handlers = self.handlers
                for h in remaining_handlers:
                    if h.check():
                        total_errors += 1
                        d = h.correct()
                        logging.error(str(d))
                        run_log[-1]["corrections"].append(d)
                        error = True

                if self.log_file is not None:
                    #Log the corrections to a json file.
                    with open(self.log_file, "w") as f:
                        logging.info("Logging to {}...".format(self.log_file))
                        json.dump(run_log, f, indent=4)

                # If there are no errors detected, perform postprocessing and
                # exit.
                if not error:
                    job.postprocess()
                    break
                elif run_log[-1]["corrections"][-1]["actions"] is None:
                    #There is an error. Check if it is unrecoverable.
                    logging.info("Unrecoverable error.")
                    unrecoverable = True
                    break

            if unrecoverable:
                break

        if total_errors == self.max_errors:
            logging.info("Max {} errors reached. Exited"
                         .format(self.max_errors))
        else:
            logging.info("Run completed")
        return run_log

class ErrorHandler(object):
    """
    Abstract base class defining the interface for an ErrorHandler.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def check(self):
        """
        This method is called at the end of a job. Returns a boolean value
        indicating if errors are detected.
        """
        pass

    @abstractmethod
    def correct(self):
        """
        This method is called at the end of a job when an error is detected.
        It should perform any corrective measures relating to the detected
        error.

        This method should return a JSON serializable dict that describes
        the errors and actions taken. E.g.
        {"errors": list_of_errors, "actions": list_of_actions_taken}.
        If this is an unfixable error, actions should be set to None.
        """
        pass

    @abstractproperty
    def is_monitor(self):
        """
        This property indicates whether the error handler is a monitor,
        i.e., a handler that monitors a job as it is running. If a
        monitor-type handler notices an error, the job will be sent a
        termination signal, the error is then corrected,
        and then the job is restarted. This is useful for catching errors
        that occur early in the run but do not cause immediate failure.
        """
        return False

    @abstractproperty
    def to_dict(self):
        """
        This method should return a JSON serializable dict describing the
        ErrorHandler, and can be deserialized using the from_dict static
        method.
        """
        pass

    @staticmethod
    def from_dict(d):
        """
        This simply raises a NotImplementedError to force subclasses to
        implement this static method. Abstract static methods are not
        implemented until Python 3+.
        """
        raise NotImplementedError("ErrorHandler objects must implement a "
                                  "from_dict static method.")

class Job(object):
    """
    Abstract base class defining the interface for a Job.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def setup(self):
        """
        This method is run before the start of a job. Allows for some
        pre-processing.
        """
        pass

    @abstractmethod
    def run(self):
        """
        This method perform the actual work for the job. If parallel error
        checking is desired, this must return a Popen process.
        """
        pass

    @abstractmethod
    def postprocess(self):
        """
        This method is called at the end of a job, *after* error detection.
        This allows post-processing, such as cleanup, analysis of results,
        etc.
        """
        pass

    @abstractproperty
    def name(self):
        """
        A nice string name for the job.
        """
        pass

    @abstractproperty
    def to_dict(self):
        """
        This method should return a JSON serializable dict describing the
        Job, and can be deserialized using the from_dict static
        method.
        """
        pass

    @staticmethod
    def from_dict(d):
        """
        This simply raises a NotImplementedError to force subclasses to
        implement this static method. Abstract static methods are not
        implemented until Python 3+.
        """
        raise NotImplementedError("Job objects must implement a from_dict"
                                  "static method.")
<commit_msg>Fix bug in custodian.<commit_after>#!/usr/bin/env python

"""
This module implements the main Custodian class, which manages a list of jobs
given a set of error handlers, and the abstract base classes for the
ErrorHandlers and Jobs.
"""

from __future__ import division

__author__ = "Shyue Ping Ong, William Davidson Richards"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "May 3, 2013"

import logging
import subprocess
import time
from abc import ABCMeta, abstractmethod, abstractproperty
import json

class Custodian(object):
    """
    The Custodian class is the manager for a list of jobs given a list of
    error handlers. The way it works is as follows:

    1. Let's say you have defined a list of jobs as [job1, job2, job3, ...] and
       you have defined a list of possible error handlers as [err1, err2, ...]
    2. Custodian will run the jobs in the order of job1, job2, ... During each
       job, custodian will monitor for errors using the handlers that have
       is_monitor == True. If an error is detected, corrective measures are
       taken and the particular job is rerun.
    3. At the end of each individual job, Custodian will run through the list
       error handlers that have is_monitor == False. If an error is detected,
       corrective measures are taken and the particular job is rerun.

    .. attribute: max_errors

        Maximum number of errors allowed.

    .. attribute: handlers

        All error handlers (including monitors). All error handlers are used
        to check for errors at the end of a run.

    .. attribute: monitors

        Error handlers that are Monitors, i.e., handlers that monitors a job
        as it is being run.

    .. attribute: polling_time_step

        The length of time in seconds between steps in which a job is
        checked for completion.

    .. attribute: monitor_freq

        The number of polling steps before monitoring occurs. For example,
        if you have a polling_time_step of 10seconds and a monitor_freq of
        30, this means that Custodian uses the monitors to check for errors
        every 30 x 10 = 300 seconds, i.e., 5 minutes.
    """

    def __init__(self, handlers, jobs, max_errors=1, polling_time_step=10,
                 monitor_freq=30, log_file="custodian.json"):
        """
        Args:
            handlers:
                Error handlers. In order of priority of fixing.
            jobs:
                List of Jobs to be run sequentially.
            max_errors:
                Maximum number of errors allowed before exiting.
            polling_time_step:
                The length of time in seconds between steps in which a
                job is checked for completion. Defaults to 10 seconds.
            monitor_freq:
                The number of polling steps before monitoring occurs. For
                example, if you have a polling_time_step of 10seconds and a
                monitor_freq of 30, this means that Custodian uses the
                monitors to check for errors every 30 x 10 = 300 seconds,
                i.e., 5 minutes.
            log_file:
                Log file to log all jobs and corrections to. Defaults
                to custodian.json. Set to None for no logging.
        """
        self.max_errors = max_errors
        self.jobs = jobs
        self.handlers = handlers
        self.monitors = filter(lambda x: x.is_monitor, handlers)
        self.polling_time_step = polling_time_step
        self.monitor_freq = monitor_freq
        self.log_file = log_file

    def run(self):
        """
        Runs the job.

        Returns:
            All errors encountered as a list of list.
            [[error_dicts for job 1], [error_dicts for job 2], ....]
        """
        run_log = []
        total_errors = 0
        unrecoverable = False
        for i, job in enumerate(self.jobs):
            run_log.append({"job": job.to_dict, "corrections": []})
            for attempt in xrange(self.max_errors):
                logging.info(
                    "Starting job no. {} ({}) attempt no. {}. Errors thus far"
                    " = {}.".format(i + 1, job.name, attempt + 1,
                                    total_errors))

                # If this is the start of the job, do the setup.
                if not run_log[-1]["corrections"]:
                    job.setup()

                p = job.run()
                # Check for errors using the error handlers and perform
                # corrections.
                error = False

                # While the job is running, we use the handlers that are
                # monitors to monitor the job.
                if isinstance(p, subprocess.Popen):
                    if self.monitors:
                        n = 0
                        while True:
                            n += 1
                            time.sleep(self.polling_time_step)
                            if p.poll() is not None:
                                break
                            if n % self.monitor_freq == 0:
                                for h in self.monitors:
                                    if h.check():
                                        p.terminate()
                                        total_errors += 1
                                        d = h.correct()
                                        logging.error(str(d))
                                        run_log[-1]["corrections"].append(d)
                                        error = True
                    else:
                        p.wait()

                # Check for errors again, since in some cases non-monitor
                # handlers fix the problems detected by monitors
                # if an error has been found, not all handlers need to run
                if error:
                    remaining_handlers = filter(lambda x: not x.is_monitor,
                                                self.handlers)
                else:
                    remaining_handlers = self.handlers
                for h in remaining_handlers:
                    if h.check():
                        total_errors += 1
                        d = h.correct()
                        logging.error(str(d))
                        run_log[-1]["corrections"].append(d)
                        error = True

                if self.log_file is not None:
                    #Log the corrections to a json file.
                    with open(self.log_file, "w") as f:
                        logging.info("Logging to {}...".format(self.log_file))
                        json.dump(run_log, f, indent=4)

                # If there are no errors detected, perform postprocessing and
                # exit.
                if not error:
                    job.postprocess()
                    break
                elif run_log[-1]["corrections"][-1]["actions"] is None:
                    #There is an error. Check if it is unrecoverable.
                    logging.info("Unrecoverable error.")
                    unrecoverable = True
                    break

            if unrecoverable:
                break

        if total_errors == self.max_errors:
            logging.info("Max {} errors reached. Exited"
                         .format(self.max_errors))
        else:
            logging.info("Run completed")
        return run_log

class ErrorHandler(object):
    """
    Abstract base class defining the interface for an ErrorHandler.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def check(self):
        """
        This method is called at the end of a job. Returns a boolean value
        indicating if errors are detected.
        """
        pass

    @abstractmethod
    def correct(self):
        """
        This method is called at the end of a job when an error is detected.
        It should perform any corrective measures relating to the detected
        error.

        This method should return a JSON serializable dict that describes
        the errors and actions taken. E.g.
        {"errors": list_of_errors, "actions": list_of_actions_taken}.
        If this is an unfixable error, actions should be set to None.
        """
        pass

    @abstractproperty
    def is_monitor(self):
        """
        This property indicates whether the error handler is a monitor,
        i.e., a handler that monitors a job as it is running. If a
        monitor-type handler notices an error, the job will be sent a
        termination signal, the error is then corrected,
        and then the job is restarted. This is useful for catching errors
        that occur early in the run but do not cause immediate failure.
        """
        return False

    @abstractproperty
    def to_dict(self):
        """
        This method should return a JSON serializable dict describing the
        ErrorHandler, and can be deserialized using the from_dict static
        method.
        """
        pass

    @staticmethod
    def from_dict(d):
        """
        This simply raises a NotImplementedError to force subclasses to
        implement this static method. Abstract static methods are not
        implemented until Python 3+.
        """
        raise NotImplementedError("ErrorHandler objects must implement a "
                                  "from_dict static method.")

class Job(object):
    """
    Abstract base class defining the interface for a Job.
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def setup(self):
        """
        This method is run before the start of a job. Allows for some
        pre-processing.
        """
        pass

    @abstractmethod
    def run(self):
        """
        This method perform the actual work for the job. If parallel error
        checking is desired, this must return a Popen process.
        """
        pass

    @abstractmethod
    def postprocess(self):
        """
        This method is called at the end of a job, *after* error detection.
        This allows post-processing, such as cleanup, analysis of results,
        etc.
        """
        pass

    @abstractproperty
    def name(self):
        """
        A nice string name for the job.
        """
        pass

    @abstractproperty
    def to_dict(self):
        """
        This method should return a JSON serializable dict describing the
        Job, and can be deserialized using the from_dict static
        method.
        """
        pass

    @staticmethod
    def from_dict(d):
        """
        This simply raises a NotImplementedError to force subclasses to
        implement this static method. Abstract static methods are not
        implemented until Python 3+.
        """
        raise NotImplementedError("Job objects must implement a from_dict"
                                  "static method.")
<|endoftext|>