<commit_before>import os
import sys
import glob
import time
import urllib2
import tempfile
from multiprocessing import Process

if sys.version_info[1] < 7:
    try:
        import unittest2 as unittest
    except ImportError:
        print 'Test Suite requires Python 2.7 or unittest2'
        sys.exit(1)
else:
    import unittest

import flask
import scrapelib

app = flask.Flask(__name__)
app.config.shaky_fail = False

@app.route('/')
def index():
    resp = app.make_response("Hello world!")
    return resp

@app.route('/ua')
def ua():
    resp = app.make_response(flask.request.headers['user-agent'])
    resp.headers['cache-control'] = 'no-cache'
    return resp

@app.route('/p/s.html')
def secret():
    return "secret"

@app.route('/redirect')
def redirect():
    return flask.redirect(flask.url_for('index'))

@app.route('/500')
def fivehundred():
    flask.abort(500)

@app.route('/robots.txt')
def robots():
    return """
    User-agent: *
    Disallow: /p/
    Allow: /
    """

@app.route('/shaky')
def shaky():
    # toggle failure state each time
    app.config.shaky_fail = not app.config.shaky_fail

    if app.config.shaky_fail:
        flask.abort(500)
    else:
        return "shaky success!"

def run_server():
    class NullFile(object):
        def write(self, s):
            pass

    sys.stdout = NullFile()
    sys.stderr = NullFile()

    app.run()

class HeaderTest(unittest.TestCase):
    def test_keys(self):
        h = scrapelib.Headers()
        h['A'] = '1'

        self.assertEqual(h['A'], '1')

        self.assertEqual(h.getallmatchingheaders('A'), ["A: 1"])
        self.assertEqual(h.getallmatchingheaders('b'), [])
        self.assertEqual(h.getheaders('A'), ['1'])
        self.assertEqual(h.getheaders('b'), [])

        # should be case-insensitive
        self.assertEqual(h['a'], '1')
        h['a'] = '2'
        self.assertEqual(h['A'], '2')

        self.assert_('a' in h)
        self.assert_('A' in h)

        del h['A']
        self.assert_('a' not in h)
        self.assert_('A' not in h)

    def test_equality(self):
        h1 = scrapelib.Headers()
        h1['Accept-Encoding'] = '*'

        h2 = scrapelib.Headers()
        self.assertNotEqual(h1, h2)

        h2['accept-encoding'] = 'not'
        self.assertNotEqual(h1, h2)

        h2['accept-encoding'] = '*'
        self.assertEqual(h1, h2)

class ScraperTest(unittest.TestCase):
    def setUp(self):
        self.cache_dir = tempfile.mkdtemp()
        self.error_dir = tempfile.mkdtemp()
        self.s = scrapelib.Scraper(requests_per_minute=0,
                                   error_dir=self.error_dir,
                                   cache_dir=self.cache_dir,
                                   use_cache_first=True)

    def tearDown(self):
        for path in glob.iglob(os.path.join(self.cache_dir, "*")):
            os.remove(path)
        os.rmdir(self.cache_dir)
        for path in glob.iglob(os.path.join(self.error_dir, "*")):
            os.remove(path)
        os.rmdir(self.error_dir)

    def test_get(self):
        self.assertEqual('Hello world!',
                         self.s.urlopen("http://localhost:5000/"))

    def test_request_throttling(self):
        requests = 0
        s = scrapelib.Scraper(requests_per_minute=30)

        begin = time.time()
        while time.time() <= (begin + 1):
            s.urlopen("http://localhost:5000/")
            requests += 1
        self.assert_(requests <= 2)

        s.requests_per_minute = 500
        requests = 0
        begin = time.time()
        while time.time() <= (begin + 1):
            s.urlopen("http://localhost:5000/")
            requests += 1
        self.assert_(requests > 5)

    def test_user_agent(self):
        resp = self.s.urlopen("http://localhost:5000/ua")
        self.assertEqual(resp, scrapelib._user_agent)

        self.s.user_agent = 'a different agent'
        resp = self.s.urlopen("http://localhost:5000/ua")
        self.assertEqual(resp, 'a different agent')

    def test_default_to_http(self):
        self.assertEqual('Hello world!',
                         self.s.urlopen("localhost:5000/"))

    def test_follow_robots(self):
        self.assertRaises(scrapelib.RobotExclusionError, self.s.urlopen,
                          "http://localhost:5000/p/s.html")
        self.assertRaises(scrapelib.RobotExclusionError, self.s.urlopen,
                          "http://localhost:5000/p/a/t/h/")

        self.s.follow_robots = False
        self.assertEqual("secret",
                         self.s.urlopen("http://localhost:5000/p/s.html"))
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/p/a/t/h/")

    def test_error_context(self):
        def raises():
            with self.s.urlopen("http://localhost:5000/"):
                raise Exception('test')

        self.assertRaises(Exception, raises)
        self.assertTrue(os.path.isfile(os.path.join(
            self.error_dir, "http:,,localhost:5000,")))

    def test_404(self):
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/does/not/exist")

        self.s.raise_errors = False
        resp = self.s.urlopen("http://localhost:5000/does/not/exist")
        self.assertEqual(404, resp.response.code)

    def test_500(self):
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/500")

        self.s.raise_errors = False
        resp = self.s.urlopen("http://localhost:5000/500")
        self.assertEqual(resp.response.code, 500)

    def test_follow_redirect(self):
        resp = self.s.urlopen("http://localhost:5000/redirect")
        self.assertEqual("http://localhost:5000/", resp.response.url)
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.requested_url)
        self.assertEqual(200, resp.response.code)

        self.s.follow_redirects = False
        resp = self.s.urlopen("http://localhost:5000/redirect")
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.url)
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.requested_url)
        self.assertEqual(302, resp.response.code)

    def test_caching(self):
        resp = self.s.urlopen("http://localhost:5000/")
        self.assertFalse(resp.response.fromcache)
        resp = self.s.urlopen("http://localhost:5000/")
        self.assert_(resp.response.fromcache)

        self.s.use_cache_first = False
        resp = self.s.urlopen("http://localhost:5000/")
        self.assertFalse(resp.response.fromcache)

    def test_urlretrieve(self):
        fname, resp = self.s.urlretrieve("http://localhost:5000/")
        with open(fname) as f:
            self.assertEqual(f.read(), "Hello world!")
            self.assertEqual(200, resp.code)
        os.remove(fname)

        (fh, set_fname) = tempfile.mkstemp()
        fname, resp = self.s.urlretrieve("http://localhost:5000/",
                                         set_fname)
        self.assertEqual(fname, set_fname)
        with open(set_fname) as f:
            self.assertEqual(f.read(), "Hello world!")
            self.assertEqual(200, resp.code)
        os.remove(set_fname)

    def test_retry_httplib2(self):
        s = scrapelib.Scraper(retry_attempts=3, retry_wait_seconds=1.5)

        # one failure, then success
        resp, content = s._do_request('http://localhost:5000/shaky',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(content, 'shaky success!')

        # TODO: on this and the other test it'd be nice to have a way to test
        # it tries 3 times for 500 and once for 404

        # 500 always
        resp, content = s._do_request('http://localhost:5000/500',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(resp.status, 500)

        # 404
        resp, content = s._do_request('http://localhost:5000/404',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(resp.status, 404)

    def test_retry_urllib2(self):
        s = scrapelib.Scraper(retry_attempts=3, retry_wait_seconds=1)

        # without httplib2
        resp = s._do_request('http://localhost:5000/shaky',
                             'GET', None, {}, use_httplib2=False)
        self.assertEqual(resp.read(), 'shaky success!')

        # 500 always
        self.assertRaises(urllib2.URLError, s._do_request,
                          'http://localhost:5000/500',
                          'GET', None, {}, use_httplib2=False)

        # 404
        self.assertRaises(urllib2.URLError, s._do_request,
                          'http://localhost:5000/404',
                          'GET', None, {}, use_httplib2=False)

    def test_disable_compression(self):
        s = scrapelib.Scraper(disable_compression=True)

        headers = s._make_headers("http://google.com")
        self.assertEqual(headers['accept-encoding'], 'text/*')

        # A supplied Accept-Encoding headers overrides the
        # disable_compression option
        s.headers['accept-encoding'] = '*'
        headers = s._make_headers('http://google.com')
        self.assertEqual(headers['accept-encoding'], '*')

if __name__ == '__main__':
    process = Process(target=run_server)
    process.start()
    time.sleep(0.1)
    unittest.main(exit=False)
    process.terminate()
<commit_msg>test callable headers param<commit_after>import os
import sys
import glob
import time
import urllib2
import tempfile
from multiprocessing import Process

if sys.version_info[1] < 7:
    try:
        import unittest2 as unittest
    except ImportError:
        print 'Test Suite requires Python 2.7 or unittest2'
        sys.exit(1)
else:
    import unittest

import flask
import scrapelib

app = flask.Flask(__name__)
app.config.shaky_fail = False

@app.route('/')
def index():
    resp = app.make_response("Hello world!")
    return resp

@app.route('/ua')
def ua():
    resp = app.make_response(flask.request.headers['user-agent'])
    resp.headers['cache-control'] = 'no-cache'
    return resp

@app.route('/p/s.html')
def secret():
    return "secret"

@app.route('/redirect')
def redirect():
    return flask.redirect(flask.url_for('index'))

@app.route('/500')
def fivehundred():
    flask.abort(500)

@app.route('/robots.txt')
def robots():
    return """
    User-agent: *
    Disallow: /p/
    Allow: /
    """

@app.route('/shaky')
def shaky():
    # toggle failure state each time
    app.config.shaky_fail = not app.config.shaky_fail

    if app.config.shaky_fail:
        flask.abort(500)
    else:
        return "shaky success!"

def run_server():
    class NullFile(object):
        def write(self, s):
            pass

    sys.stdout = NullFile()
    sys.stderr = NullFile()

    app.run()

class HeaderTest(unittest.TestCase):
    def test_keys(self):
        h = scrapelib.Headers()
        h['A'] = '1'

        self.assertEqual(h['A'], '1')

        self.assertEqual(h.getallmatchingheaders('A'), ["A: 1"])
        self.assertEqual(h.getallmatchingheaders('b'), [])
        self.assertEqual(h.getheaders('A'), ['1'])
        self.assertEqual(h.getheaders('b'), [])

        # should be case-insensitive
        self.assertEqual(h['a'], '1')
        h['a'] = '2'
        self.assertEqual(h['A'], '2')

        self.assert_('a' in h)
        self.assert_('A' in h)

        del h['A']
        self.assert_('a' not in h)
        self.assert_('A' not in h)

    def test_equality(self):
        h1 = scrapelib.Headers()
        h1['Accept-Encoding'] = '*'

        h2 = scrapelib.Headers()
        self.assertNotEqual(h1, h2)

        h2['accept-encoding'] = 'not'
        self.assertNotEqual(h1, h2)

        h2['accept-encoding'] = '*'
        self.assertEqual(h1, h2)

class ScraperTest(unittest.TestCase):
    def setUp(self):
        self.cache_dir = tempfile.mkdtemp()
        self.error_dir = tempfile.mkdtemp()
        self.s = scrapelib.Scraper(requests_per_minute=0,
                                   error_dir=self.error_dir,
                                   cache_dir=self.cache_dir,
                                   use_cache_first=True)

    def tearDown(self):
        for path in glob.iglob(os.path.join(self.cache_dir, "*")):
            os.remove(path)
        os.rmdir(self.cache_dir)
        for path in glob.iglob(os.path.join(self.error_dir, "*")):
            os.remove(path)
        os.rmdir(self.error_dir)

    def test_get(self):
        self.assertEqual('Hello world!',
                         self.s.urlopen("http://localhost:5000/"))

    def test_request_throttling(self):
        requests = 0
        s = scrapelib.Scraper(requests_per_minute=30)

        begin = time.time()
        while time.time() <= (begin + 1):
            s.urlopen("http://localhost:5000/")
            requests += 1
        self.assert_(requests <= 2)

        s.requests_per_minute = 500
        requests = 0
        begin = time.time()
        while time.time() <= (begin + 1):
            s.urlopen("http://localhost:5000/")
            requests += 1
        self.assert_(requests > 5)

    def test_user_agent(self):
        resp = self.s.urlopen("http://localhost:5000/ua")
        self.assertEqual(resp, scrapelib._user_agent)

        self.s.user_agent = 'a different agent'
        resp = self.s.urlopen("http://localhost:5000/ua")
        self.assertEqual(resp, 'a different agent')

    def test_default_to_http(self):
        self.assertEqual('Hello world!',
                         self.s.urlopen("localhost:5000/"))

    def test_follow_robots(self):
        self.assertRaises(scrapelib.RobotExclusionError, self.s.urlopen,
                          "http://localhost:5000/p/s.html")
        self.assertRaises(scrapelib.RobotExclusionError, self.s.urlopen,
                          "http://localhost:5000/p/a/t/h/")

        self.s.follow_robots = False
        self.assertEqual("secret",
                         self.s.urlopen("http://localhost:5000/p/s.html"))
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/p/a/t/h/")

    def test_error_context(self):
        def raises():
            with self.s.urlopen("http://localhost:5000/"):
                raise Exception('test')

        self.assertRaises(Exception, raises)
        self.assertTrue(os.path.isfile(os.path.join(
            self.error_dir, "http:,,localhost:5000,")))

    def test_404(self):
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/does/not/exist")

        self.s.raise_errors = False
        resp = self.s.urlopen("http://localhost:5000/does/not/exist")
        self.assertEqual(404, resp.response.code)

    def test_500(self):
        self.assertRaises(scrapelib.HTTPError, self.s.urlopen,
                          "http://localhost:5000/500")

        self.s.raise_errors = False
        resp = self.s.urlopen("http://localhost:5000/500")
        self.assertEqual(resp.response.code, 500)

    def test_follow_redirect(self):
        resp = self.s.urlopen("http://localhost:5000/redirect")
        self.assertEqual("http://localhost:5000/", resp.response.url)
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.requested_url)
        self.assertEqual(200, resp.response.code)

        self.s.follow_redirects = False
        resp = self.s.urlopen("http://localhost:5000/redirect")
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.url)
        self.assertEqual("http://localhost:5000/redirect",
                         resp.response.requested_url)
        self.assertEqual(302, resp.response.code)

    def test_caching(self):
        resp = self.s.urlopen("http://localhost:5000/")
        self.assertFalse(resp.response.fromcache)
        resp = self.s.urlopen("http://localhost:5000/")
        self.assert_(resp.response.fromcache)

        self.s.use_cache_first = False
        resp = self.s.urlopen("http://localhost:5000/")
        self.assertFalse(resp.response.fromcache)

    def test_urlretrieve(self):
        fname, resp = self.s.urlretrieve("http://localhost:5000/")
        with open(fname) as f:
            self.assertEqual(f.read(), "Hello world!")
            self.assertEqual(200, resp.code)
        os.remove(fname)

        (fh, set_fname) = tempfile.mkstemp()
        fname, resp = self.s.urlretrieve("http://localhost:5000/",
                                         set_fname)
        self.assertEqual(fname, set_fname)
        with open(set_fname) as f:
            self.assertEqual(f.read(), "Hello world!")
            self.assertEqual(200, resp.code)
        os.remove(set_fname)

    def test_retry_httplib2(self):
        s = scrapelib.Scraper(retry_attempts=3, retry_wait_seconds=1.5)

        # one failure, then success
        resp, content = s._do_request('http://localhost:5000/shaky',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(content, 'shaky success!')

        # TODO: on this and the other test it'd be nice to have a way to test
        # it tries 3 times for 500 and once for 404

        # 500 always
        resp, content = s._do_request('http://localhost:5000/500',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(resp.status, 500)

        # 404
        resp, content = s._do_request('http://localhost:5000/404',
                                      'GET', None, {}, use_httplib2=True)
        self.assertEqual(resp.status, 404)

    def test_retry_urllib2(self):
        s = scrapelib.Scraper(retry_attempts=3, retry_wait_seconds=1)

        # without httplib2
        resp = s._do_request('http://localhost:5000/shaky',
                             'GET', None, {}, use_httplib2=False)
        self.assertEqual(resp.read(), 'shaky success!')

        # 500 always
        self.assertRaises(urllib2.URLError, s._do_request,
                          'http://localhost:5000/500',
                          'GET', None, {}, use_httplib2=False)

        # 404
        self.assertRaises(urllib2.URLError, s._do_request,
                          'http://localhost:5000/404',
                          'GET', None, {}, use_httplib2=False)

    def test_disable_compression(self):
        s = scrapelib.Scraper(disable_compression=True)

        headers = s._make_headers("http://google.com")
        self.assertEqual(headers['accept-encoding'], 'text/*')

        # A supplied Accept-Encoding headers overrides the
        # disable_compression option
        s.headers['accept-encoding'] = '*'
        headers = s._make_headers('http://google.com')
        self.assertEqual(headers['accept-encoding'], '*')

    def test_callable_headers(self):
        s = scrapelib.Scraper(headers=lambda url: {'URL': url})

        headers = s._make_headers('http://google.com')
        self.assertEqual(headers['url'], 'http://google.com')

        # Make sure it gets called freshly each time
        headers = s._make_headers('example.com')
        self.assertEqual(headers['url'], 'example.com')

if __name__ == '__main__':
    process = Process(target=run_server)
    process.start()
    time.sleep(0.1)
    unittest.main(exit=False)
    process.terminate()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest

from address_normalizer import expand_street_address

street_test_cases = (
    # input, expected output
    ('MAPLE ST.', 'maple street'),
    ('ST ISIDORE DR', 'saint isidore drive'),
    ('ST. Sebastian ST', 'saint sebastian street'),
    ("St John's St.", 'saint johns street'),
    ('MORNINGTON CR', 'mornington crescent'),
    ('Cércle rouge', 'cercle rouge'),
    ('Third Street', '3rd street'),
)

dupe_test_cases = (
    # addresses should share at least one expansion in common
    ('30 West Twenty-sixth Street Floor Number 7',
     '30 W 26th St Fl #7'),
    ('123 Dr. Martin Luther King Jr. Dr.',
     '123 Doctor Martin Luther King Junior Drive')
)

class TestNormalization(unittest.TestCase):
    def test_street_name(self):
        for original, expected in street_test_cases:
            self.assertTrue(expected in expand_street_address(original))

    def test_dupes(self):
        for a1, a2 in dupe_test_cases:
            self.assertTrue(expand_street_address(a1) &
                            expand_street_address(a2))

if __name__ == '__main__':
    unittest.main()
<commit_msg>[tests] tests of encoding/decoding, diacritics<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import unittest

from address_normalizer import expand_street_address
from address_normalizer.text.encoding import safe_encode, safe_decode

street_test_cases = (
    # input, expected output
    ('MAPLE ST.', 'maple street'),
    ('ST ISIDORE DR', 'saint isidore drive'),
    ('ST. Sebastian ST', 'saint sebastian street'),
    ("St John's St.", 'saint johns street'),
    ('MORNINGTON CR', 'mornington crescent'),
    ('Cércle rouge', 'cercle rouge'),
    ('Third Street', '3rd street'),
)

dupe_test_cases = (
    # addresses should share at least one expansion in common
    ('30 West Twenty-sixth Street Floor Number 7',
     '30 W 26th St Fl #7'),
    ('123 Dr. Martin Luther King Jr. Dr.',
     '123 Doctor Martin Luther King Junior Drive')
)

class TestNormalization(unittest.TestCase):
    def test_street_name(self):
        for original, expected in street_test_cases:
            self.assertTrue(expected in expand_street_address(original))

    def test_dupes(self):
        for a1, a2 in dupe_test_cases:
            self.assertTrue(expand_street_address(a1) &
                            expand_street_address(a2))

    def test_diacritics(self):
        orig = 'cércle rouge'
        self.assertTrue(orig in expand_street_address(orig, remove_accents=False))
        self.assertTrue('cercle rouge' in expand_street_address(orig, remove_accents=True))

    def test_decoding(self):
        self.assertEqual(safe_decode(b'é'), 'é')
        self.assertEqual(safe_decode('é'), 'é')
        self.assertEqual(safe_decode(1), '1')

    def test_encoding(self):
        self.assertEqual(safe_encode('é'), b'é')
        self.assertEqual(safe_encode(b'é'), b'é')
        self.assertEqual(safe_encode(1), b'1')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
import unittest
from table_fu import TableFu
from table_fu.formatting import Formatter

class TableTest(unittest.TestCase):

    def setUp(self):
        self.csv_file = open('tests/test.csv')
        self.table = [['Author', 'Best Book', 'Number of Pages', 'Style'],
            ['Samuel Beckett', 'Malone Muert', '120', 'Modernism'],
            ['James Joyce', 'Ulysses', '644', 'Modernism'],
            ['Nicholson Baker', 'Mezannine', '150', 'Minimalism'],
            ['Vladimir Sorokin', 'The Queue', '263', 'Satire']]

    def tearDown(self):
        self.csv_file.close()

class BigTableTest(TableTest):

    def testTable(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.assertEqual(t.table, self.table)

    def testTableFromList(self):
        t = TableFu(self.table)
        self.table.pop(0)
        self.assertEqual(t.table, self.table)

    def testTableTwoWays(self):
        t1 = TableFu(self.csv_file)
        t2 = TableFu(self.table)
        self.assertEqual(t1.table, t2.table)

class ColumnTest(TableTest):

    def testgetColumns(self):
        t = TableFu(self.csv_file)
        self.assertEqual(t.columns, self.table[0])

    def testsetColumns(self):
        t = TableFu(self.csv_file)
        columns = ['Style', 'Author']
        t.columns = columns
        self.assertEqual(t.columns, columns)

class RowTest(TableTest):

    def testCountRows(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.assertEqual(len(t.rows), len(self.table))

    def testGetRow(self):
        t = TableFu(self.csv_file)
        self.assertEqual(t[1], t.rows[1])

    def testCheckRow(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        for i, row in enumerate(self.table):
            self.assertEqual(
                t[i].cells,
                self.table[i]
            )

class RowColumnTest(TableTest):

    def testLimitColumns(self):
        t = TableFu(self.csv_file)
        t.columns = ['Author', 'Style']
        self.assertEqual(
            str(t[0]),
            'Samuel Beckett, Modernism'
            )

class DatumTest(TableTest):

    def testGetDatum(self):
        t = TableFu(self.csv_file)
        for row in t.rows:
            for c in self.table[0]:
                self.assertEqual(c, row[c].column_name)

    def testDatumValues(self):
        t = TableFu(self.csv_file)
        columns = self.table.pop(0)
        for i, row in enumerate(t.rows):
            for index, column in enumerate(columns):
                self.assertEqual(
                    self.table[i][index],
                    str(row[column])
                )

class ErrorTest(TableTest):

    def testBadKey(self):
        t = TableFu(self.csv_file)
        for row in t.rows:
            self.assertRaises(
                KeyError,
                row.__getitem__,
                'not-a-key'
            )

    def testBadTotal(self):
        t = TableFu(self.csv_file)
        self.assertRaises(ValueError, t.total, 'Author')

class SortTest(TableTest):

    def testSort(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.table.sort(key=lambda row: row[0])
        t.sort('Author')
        self.assertEqual(
            t[0].cells,
            self.table[0]
        )

class ValuesTest(TableTest):

    def testValues(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        authors = [row[0] for row in self.table]
        self.assertEqual(authors, t.values('Author'))

    def testTotals(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        pages = sum([float(row[2]) for row in self.table])
        self.assertEqual(pages, t.total('Number of Pages'))

class FacetTest(TableTest):

    def testFacet(self):
        t = TableFu(self.csv_file)
        tables = t.facet_by('Style')
        style_row = self.table[4]
        self.assertEqual(
            style_row,
            tables[2][0].cells
        )

class OptionsTest(TableTest):

    def testSortOption(self):
        t = TableFu(self.csv_file, sorted_by={"Author": {'reverse': True}})
        self.table.pop(0)
        self.table.sort(key=lambda row: row[0], reverse=True)
        self.assertEqual(t[0].cells, self.table[0])

class DatumFormatTest(TableTest):

    def setUp(self):
        self.csv_file = open('tests/sites.csv')

    def testCellFormat(self):
        t = TableFu(self.csv_file)
        t.formatting = {'Name': {
            'filter': 'link',
            'args': ['URL']
            }
        }

        self.assertEqual(
            str(t[0]['Name']),
            'ChrisAmico.com'
        )

class FormatTest(unittest.TestCase):

    def setUp(self):
        self.format = Formatter()

class RegisterTest(FormatTest):

    def testRegister(self):

        def test(value, *args):
            args = list(args)
            args.insert(0, value)
            return args

        self.format.register(test)
        self.assertEqual(test, self.format._filters['test'])

    def testIntComma(self):

        self.assertEqual(
            self.format(1200, 'intcomma'),
            '1,200'
        )

if __name__ == '__main__':
    unittest.main()
<commit_msg>More tests. Moving toward outputting HTML<commit_after>#! /usr/bin/env python
import unittest
from table_fu import TableFu
from table_fu.formatting import Formatter

class TableTest(unittest.TestCase):

    def setUp(self):
        self.csv_file = open('tests/test.csv')
        self.table = [['Author', 'Best Book', 'Number of Pages', 'Style'],
            ['Samuel Beckett', 'Malone Muert', '120', 'Modernism'],
            ['James Joyce', 'Ulysses', '644', 'Modernism'],
            ['Nicholson Baker', 'Mezannine', '150', 'Minimalism'],
            ['Vladimir Sorokin', 'The Queue', '263', 'Satire']]

    def tearDown(self):
        self.csv_file.close()

class BigTableTest(TableTest):

    def testTable(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.assertEqual(t.table, self.table)

    def testTableFromList(self):
        t = TableFu(self.table)
        self.table.pop(0)
        self.assertEqual(t.table, self.table)

    def testTableTwoWays(self):
        t1 = TableFu(self.csv_file)
        t2 = TableFu(self.table)
        self.assertEqual(t1.table, t2.table)

class ColumnTest(TableTest):

    def testgetColumns(self):
        t = TableFu(self.csv_file)
        self.assertEqual(t.columns, self.table[0])

    def testsetColumns(self):
        t = TableFu(self.csv_file)
        columns = ['Style', 'Author']
        t.columns = columns
        self.assertEqual(t.columns, columns)

class RowTest(TableTest):

    def testCountRows(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.assertEqual(len(t.rows), len(self.table))

    def testGetRow(self):
        t = TableFu(self.csv_file)
        self.assertEqual(t[1], t.rows[1])

    def testCheckRow(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        for i, row in enumerate(self.table):
            self.assertEqual(
                t[i].cells,
                self.table[i]
            )

class RowColumnTest(TableTest):

    def testLimitColumns(self):
        t = TableFu(self.csv_file)
        t.columns = ['Author', 'Style']
        self.assertEqual(
            str(t[0]),
            'Samuel Beckett, Modernism'
            )

class DatumTest(TableTest):

    def testGetDatum(self):
        t = TableFu(self.csv_file)
        for row in t.rows:
            for c in self.table[0]:
                self.assertEqual(c, row[c].column_name)

    def testDatumValues(self):
        t = TableFu(self.csv_file)
        columns = self.table.pop(0)
        for i, row in enumerate(t.rows):
            for index, column in enumerate(columns):
                self.assertEqual(
                    self.table[i][index],
                    str(row[column])
                )

class ErrorTest(TableTest):

    def testBadKey(self):
        t = TableFu(self.csv_file)
        for row in t.rows:
            self.assertRaises(
                KeyError,
                row.__getitem__,
                'not-a-key'
            )

    def testBadTotal(self):
        t = TableFu(self.csv_file)
        self.assertRaises(ValueError, t.total, 'Author')

class SortTest(TableTest):

    def testSort(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        self.table.sort(key=lambda row: row[0])
        t.sort('Author')
        self.assertEqual(
            t[0].cells,
            self.table[0]
        )

class ValuesTest(TableTest):

    def testValues(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        authors = [row[0] for row in self.table]
        self.assertEqual(authors, t.values('Author'))

    def testTotals(self):
        t = TableFu(self.csv_file)
        self.table.pop(0)
        pages = sum([float(row[2]) for row in self.table])
        self.assertEqual(pages, t.total('Number of Pages'))

class FacetTest(TableTest):

    def testFacet(self):
        t = TableFu(self.csv_file)
        tables = t.facet_by('Style')
        style_row = self.table[4]
        self.assertEqual(
            style_row,
            tables[2][0].cells
        )

class OptionsTest(TableTest):

    def testSortOption(self):
        t = TableFu(self.csv_file, sorted_by={"Author": {'reverse': True}})
        self.table.pop(0)
        self.table.sort(key=lambda row: row[0], reverse=True)
        self.assertEqual(t[0].cells, self.table[0])

class DatumFormatTest(TableTest):

    def setUp(self):
        self.csv_file = open('tests/sites.csv')

    def testCellFormat(self):
        t = TableFu(self.csv_file)
        t.formatting = {'Name': {
            'filter': 'link',
            'args': ['URL']
            }
        }

        self.assertEqual(
            str(t[0]['Name']),
            'ChrisAmico.com'
        )

class HTMLTest(TableTest):

    def testDatumTD(self):

        t = TableFu(self.csv_file)
        beckett = t[0]['Author']
        self.assertEqual(
            beckett.as_td(),
            'Samuel Beckett	'
        )

    def testRowTR(self):

        t = TableFu(self.csv_file)
        row = t[0]
        self.assertEqual(
            row.as_tr(),
            '
Samuel Beckett	Malone Muert	120	Modernism
'
        )

class FormatTest(unittest.TestCase):

    def setUp(self):
        self.format = Formatter()

class RegisterTest(FormatTest):

    def testRegister(self):

        def test(value, *args):
            args = list(args)
            args.insert(0, value)
            return args

        self.format.register(test)
        self.assertEqual(test, self.format._filters['test'])

    def testIntComma(self):

        self.assertEqual(
            self.format(1200, 'intcomma'),
            '1,200'
        )

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
import sys
import utils
import cyvcf2
import getpass
import binascii
import argparse
import vcf_parsing
import configuration

from cyvcf2 import VCF
from bson import ObjectId
from datetime import datetime
from pymongo import MongoClient
from collections import defaultdict

def get_population_freqs(variant):
    freqs = {'esp_ea': variant.INFO.get('aaf_esp_ea') or -1,
             'esp_aa': variant.INFO.get('aaf_esp_aa') or -1,
             'esp_all': variant.INFO.get('aaf_esp_all') or -1,
             '1kg_amr': variant.INFO.get('aaf_1kg_amr') or -1,
             '1kg_eas': variant.INFO.get('aaf_1kg_eas') or -1,
             '1kg_sas': variant.INFO.get('aaf_1kg_sas') or -1,
             '1kg_afr': variant.INFO.get('aaf_1kg_afr') or -1,
             '1kg_eur': variant.INFO.get('aaf_1kg_eur') or -1,
             '1kg_all': variant.INFO.get('aaf_1kg_all') or -1,
             'exac_all': variant.INFO.get('aaf_exac_all') or -1,
             'adj_exac_all': variant.INFO.get('aaf_adj_exac_all') or -1,
             'adj_exac_afr': variant.INFO.get('aaf_adj_exac_afr') or -1,
             'adj_exac_amr': variant.INFO.get('aaf_adj_exac_amr') or -1,
             'adj_exac_eas': variant.INFO.get('aaf_adj_exac_eas') or -1,
             'adj_exac_fin': variant.INFO.get('aaf_adj_exac_fin') or -1,
             'adj_exac_nfe': variant.INFO.get('aaf_adj_exac_nfe') or -1,
             'adj_exac_oth': variant.INFO.get('aaf_adj_exac_oth') or -1,
             'adj_exac_sas': variant.INFO.get('aaf_adj_exac_sas') or -1}

    return freqs

def get_amplicon_data(variant):
    data = {'amplicon': variant.INFO.get('amplicon_target') or "None",
            'panel_amplicon': variant.INFO.get('panel_target') or "None",
            'intersect': variant.INFO.get('amplicon_intersect') or "None"}

    return data

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    # parser.add_argument('-a', '--address', help="IP Address for Cassandra connection", default='127.0.0.1')
    # parser.add_argument('-u', '--username', help='Cassandra username for login', default=None)

    args = parser.parse_args()
    args.logLevel = "INFO"

    date = datetime.today()

    client = MongoClient()
    db = client['ddb']

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    parse_functions = {'mutect': vcf_parsing.parse_mutect_vcf_record,
                       'freebayes': vcf_parsing.parse_freebayes_vcf_record,
                       'vardict': vcf_parsing.parse_vardict_vcf_record,
                       'scalpel': vcf_parsing.parse_scalpel_vcf_record,
                       'platypus': vcf_parsing.parse_platypus_vcf_record,
                       'pindel': vcf_parsing.parse_pindel_vcf_record}

    for sample in samples:
        caller_records = defaultdict(lambda: dict())

        sys.stdout.write("Parsing Caller VCF Files\n")
        vcf_parsing.parse_vcf("{}.mutect.normalized.vcf".format(sample), "mutect", caller_records)
        vcf_parsing.parse_vcf("{}.vardict.normalized.vcf".format(sample), "vardict", caller_records)
        vcf_parsing.parse_vcf("{}.freebayes.normalized.vcf".format(sample), "freebayes", caller_records)
        vcf_parsing.parse_vcf("{}.scalpel.normalized.vcf".format(sample), "scalpel", caller_records)
        vcf_parsing.parse_vcf("{}.platypus.normalized.vcf".format(sample), "platypus", caller_records)
        vcf_parsing.parse_vcf("{}.pindel.normalized.vcf".format(sample), "pindel", caller_records)

        annotated_vcf = "{}.vcfanno.snpEff.GRCh37.75.vcf".format(sample)

        sys.stdout.write("Parsing VCFAnno VCF\n")
        vcf = VCF(annotated_vcf)

        sys.stdout.write("Parsing VCFAnno VCF with CyVCF2\n")
        reader = cyvcf2.VCFReader(annotated_vcf)
        desc = reader["ANN"]["Description"]
        annotation_keys = [x.strip("\"'") for x in re.split("\s*\|\s*", desc.split(":", 1)[1].strip('" '))]

        # Filter out variants with minor allele frequencies above the threshold but
        # retain any that are above the threshold but in COSMIC or in ClinVar and not listed as benign.
        sys.stdout.write("Processing individual variants\n")

        lib_variant_inserts = list()
        variant_inserts = list()

        for var in vcf:
            # Parsing VCF and creating data structures for Cassandra model
            callers = var.INFO.get('CALLERS').split(',')
            effects = utils.get_effects(var, annotation_keys)
            top_impact = utils.get_top_impact(effects)
            amplicon_data = get_amplicon_data(var)

            var_key_string = "{}_{}_{}_{}_{}".format(config['genome_version'], var.CHROM, var.start, var.REF,
                                                     var.ALT[0])

            library_var_key_string = "{}_{}_{}_{}_{}_{}".format(config['genome_version'],
                                                                samples[sample]['library_name'], var.CHROM,
                                                                var.start, var.REF, var.ALT[0])

            var_id = ObjectId(bytes(var_key_string.encode('UTF-8')))
            lib_var_id = ObjectId(bytes(library_var_key_string.encode('UTF-8')))

            caller_variant_data_dicts = defaultdict(dict)
            max_som_aaf = -1.00
            max_depth = -1
            min_depth = 100000000

            key = ("chr{}".format(var.CHROM), int(var.start), int(var.end), var.REF, var.ALT[0])
            for caller in callers:
                caller_variant_data_dicts[caller] = parse_functions[caller](caller_records[caller][key])
                if float(caller_variant_data_dicts[caller]['AAF']) > max_som_aaf:
                    max_som_aaf = float(caller_variant_data_dicts[caller]['AAF'])
                if int(caller_variant_data_dicts[caller]['DP']) < min_depth:
                    min_depth = int(caller_variant_data_dicts[caller]['DP'])
                if int(caller_variant_data_dicts[caller]['DP']) > max_depth:
                    max_depth = int(caller_variant_data_dicts[caller]['DP'])

            if min_depth == 100000000:
                min_depth = -1

            variant_inserts.append(
                {
                    "_id": var_key_string,
                    "chr": var.CHROM,
                    "start": var.start,
                    "end": var.end,
                    "ref": var.REF,
                    "alt": var.ALT[0],
                    "subtype": var.INFO.get('sub_type'),
                    "type": var.INFO.get('type'),
                    "rs_id": var.ID,
                    "date_annotated": datetime.now(),

                    "gene_info": {
                        "gene": top_impact.gene,
                        "transcript": top_impact.transcript,
                        "exon": top_impact.exon,
                        "codon_change": top_impact.codon_change,
                        "biotype": top_impact.biotype,
                        "aa_change": top_impact.aa_change,
                    },

                    "impact_info": {
                        "severity": top_impact.effect_severity,
                        "impact": top_impact.top_consequence,
                        "impact_so": top_impact.so,
                        "is_clinvar_pathogenic": vcf_parsing.var_is_pathogenic(var),
                        "is_lof": vcf_parsing.var_is_lof(var),
                        "is_coding": vcf_parsing.var_is_coding(var),
                        "is_splicing": vcf_parsing.var_is_splicing(var),
                        "in_clinvar": vcf_parsing.var_is_in_clinvar(var)
                    },

                    "cosmic_info": {
                        "in_cosmic": vcf_parsing.var_is_in_cosmic(var)
                    },

                    # "transcripts_data": utils.get_transcript_effects(effects),
                    "clinvar_data": utils.get_clinvar_info(var),
                    "cosmic_data": utils.get_cosmic_info(var),

                    "freq_info": {
                        "max_maf_all": var.INFO.get('max_aaf_all') or -1,
                        "max_maf_no_fin": var.INFO.get('max_aaf_no_fin') or -1,
                        "population_freqs": get_population_freqs(var)
                    },

                    "rs_ids": vcf_parsing.parse_rs_ids(var),
                    "cosmic_ids": vcf_parsing.parse_cosmic_ids(var),
                })

            lib_variant_inserts.append(
                {
                    "_id": library_var_key_string,
                    "sample": samples[sample]['sample_name'],
                    "extraction": samples[sample]['extraction'],
                    "library_name": samples[sample]['library_name'],
                    "run_id": samples[sample]['run_id'],
                    "panel_name": samples[sample]['panel'],
                    "target_pool": samples[sample]['target_pool'],
                    "sequencer": samples[sample]['sequencer'],

                    "callers": callers,
                    "amplicon_data": amplicon_data,
                    "max_som_aaf": max_som_aaf,
                    "min_depth": min_depth,
                    "max_depth": max_depth,

                    "variant_calling_data": {
                        "mutect": caller_variant_data_dicts['mutect'] or dict(),
                        "freebayes": caller_variant_data_dicts['freebayes'] or dict(),
                        "scalpel": caller_variant_data_dicts['scalpel'] or dict(),
                        "platypus": caller_variant_data_dicts['platypus'] or dict(),
                        "pindel": caller_variant_data_dicts['pindel'] or dict(),
                        "vardict": caller_variant_data_dicts['vardict'] or dict(),
                        "manta": caller_variant_data_dicts['manta'] or dict()
                    }
                })

        sys.stdout.write("Adding info to database for library: {}\n".format(samples[sample]['library_name']))
        db['lib_var_collection'].insert_many(lib_variant_inserts)
        db['var_collection'].insert_many(variant_inserts)
<commit_msg>fixing encoding issues<commit_after>#!/usr/bin/env python

import re
import sys
import utils
import cyvcf2
import getpass
import binascii
import argparse
import vcf_parsing
import configuration

from cyvcf2 import VCF
from bson import ObjectId
from datetime import datetime
from pymongo import MongoClient
from collections import defaultdict

def get_population_freqs(variant):
    freqs = {'esp_ea': variant.INFO.get('aaf_esp_ea') or -1,
             'esp_aa': variant.INFO.get('aaf_esp_aa') or -1,
             'esp_all': variant.INFO.get('aaf_esp_all') or -1,
             '1kg_amr': variant.INFO.get('aaf_1kg_amr') or -1,
             '1kg_eas': variant.INFO.get('aaf_1kg_eas') or -1,
             '1kg_sas': variant.INFO.get('aaf_1kg_sas') or -1,
             '1kg_afr': variant.INFO.get('aaf_1kg_afr') or -1,
             '1kg_eur': variant.INFO.get('aaf_1kg_eur') or -1,
             '1kg_all': variant.INFO.get('aaf_1kg_all') or -1,
             'exac_all': variant.INFO.get('aaf_exac_all') or -1,
             'adj_exac_all': variant.INFO.get('aaf_adj_exac_all') or -1,
             'adj_exac_afr': variant.INFO.get('aaf_adj_exac_afr') or -1,
             'adj_exac_amr': variant.INFO.get('aaf_adj_exac_amr') or -1,
             'adj_exac_eas': variant.INFO.get('aaf_adj_exac_eas') or -1,
             'adj_exac_fin': variant.INFO.get('aaf_adj_exac_fin') or -1,
             'adj_exac_nfe': variant.INFO.get('aaf_adj_exac_nfe') or -1,
             'adj_exac_oth': variant.INFO.get('aaf_adj_exac_oth') or -1,
             'adj_exac_sas': variant.INFO.get('aaf_adj_exac_sas') or -1}

    return freqs

def get_amplicon_data(variant):
    data = {'amplicon': variant.INFO.get('amplicon_target') or "None",
            'panel_amplicon': variant.INFO.get('panel_target') or "None",
            'intersect': variant.INFO.get('amplicon_intersect') or "None"}

    return data

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    # parser.add_argument('-a', '--address', help="IP Address for Cassandra connection", default='127.0.0.1')
    # parser.add_argument('-u', '--username', help='Cassandra username for login', default=None)

    args = parser.parse_args()
    args.logLevel = "INFO"

    date = datetime.today()

    client = MongoClient()
    db = client['ddb']

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    parse_functions = {'mutect': vcf_parsing.parse_mutect_vcf_record,
                       'freebayes': vcf_parsing.parse_freebayes_vcf_record,
                       'vardict': vcf_parsing.parse_vardict_vcf_record,
                       'scalpel': vcf_parsing.parse_scalpel_vcf_record,
                       'platypus': vcf_parsing.parse_platypus_vcf_record,
                       'pindel': vcf_parsing.parse_pindel_vcf_record}

    for sample in samples:
        caller_records = defaultdict(lambda: dict())

        sys.stdout.write("Parsing Caller VCF Files\n")
        vcf_parsing.parse_vcf("{}.mutect.normalized.vcf".format(sample), "mutect", caller_records)
        vcf_parsing.parse_vcf("{}.vardict.normalized.vcf".format(sample), "vardict", caller_records)
        vcf_parsing.parse_vcf("{}.freebayes.normalized.vcf".format(sample), "freebayes", caller_records)
        vcf_parsing.parse_vcf("{}.scalpel.normalized.vcf".format(sample), "scalpel", caller_records)
        vcf_parsing.parse_vcf("{}.platypus.normalized.vcf".format(sample), "platypus", caller_records)
        vcf_parsing.parse_vcf("{}.pindel.normalized.vcf".format(sample), "pindel", caller_records)

        annotated_vcf = "{}.vcfanno.snpEff.GRCh37.75.vcf".format(sample)

        sys.stdout.write("Parsing VCFAnno VCF\n")
        vcf = VCF(annotated_vcf)

        sys.stdout.write("Parsing VCFAnno VCF with CyVCF2\n")
        reader = cyvcf2.VCFReader(annotated_vcf)
        desc = reader["ANN"]["Description"]
        annotation_keys = [x.strip("\"'") for x in re.split("\s*\|\s*", desc.split(":", 1)[1].strip('" '))]

        # Filter out variants with minor allele frequencies above the threshold but
        # retain any that are above the threshold but in COSMIC or in ClinVar and not listed as benign.
        sys.stdout.write("Processing individual variants\n")

        lib_variant_inserts = list()
        variant_inserts = list()

        for var in vcf:
            # Parsing VCF and creating data structures for Cassandra model
            callers = var.INFO.get('CALLERS').split(',')
            effects = utils.get_effects(var, annotation_keys)
            top_impact = utils.get_top_impact(effects)
            amplicon_data = get_amplicon_data(var)

            var_key_string = "{}_{}_{}_{}_{}".format(config['genome_version'], var.CHROM, var.start, var.REF,
                                                     var.ALT[0])

            library_var_key_string = "{}_{}_{}_{}_{}_{}".format(config['genome_version'],
                                                                samples[sample]['library_name'], var.CHROM,
                                                                var.start, var.REF, var.ALT[0])

            var_id = ObjectId(var_key_string.encode('utf-16'))
            lib_var_id = ObjectId(library_var_key_string.encode('utf-16'))

            caller_variant_data_dicts = defaultdict(dict)
            max_som_aaf = -1.00
            max_depth = -1
            min_depth = 100000000

            key = ("chr{}".format(var.CHROM), int(var.start), int(var.end), var.REF, var.ALT[0])
            for caller in callers:
                caller_variant_data_dicts[caller] = parse_functions[caller](caller_records[caller][key])
                if float(caller_variant_data_dicts[caller]['AAF']) > max_som_aaf:
                    max_som_aaf = float(caller_variant_data_dicts[caller]['AAF'])
                if int(caller_variant_data_dicts[caller]['DP']) < min_depth:
                    min_depth = int(caller_variant_data_dicts[caller]['DP'])
                if int(caller_variant_data_dicts[caller]['DP']) > max_depth:
                    max_depth = int(caller_variant_data_dicts[caller]['DP'])

            if min_depth == 100000000:
                min_depth = -1

            variant_inserts.append(
                {
                    "_id": var_key_string,
                    "chr": var.CHROM,
                    "start": var.start,
                    "end": var.end,
                    "ref": var.REF,
                    "alt": var.ALT[0],
                    "subtype": var.INFO.get('sub_type'),
                    "type": var.INFO.get('type'),
                    "rs_id": var.ID,
                    "date_annotated": datetime.now(),

                    "gene_info": {
                        "gene": top_impact.gene,
                        "transcript": top_impact.transcript,
                        "exon": top_impact.exon,
                        "codon_change": top_impact.codon_change,
                        "biotype": top_impact.biotype,
                        "aa_change": top_impact.aa_change,
                    },

                    "impact_info": {
                        "severity": top_impact.effect_severity,
                        "impact": top_impact.top_consequence,
                        "impact_so": top_impact.so,
                        "is_clinvar_pathogenic": vcf_parsing.var_is_pathogenic(var),
                        "is_lof": vcf_parsing.var_is_lof(var),
                        "is_coding": vcf_parsing.var_is_coding(var),
                        "is_splicing": vcf_parsing.var_is_splicing(var),
                        "in_clinvar": vcf_parsing.var_is_in_clinvar(var)
                    },

                    "cosmic_info": {
                        "in_cosmic": vcf_parsing.var_is_in_cosmic(var)
                    },

                    # "transcripts_data": utils.get_transcript_effects(effects),
                    "clinvar_data": utils.get_clinvar_info(var),
                    "cosmic_data": utils.get_cosmic_info(var),

                    "freq_info": {
                        "max_maf_all": var.INFO.get('max_aaf_all') or -1,
                        "max_maf_no_fin": var.INFO.get('max_aaf_no_fin') or -1,
                        "population_freqs": get_population_freqs(var)
                    },

                    "rs_ids": vcf_parsing.parse_rs_ids(var),
                    "cosmic_ids": vcf_parsing.parse_cosmic_ids(var),
                })

            lib_variant_inserts.append(
                {
                    "_id": library_var_key_string,
                    "sample": samples[sample]['sample_name'],
                    "extraction": samples[sample]['extraction'],
                    "library_name": samples[sample]['library_name'],
                    "run_id": samples[sample]['run_id'],
                    "panel_name": samples[sample]['panel'],
                    "target_pool": samples[sample]['target_pool'],
                    "sequencer": samples[sample]['sequencer'],

                    "callers": callers,
                    "amplicon_data": amplicon_data,
                    "max_som_aaf": max_som_aaf,
                    "min_depth": min_depth,
                    "max_depth": max_depth,

                    "variant_calling_data": {
                        "mutect": caller_variant_data_dicts['mutect'] or dict(),
                        "freebayes": caller_variant_data_dicts['freebayes'] or dict(),
                        "scalpel": caller_variant_data_dicts['scalpel'] or dict(),
                        "platypus": caller_variant_data_dicts['platypus'] or dict(),
                        "pindel": caller_variant_data_dicts['pindel'] or dict(),
                        "vardict": caller_variant_data_dicts['vardict'] or dict(),
                        "manta": caller_variant_data_dicts['manta'] or dict()
                    }
                })

        sys.stdout.write("Adding info to database for library: {}\n".format(samples[sample]['library_name']))
        db['lib_var_collection'].insert_many(lib_variant_inserts)
        db['var_collection'].insert_many(variant_inserts)
<|endoftext|>


<commit_before>__author__ = 'mike.lyons'
print "test v1.0.4"
print "Hello Customer!"

while True:
    x = raw_input("Enter any number: ")
    y = raw_input("Enter another number: ")

    try:
        x = float(x)    # use float so that division works
        y = float(y)
    except ValueError:
        x = 0
        y = 0

    print "Sum: " + str(x+y)
    print "Difference: " + str(x-y)
    print "Product: " + str(x*y)
    print "Quotient: " + (str(x/y) if y != 0.0 else str(0.0))
    print "Square of 2nd number: " + str(y**2)

    user_exit = raw_input("Exit? (y/n): ")
    if user_exit == 'y' or user_exit == 'Y':
        break
print "Goodbye Customer!"
<commit_msg>release change<commit_after>__author__ = 'mike.lyons'
print "test v1.0.5"
print "Hello Customer!"

while True:
    x = raw_input("Enter any number: ")
    y = raw_input("Enter another number: ")

    try:
        x = float(x)    # use float so that division works
        y = float(y)
    except ValueError:
        x = 0
        y = 0

    print "Sum: " + str(x+y)
    print "Difference: " + str(x-y)
    print "Product: " + str(x*y)
    print "Quotient: " + (str(x/y) if y != 0.0 else str(0.0))
    print "Square of 2nd number: " + str(y**2)

    user_exit = raw_input("Exit? (y/n): ")
    if user_exit == 'y' or user_exit == 'Y':
        break
print "Goodbye Customer!"
<|endoftext|>


<commit_before>import pyximport; pyximport.install()

import os
os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"

from libhdfs3 import chdfs

client = chdfs.HDFSClient()

# print(chdfs.O_RDONLY)
# print(chdfs.O_WRONLY)
# print(chdfs.O_APPEND)
# print(chdfs.O_WRONLY | chdfs.O_APPEND)

# print client.exists('/tmp')
# print client.exists('/fake')

# print client.rename('/tmp/hadoop-yarn', '/tmp/hhh')
# print client.rename('/tmp/test', '/tmp/test2')

# print client.createDirectory('/tmp/mydir')

# print client.delete('/tmp/test2')
# print client.delete('/tmp/mydir', recursive=True)

# l = client.ls("/tmp")
# print l[0]
# print l[1]

# FILE OPERATIONS

# f = client.open('/tmp/test', 'w')
# msg = b"a" * 100
# f.write(msg)
# f.close()

# f = client.open('/tmp/test', 'r')
# print f.read()
# f.close()

# f = client.open('/tmp/test', 'r')
# print f.tell()
# print f.seek(10)
# print f.tell()
# f.close()

# f = client.open('/tmp/test2', 'w')
# msg = b"a" * 10
# f.write(msg)
# msg = b"b" * 10
# f.write(msg)
# msg = b"c" * 10
# f.write(msg)
# f.close()

# f = client.open('/tmp/test2', 'r')
# print f.read(10)
# print f.read(10)
# print f.read(10)
# f.close()

# f = client.open('/tmp/test2', 'r')
# print f.seek(10)
# print f.tell()
# print f.read(10)
# f.close()

# f = client.open('/tmp/test2', 'r')
# print f.seek(20)
# print f.tell()
# print f.read(10)
# f.close()

# f = client.open('/tmp/test', 'a')
# print client.getLastError()
# msg = b"b" * 100
# f.write(msg)
# msg = b"c" * 100
# f.write(msg)
# f.close()

print client.getLastError()
<commit_msg>Update<commit_after>import pyximport; pyximport.install()

import os
os.environ["LIBHDFS3_CONF"] = "/etc/hadoop/conf/hdfs-site.xml"

from libhdfs3 import chdfs

print('-----------------')

client = chdfs.HDFSClient()

# print(client.list_dir('/', recurse=True))

# print client.exists('/tmp')

# print client.exists('/tmp/mydir')
# print client.create_dir('/tmp/mydir')
# print client.exists('/tmp/mydir')
# print client.delete('/tmp/mydir', recursive=True)
# print client.exists('/tmp/mydir')

# l = client.ls("/tmp")
# print l[0]
# print l[1]

# FILE OPERATIONS

# print(chdfs.O_RDONLY)
# print(chdfs.O_WRONLY)
# print(chdfs.O_APPEND)
# print(chdfs.O_WRONLY | chdfs.O_APPEND)

# f = client.open('/tmp/test', 'w')
# msg = b"a" * 100
# f.write(msg)
# f.close()

# f = client.open('/tmp/test', 'r')
# print f.read()
# f.close()

# f = client.open('/tmp/test', 'a')
# msg = b"z" * 100
# f.write(msg)
# f.close()

f = client.open('/tmp/test', 'r')
print f.tell()
print f.seek(800)
print f.tell()
print f.read(100)
f.close()

# print client.getLastError()
<|endoftext|>


<commit_before>import sys
import json
import pprint
import codecs

def lines(fp):
    print str(len(fp.readlines()))

tweet_input = '/home/venkatesan/coursera/data_science/datasci_course_materials/assignment1/problem_1_submission.txt' #output.txt problem_1_submission.txt
sentiment_input = '/home/venkatesan/coursera/data_science/datasci_course_materials/assignment1/AFINN-111.txt'

pp = pprint.PrettyPrinter(indent=4)

def sentiment(infile):
    scores = dict()
    sentiment_file = open(infile, "r")

    for line in sentiment_file:
        term, score = line.split("\t")
        scores[term] = int(score)

    sentiment_file.close()
    return scores

sentiment_scores = sentiment(sentiment_input)

#infile = open(tweet_input, "r")
infile = codecs.open(tweet_input,'r', encoding='utf-8')#.read()
lines = infile.readlines()
data_struc = dict()#dict() json.loads(infile)

#with codecs.open(tweet_input) as infile: #, 'rU', 'utf-8'
#    for line in infile:
#        data_struc.update(json.loads(line, encoding='utf-8'))

for line in lines:
#    try:
    data_struc.update(json.loads(line)) #, encoding='utf-8'
#    except:
#        print "Error in line:" + line #continue

#for line in lines:
#    data_struc.update(json.loads(line, encoding='utf-8'))
#print len(data_struc)
#pp.pprint(data_struc)
score = 0
tweets = list()
for data in data_struc:
#    for sentiment in sentiment_scores:
#        sentiment.decode()
    if data == "text":
        tweets.append(data_struc[data].encode('utf-8'))
#        tweet = data_struc[data].encode('utf-8')
#pp.pprint(tweets)
#            if sentiment in tweet: #data_struc[data]:
#                score += sentiment_scores[sentiment]
#                print sentiment + "\t" + str(sentiment_scores[sentiment])#str(sentiment_scores[sentiment]) str(score)
#            else:
#                print score
for sentiment in sentiment_scores:
    for tweet in tweets:
        if sentiment in tweet:
            score += sentiment_scores[sentiment]

print score
#            if sentiment not in tweet:

#            print data_struc[data]

#            data_struc[data].decode('utf-8')
#            if sentiment in data_struc[data]:
#                print sentiment + "\t" + str(sentiment_scores[sentiment])
#for data in data_struc:
#    if data == 'text':
#        print data_struc[data]
#    pp.pprint(data)
infile.close()
<commit_msg>Delete test.py<commit_after><|endoftext|>


<commit_before>

test = input()
if test == '\x0D':
    print('enter')
<commit_msg>This is a first test<commit_after>

test = input()
if test == '\x0D':
    print('enter')

test
<|endoftext|>


<commit_before>from common import bounty #pragma: no cover
from common.peers import * #pragma: no cover
from common import settings #pragma: no cover
from common.safeprint import safeprint #pragma: no cover
from multiprocessing import Queue, Process, Value #pragma: no cover
from time import sleep, time #pragma: no cover
import pickle, sys #pragma: no cover

def testBounty(ip, btc, rwd, desc, data=None):  #pragma: no cover
    safeprint(desc)
    test = bounty.Bounty(ip,btc,rwd,dataDict=data)
    dumped = pickle.dumps(test,1)
    safeprint(bounty.addBounty(dumped))

def waitForty(v,q): #pragma: no cover
    stamp = time()
    while q.empty():
        if time() - 10 > stamp:
            break #pragma: no cover
    try:
        feedback = q.get(False)
    except: #pragma: no cover
        safeprint("No feedback received from listener")
    safeprint("YOUR IP IS " + get_lan_ip() + ":44566")
    requestPeerlist(get_lan_ip() + ":44566")
    sleep(4)
    requestPeerlist("localhost:44566")
    sleep(4)
    if sys.version_info[0] == 3:
        requestBounties(get_lan_ip() + ":44566")
        sleep(4)
        requestBounties("localhost:44566")
        sleep(4)
        safeprint(bounty.getBountyList())
    #TODO move test incoming_bounty here
    #TODO move test incoming_bounty here
    safeprint("Sending term signal")
    v.value = False
    try:
        requestPeerlist("localhost:44566")
    except:
        safeprint("This was supposed to fail. Good job")

if __name__ == "__main__": #pragma: no cover
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Correctly formed bounty")
    testBounty('8.8.8.8',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 1 (ip failure)")
    testBounty('8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 2 (ip failure)")
    testBounty('8.8.8.8:88888888888888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 3 (ip failure)")
    testBounty('8.8.12348.8',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 4 (ip failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGGww4ZEDmP7u9",1440,"Malformed bounty 5 (btc failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",-1440,"Malformed bounty 6 (reward failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",0,"Malformed bounty 7 (signature failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 8 (requirement failure)", data={'reqs':{("__builtins__","pow",2,2):4}})
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 9 (requirements failure)", data={'reqs':{("sys","platform","index=2","end=3"):"win33"}})
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 10 (requirements failure)", data={'reqs':{("__builtins__","pow",2,2):4,("sys","platform"):"win33"}})
    safeprint("Malformed bounty 11 (timeout error)")
    safeprint(addBounty(pickle.dumps(bounty.Bounty('8.8.8.8:8888',"1LhPsd4ng3AXmVPzrLWprWFx351pW4HJm8",10900,timeout=1),0)))
    testBounty('8.8.8.8:8888',"1LhPsd4ng3AXmVPzrLWprWFx351pW4HJm8",10900,"Correctly formed bounty 2")
    testBounty('8.8.8.8:8888',"1MWSdYMKEpfWVxC6BGYARxsksaQuyEWzG5",1480,"Correctly formed bounty 3")
    testBounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290,"Correctly formed bounty 4")
    safeprint(bounty.getBountyList())
    bounty.saveToFile(bounty.getBountyList())
    bounty.loadFromFile()
    safeprint(bounty.getBountyList())
    safeprint("3 bounties should follow")
    safeprint(bounty.getBounty(settings.config.get('charity'),settings.config.get('propagate_factor')))
    safeprint(bounty.getBounty(False,2))
    safeprint(bounty.getBounty(True,2))
    settings.saveSettings()
    settings.loadSettings()
    saveToFile()
    getFromFile()
    if not (sys.version_info[0] < 3 and sys.platform == "win32"):
        safeprint("Test listener begin")
        v = Value('b',True)
        q = Queue()
        a = Process(target=waitForty,args=(v,q))
        a.start()
        listen(44566,False,q,v,False)
    from common.call import call
    safeprint(call("random","random"))
    safeprint(call("sys","platform",index=0))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) <= bounty.getBounty(False,2))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) >= bounty.getBounty(False,2))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) != bounty.getBounty(False,2))
    import rsa
    pub, priv = rsa.newkeys(1024)
    testSig = bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",0,ident="ABCDEFGHIJKLMNOP",dataDict={'author':"QRSTUVWXYZ"})
    bounty.addKey((pub.n,pub.e))
    testSig.sign(priv)
    safeprint("Test internal signature verification when signed")
    safeprint(testSig.isValid())
    safeprint("Test external signature verification when signed")
    safeprint(bounty.verify(testSig))
    safeprint("Test external signature verification when signed and fed from pickle")
    safeprint(bounty.verify(pickle.dumps(testSig,0)))
    safeprint("Testing addBounties")
    safeprint(bounty.addBounties([bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290),bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",0)]))
<commit_msg>Add tests [skip ci]<commit_after>from common import bounty #pragma: no cover
from common.peers import * #pragma: no cover
from common import settings #pragma: no cover
from common.safeprint import safeprint #pragma: no cover
from multiprocessing import Queue, Process, Value #pragma: no cover
from time import sleep, time #pragma: no cover
import pickle, sys #pragma: no cover

def testBounty(ip, btc, rwd, desc, data=None):  #pragma: no cover
    safeprint(desc)
    test = bounty.Bounty(ip,btc,rwd,dataDict=data)
    dumped = pickle.dumps(test,1)
    safeprint(bounty.addBounty(dumped))

def waitForty(v,q): #pragma: no cover
    stamp = time()
    while q.empty():
        if time() - 10 > stamp:
            break #pragma: no cover
    try:
        feedback = q.get(False)
    except: #pragma: no cover
        safeprint("No feedback received from listener")
    safeprint("YOUR IP IS " + get_lan_ip() + ":44566")
    requestPeerlist(get_lan_ip() + ":44566")
    sleep(4)
    requestPeerlist("localhost:44566")
    sleep(4)
    if sys.version_info[0] == 3:
        requestBounties(get_lan_ip() + ":44566")
        sleep(4)
        requestBounties("localhost:44566")
        sleep(4)
        safeprint(bounty.getBountyList())
    #TODO move test incoming_bounty here
    #TODO move test incoming_bounty here
    safeprint("Sending term signal")
    v.value = False
    try:
        requestPeerlist("localhost:44566")
    except:
        safeprint("This was supposed to fail. Good job")

if __name__ == "__main__": #pragma: no cover
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Correctly formed bounty")
    testBounty('8.8.8.8',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 1 (ip failure)")
    testBounty('8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 2 (ip failure)")
    testBounty('8.8.8.8:88888888888888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 3 (ip failure)")
    testBounty('8.8.12348.8',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 4 (ip failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGGww4ZEDmP7u9",1440,"Malformed bounty 5 (btc failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",-1440,"Malformed bounty 6 (reward failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",0,"Malformed bounty 7 (signature failure)")
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 8 (requirement failure)", data={'reqs':{("__builtins__","pow",2,2):4}})
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 9 (requirements failure)", data={'reqs':{("sys","platform","index=2","end=3"):"win33"}})
    testBounty('8.8.8.8:8888',"1JTGcHS3GMhBGLcFRuHLk6Gww4ZEDmP7u9",1440,"Malformed bounty 10 (requirements failure)", data={'reqs':{("__builtins__","pow",2,2):4,("sys","platform"):"win33"}})
    safeprint("Malformed bounty 11 (timeout error)")
    safeprint(addBounty(pickle.dumps(bounty.Bounty('8.8.8.8:8888',"1LhPsd4ng3AXmVPzrLWprWFx351pW4HJm8",10900,timeout=1),0)))
    testBounty('8.8.8.8:8888',"1LhPsd4ng3AXmVPzrLWprWFx351pW4HJm8",10900,"Correctly formed bounty 2")
    testBounty('8.8.8.8:8888',"1MWSdYMKEpfWVxC6BGYARxsksaQuyEWzG5",1480,"Correctly formed bounty 3")
    testBounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290,"Correctly formed bounty 4")
    safeprint(bounty.getBountyList())
    bounty.saveToFile(bounty.getBountyList())
    bounty.loadFromFile()
    safeprint(bounty.getBountyList())
    safeprint("3 bounties should follow")
    safeprint(bounty.getBounty(settings.config.get('charity'),settings.config.get('propagate_factor')))
    safeprint(bounty.getBounty(False,2))
    safeprint(bounty.getBounty(True,2))
    settings.saveSettings()
    settings.loadSettings()
    saveToFile()
    getFromFile()
    if not (sys.version_info[0] < 3 and sys.platform == "win32"):
        safeprint("Test listener begin")
        v = Value('b',True)
        q = Queue()
        a = Process(target=waitForty,args=(v,q))
        a.start()
        listen(44566,False,q,v,False)
    from common.call import call
    safeprint(call("random","random"))
    safeprint(call("sys","platform",index=0))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) <= bounty.getBounty(False,2))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) >= bounty.getBounty(False,2))
    safeprint(bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290) != bounty.getBounty(False,2))
    import rsa
    pub, priv = rsa.newkeys(1024)
    testSig = bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",0,keypair=priv,ident="ABCDEFGHIJKLMNOP",dataDict={'author':"QRSTUVWXYZ"})
    bounty.addKey((pub.n,pub.e))
    safeprint("Test internal signature verification when signed")
    safeprint(testSig.isValid())
    safeprint("Test external signature verification when signed")
    safeprint(bounty.verify(testSig))
    safeprint("Test external signature verification when signed and fed from pickle")
    safeprint(bounty.verify(pickle.dumps(testSig,0)))
    safeprint("Testing addBounties")
    bounty.addValidBounty(testSig)
    safeprint(bounty.addBounties([bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",10290),bounty.Bounty('8.8.8.8:8888',"1EgGfDetymjMzcQ1AEhHjHEyXHjnEavwgg",0)]))
<|endoftext|>


<commit_before>def test_good():
    assert True

def test_bad():
    assert False

def test_boom():
    what

def test_skip():
    from nose.plugins.skip import SkipTest
    raise SkipTest
<commit_msg>Add more silly tests to non-test test file re #8<commit_after>def test_good():
    assert True

def test_bad():
    assert False

def test_boom():
    what

def test_skip():
    from nose.plugins.skip import SkipTest
    raise SkipTest

class Foo(object):
    def has_no_underscore(self):
        assert True

    def has_no_Test(self):
        assert True

class Foo_(object):
    def should_print_out_as_Foo(self):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

from tempfile import NamedTemporaryFile
import unittest

import socket
import wait

class TestWait(unittest.TestCase):

    def setUp(self):
        self.file = NamedTemporaryFile()
        self.port = 9999
        self.patterns = ['foo', 'bar', 'f.*']

    def pattern(self, *args, **kwargs):
        return wait.log.pattern(self.file.name, *args, **kwargs)

    def write(self, s):
        self.file.write(s.encode('utf-8'))
        self.file.write('\n'.encode('utf-8'))
        self.file.flush()

    def test_log_exists(self):
        assert wait.log.exists(self.file.name)

    def test_log_exists_timeout(self):
        assert not wait.log.exists('/tmp/nolog', timeout=1)

    def test_log_pattern_list(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert self.pattern(self.patterns, seek=seek, timeout=5)

    def test_log_pattern_tuple(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert self.pattern(tuple(self.patterns), seek=seek, timeout=5)

    def test_log_pattern_string(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        assert self.pattern(self.patterns[0], seek=seek, timeout=5)

    def test_log_pattern_nostart(self):
        p = self.pattern(self.patterns, run=False, timeout=5)
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert p()

    def test_log_pattern_timeout(self):
        assert not wait.log.pattern('/tmp/nolog', self.patterns, timeout=1)
        assert not self.pattern(self.patterns, timeout=1)

    def test_tcp_closed(self):
        assert wait.tcp.closed(self.port, timeout=1)
        assert not wait.tcp.open(self.port, timeout=1)

    def test_tcp_open(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(('localhost', self.port))
        s.listen(0)
        assert not wait.tcp.closed(self.port, timeout=1)
        assert wait.tcp.open(self.port, timeout=1)
        assert wait.tcp.open(80, host='www.google.com', timeout=5)
        s.close()

    def test_tcp_open_timeout(self):
        assert not wait.tcp.open(self.port, timeout=1)

    def tearDown(self):
        self.file.close()

if __name__ == '__main__':
    unittest.main()

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<commit_msg>TCP socket timeout test case<commit_after>#!/usr/bin/env python

from tempfile import NamedTemporaryFile
import unittest

import socket
import wait

class TestWait(unittest.TestCase):

    def setUp(self):
        self.file = NamedTemporaryFile()
        self.port = 9999
        self.patterns = ['foo', 'bar', 'f.*']

    def pattern(self, *args, **kwargs):
        return wait.log.pattern(self.file.name, *args, **kwargs)

    def write(self, s):
        self.file.write(s.encode('utf-8'))
        self.file.write('\n'.encode('utf-8'))
        self.file.flush()

    def test_log_exists(self):
        assert wait.log.exists(self.file.name)

    def test_log_exists_timeout(self):
        assert not wait.log.exists('/tmp/nolog', timeout=1)

    def test_log_pattern_list(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert self.pattern(self.patterns, seek=seek, timeout=5)

    def test_log_pattern_tuple(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert self.pattern(tuple(self.patterns), seek=seek, timeout=5)

    def test_log_pattern_string(self):
        seek = self.file.tell()
        self.write(self.patterns[0])
        assert self.pattern(self.patterns[0], seek=seek, timeout=5)

    def test_log_pattern_nostart(self):
        p = self.pattern(self.patterns, run=False, timeout=5)
        self.write(self.patterns[0])
        self.write(self.patterns[1])
        assert p()

    def test_log_pattern_timeout(self):
        assert not wait.log.pattern('/tmp/nolog', self.patterns, timeout=1)
        assert not self.pattern(self.patterns, timeout=1)

    def test_tcp_closed(self):
        assert wait.tcp.closed(self.port, timeout=1)
        assert not wait.tcp.open(self.port, timeout=1)

    def test_tcp_open(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(('localhost', self.port))
        s.listen(0)
        assert not wait.tcp.closed(self.port, timeout=1)
        assert wait.tcp.open(self.port, timeout=1)
        assert wait.tcp.open(80, host='www.google.com', timeout=5)
        s.close()

    def test_tcp_socket_timeout(self):
        assert wait.tcp.closed(self.port, host='10.255.255.1', timeout=1)
        assert not wait.tcp.open(self.port, host='10.255.255.1', timeout=1)

    def test_tcp_open_timeout(self):
        assert not wait.tcp.open(self.port, timeout=1)

    def tearDown(self):
        self.file.close()

if __name__ == '__main__':
    unittest.main()

# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
<|endoftext|>


<commit_before>#! /usr/bin/env python

import os
import sys
import subprocess

argv    = sys.argv
pixels  = 4
bpixels = 8
zratio  = .25
bhabit  = ""

if(len(argv) < 4):
  print "no much argments."
elif(argv[2] == "match" or argv[2] == "match0" or argv[2] == "matcho"):
  root0, ext0 = os.path.splitext(argv[3])
  root1, ext1 = os.path.splitext(argv[4])
  nemph       = 4
  if((argv[2] == "match" or argv[2] == "match0") and len(argv) > 5):
    pixels = int(argv[5])
  elif(argv[2] == "matcho" and len(argv) > 6):
    pixels = int(argv[6])
    if(len(argv) > 7):
      nemph = int(argv[7])
  if(ext1 == ".obj" and (argv[2] == "match" or argv[2] == "match0")):
    subprocess.call([argv[1], argv[2], "16", "40", str(pixels), str(pixels), str(zratio), root0 + ".ppm", root0 + ".ppm", root0 + "-bump.ppm", argv[4], "match-" + root0 + "-" + argv[4]])
  elif(ext1 == ".obj"):
    subprocess.call([argv[1], argv[2], argv[5], str(nemph), str(pixels), str(pixels), str(zratio), root0 + ".ppm", root0 + ".ppm", root0 + "-bump.ppm", argv[4], argv[5]])
  elif(argv[2] == "match" or argv[2] == "match0"):
    subprocess.call([argv[1], argv[2], "16", "40", str(pixels), str(pixels), str(zratio), root0 + ".ppm", root1 + ".ppm", root0 + "-bump.ppm", root1 + "-bump.ppm", root0 + "-mask.ppm", root1 + "-mask.ppm", "match-" + root0 + "-" + root1])
  else:
    subprocess.call([argv[1], argv[2], argv[5], str(nemph), str(pixels), str(pixels), str(zratio), root0 + ".ppm", root1 + ".ppm", root0 + "-bump.ppm", root1 + "-bump.ppm", argv[5]])
  if(argv[2] == "matcho"):
    subprocess.call(["ffmpeg", "-loop", "1", "-i", argv[5] + "-%d-" + str(nemph) + ".ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", argv[5] + ".mp4"])
elif(argv[2] == "pcopy" or argv[2] == "ppred"):
  roots = []
  exts  = []
  for s in argv[3:]:
    try:
      pixels = int(s)
      continue
    except:
      r, e = os.path.splitext(s)
      roots.append(r)
      exts.append(e)
  cmd = [argv[1], argv[2], "1", ".1", str(zratio), str(pixels), "pose"]
  if(argv[2] == "pcopy"):
    cmd.extend([roots[0] + ".ppm", roots[0] + "-bump.ppm", roots[1] + ".ppm", roots[1] + "-bump.ppm"])
    subprocess.call(cmd)
  else:
    for s in roots:
      cmd.append(s + ".ppm")
      cmd.append(s + "-bump.ppm")
    subprocess.call(cmd)
  for s in range(0, pixels):
    subprocess.call(["cp", "pose" + str(pixels - 1 - s) + ".ppm", "pose" + str(pixels + s) + ".ppm"])
  subprocess.call(["ffmpeg", "-loop", "1", "-i", "pose%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", "pose.mp4"])
else:
  for line in argv[3:]:
    try:
      pixels = int(line)
      continue
    except:
      root, ext = os.path.splitext(line)
    if(ext != ".ppm"):
      subprocess.call(["convert", line, "-compress", "none", root + ".ppm"])
    if(argv[2] == "collect"):
      subprocess.call([argv[1], argv[2], root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "pextend" or argv[2] == "light"):
      subprocess.call([argv[1], argv[2], str(pixels), root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "bump"):
      subprocess.call([argv[1], argv[2], str(bpixels), root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "penetrate"):
      subprocess.call(["cp", root + ".ppm", root + "-penetrate-light.ppm"])
      for s in range(0, pixels):
        subprocess.call(["convert", root + "-penetrate-light.ppm", "-blur", "2x2", "-compress", "none", root + "-penetrate.ppm"])
        subprocess.call([argv[1], "light", "1", root + "-penetrate.ppm", root + "-penetrate-light.ppm"])
      subprocess.call(["convert", root + "-penetrate-light.ppm", root + "-penetrate.png"])
    elif(argv[2] == "enlarge"):
      subprocess.call(["cp", root + ".ppm", root + "-enl.ppm"])
      for s in range(0, pixels):
        subprocess.call([argv[1], argv[2], root + "-enl.ppm", root + "-enl0.ppm"])
        subprocess.call(["convert", root + "-enl0.ppm", "-resize", "75%", "-compress", "none", root + "-enl.ppm"])
      subprocess.call(["convert", root + "-enl.ppm", root + "-enl.png"])
    elif(argv[2] == "obj"):
      subprocess.call([argv[1], "obj", str(pixels), "1",  str(zratio), "0", root + "-bump.ppm", root + ".obj"])
      subprocess.call([argv[1], "obj", str(pixels), ".1", str(zratio), ".4", root + "-bump.ppm", root + "-mask.ppm", root + "-stand.obj"])
      subprocess.call(["xcrun", "scntool", "--convert", root + ".obj", "--format", "scn", "--output", root + ".scn"])
    elif(argv[2] == "jps"):
      subprocess.call([argv[1], "tilt", "1", "4", ".005", root + ".ppm", root + ".obj", root + "-L.ppm"])
      subprocess.call([argv[1], "tilt", "3", "4", ".005", root + ".ppm", root + ".obj", root + "-R.ppm"])
      subprocess.call(["montage", root + "-R.ppm", root + "-L.ppm", "-geometry", "100%x100%", root + "-stereo.jps"])
      subprocess.call(["montage", root + "-stereo.jps", "-geometry", "100%x100%", root + "-stereo.png"])
    elif(argv[2] == "tilt"):
      for s in range(0, pixels):
        subprocess.call([argv[1], "tilt", str(s), str(pixels), ".05", root + ".ppm", root + ".obj", root + "-tilt-base-" + str(s) + ".ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-tilt-base-%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + ".mp4"])
    elif(argv[2] == "btilt"):
      for s in range(0, pixels * 2):
        subprocess.call([argv[1], "tilt", "1", "4", str((s - pixels) / float(pixels) * .1), root + ".ppm", root + ".obj", root + "-btilt-base-" + str(s) + ".ppm"])
        subprocess.call(["cp", root + "-btilt-base-" + str(s) + ".ppm", root + "-btilt-base-" + str(pixels * 4 - s - 1) + ".ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-btilt-base-%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-b.mp4"])
    elif(argv[2] == "btilt2"):
      subprocess.call([argv[1], "tilt", "1", "4", ".0125", root + ".ppm", root + ".obj", root + "-btilt2-base-0.ppm"])
      subprocess.call([argv[1], "tilt", "3", "4", ".0125", root + ".ppm", root + ".obj", root + "-btilt2-base-1.ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-btilt2-base-%d.ppm", "-r", "20", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-b2.mp4"])
    elif(argv[2] == "flicker"):
      for s in range(0, pixels):
        subprocess.call([argv[1], "obj", "1", "1", str(s / float(pixels) * zratio), "0", root + "-bump.ppm", root + "-flicker.obj"])
        subprocess.call([argv[1], "tilt", "1", "4", "0.05", root + ".ppm", root + "-flicker.obj", root + "-flicker-base-" + str(s) + "-L.ppm"])
        subprocess.call([argv[1], "tilt", "3", "4", "0.05", root + ".ppm", root + "-flicker.obj", root + "-flicker-base-" + str(s) + "-R.ppm"])
        subprocess.call(["montage", root + "-flicker-base-" + str(s) + "-R.ppm", root + "-flicker-base-" + str(s) + "-L.ppm", "-geometry", "100%x100%", root + "-flicker-base-" + str(s) + ".png"])
        subprocess.call(["cp", root + "-flicker-base-" + str(s) + ".png", root + "-flicker-base-" + str(16 * 2 - s - 1) + ".png"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-flicker-base-%d.png", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-ficker.mp4"])
    elif(argv[2] == "sbox"):
      for s in range(0, pixels):
        subprocess.call([argv[1], argv[2], str(- s * 4 / float(pixels)), "1", root + ".ppm", root + ".obj", root + "-sbox-" + str(pixels - 1 - s) + ".ppm"])
    elif(argv[2] == "demosaic"):
      subprocess.call(["convert", line, "-resize", str(int(10000. * pow(1.5, - float(pixels))) / 100.) + "%", "-compress", "none", root + "-demosaic0.ppm"])
      subprocess.call(["python2", argv[0], argv[1], "enlarge", str(pixels), root + "-demosaic0.ppm"])
    elif(argv[2] == "extend"):
      for tami in range(1, pixels + 1):
        tam = tami / 360. * 60 / pixels
        for s in range(0, 4):
          subprocess.call([argv[1], "tilt", str(s), "4", str(tam), root + ".ppm", root + ".obj", root + "-tilt" + str(s) + ".ppm"])
          subprocess.call([argv[1], "bump", str(bpixels), root + "-tilt" + str(s) + ".ppm", root + "-bumpext" + str(s) + ".ppm"])
          subprocess.call([argv[1], "obj", "1", "1", str(zratio), "0", root + "-bumpext" + str(s) + ".ppm", root + "-bumpext" + str(s) + ".obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext0.obj", "0", "4", str(tam), root + "-bumpextA.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext1.obj", "1", "4", str(tam), root + "-bumpextB.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext2.obj", "2", "4", str(tam), root + "-bumpextC.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext3.obj", "3", "4", str(tam), root + "-bumpextD.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextA.obj", root + "-bumpextC.obj", "0", "1", "0", root + "-bumpextAC.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextB.obj", root + "-bumpextD.obj", "0", "1", "0", root + "-bumpextBD.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextAC.obj", root + "-bumpextBD.obj", "0", "1", "0", root + "-bumpext.obj"])
        subprocess.call(["cp", root + "-bumpext.obj", root + ".obj"])
    elif(argv[2] == "prep"):
      subprocess.call(["convert", line, "-resize", str(pixels) + "@>", root + "-prep.png"])
    elif(argv[2] == "prepsq"):
      subprocess.call(["convert", line, "-resize", str(pixels) + "x" + str(pixels) + "!", root + "-prepsq.png"])
    elif(argv[2] == "mask"):
      subprocess.call(["convert", root + "-mask.png", "-compress", "none", root + "-mask.ppm"])
    elif(argv[2] == "mask0"):
      subprocess.call(["convert", root + ".ppm", "-fill", "black", "-colorize", "100", root + "-mask.png"])

<commit_msg>code clean.<commit_after>#! /usr/bin/env python

import os
import sys
import subprocess

argv    = sys.argv
pixels  = 4
bpixels = 8
zratio  = .25
psi     = .0125
bhabit  = ""

if(len(argv) < 4):
  print "no much argments."
elif(argv[2] == "match" or argv[2] == "match0" or argv[2] == "matcho"):
  root0, ext0 = os.path.splitext(argv[3])
  root1, ext1 = os.path.splitext(argv[4])
  nemph       = 4
  if((argv[2] == "match" or argv[2] == "match0") and len(argv) > 5):
    pixels = int(argv[5])
  elif(argv[2] == "matcho" and len(argv) > 6):
    pixels = int(argv[6])
    if(len(argv) > 7):
      nemph = int(argv[7])
  if(ext1 == ".obj" and (argv[2] == "match" or argv[2] == "match0")):
    subprocess.call([argv[1], argv[2], "16", "40", str(pixels), str(pixels), str(zratio), root0 + ".ppm", root0 + ".ppm", root0 + "-bump.ppm", argv[4], "match-" + root0 + "-" + argv[4]])
  elif(ext1 == ".obj"):
    subprocess.call([argv[1], argv[2], argv[5], str(nemph), str(pixels), str(pixels), str(zratio), root0 + ".ppm", root0 + ".ppm", root0 + "-bump.ppm", argv[4], argv[5]])
  elif(argv[2] == "match" or argv[2] == "match0"):
    subprocess.call([argv[1], argv[2], "16", "40", str(pixels), str(pixels), str(zratio), root0 + ".ppm", root1 + ".ppm", root0 + "-bump.ppm", root1 + "-bump.ppm", root0 + "-mask.ppm", root1 + "-mask.ppm", "match-" + root0 + "-" + root1])
  else:
    subprocess.call([argv[1], argv[2], argv[5], str(nemph), str(pixels), str(pixels), str(zratio), root0 + ".ppm", root1 + ".ppm", root0 + "-bump.ppm", root1 + "-bump.ppm", argv[5]])
  if(argv[2] == "matcho"):
    subprocess.call(["ffmpeg", "-loop", "1", "-i", argv[5] + "-%d-" + str(nemph) + ".ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", argv[5] + ".mp4"])
elif(argv[2] == "pred"):
  cmd = [argv[1], argv[2], "pred.ppm"]
  for s in argv[3:]:
    r, e = os.path.splitext(s)
    cmd.append(r + ".ppm")
  subprocess.call(cmd)
elif(argv[2] == "pcopy" or argv[2] == "ppred"):
  roots = []
  exts  = []
  for s in argv[3:]:
    try:
      pixels = int(s)
      continue
    except:
      r, e = os.path.splitext(s)
      roots.append(r)
      exts.append(e)
  cmd = [argv[1], argv[2], "1", ".1", str(zratio), str(pixels), "pose"]
  if(argv[2] == "pcopy"):
    cmd.extend([roots[0] + ".ppm", roots[0] + "-bump.ppm", roots[1] + ".ppm", roots[1] + "-bump.ppm"])
  else:
    for s in roots:
      cmd.append(s + ".ppm")
      cmd.append(s + "-bump.ppm")
  subprocess.call(cmd)
  for s in range(0, pixels):
    subprocess.call(["cp", "pose" + str(pixels - 1 - s) + ".ppm", "pose" + str(pixels + s) + ".ppm"])
  subprocess.call(["ffmpeg", "-loop", "1", "-i", "pose%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", "pose.mp4"])
else:
  for line in argv[3:]:
    try:
      pixels = int(line)
      continue
    except:
      root, ext = os.path.splitext(line)
    if(ext != ".ppm"):
      subprocess.call(["convert", line, "-compress", "none", root + ".ppm"])
    if(argv[2] == "collect"):
      subprocess.call([argv[1], argv[2], root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "pextend" or argv[2] == "light"):
      subprocess.call([argv[1], argv[2], str(pixels), root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "bump"):
      subprocess.call([argv[1], argv[2], str(bpixels), root + ".ppm", root + "-" + argv[2] + ".ppm"])
    elif(argv[2] == "penetrate"):
      subprocess.call(["cp", root + ".ppm", root + "-penetrate-light.ppm"])
      for s in range(0, pixels):
        subprocess.call(["convert", root + "-penetrate-light.ppm", "-blur", "2x2", "-compress", "none", root + "-penetrate.ppm"])
        subprocess.call([argv[1], "light", "1", root + "-penetrate.ppm", root + "-penetrate-light.ppm"])
      subprocess.call(["convert", root + "-penetrate-light.ppm", root + "-penetrate.png"])
    elif(argv[2] == "enlarge"):
      subprocess.call(["cp", root + ".ppm", root + "-enl.ppm"])
      for s in range(0, pixels):
        subprocess.call([argv[1], argv[2], root + "-enl.ppm", root + "-enl0.ppm"])
        subprocess.call(["convert", root + "-enl0.ppm", "-resize", "75%", "-compress", "none", root + "-enl.ppm"])
      subprocess.call(["convert", root + "-enl.ppm", root + "-enl.png"])
    elif(argv[2] == "obj"):
      subprocess.call([argv[1], "obj", str(pixels), "1",  str(zratio), "0", root + "-bump.ppm", root + ".obj"])
      subprocess.call([argv[1], "obj", str(pixels), ".1", str(zratio), ".4", root + "-bump.ppm", root + "-mask.ppm", root + "-stand.obj"])
      subprocess.call(["xcrun", "scntool", "--convert", root + ".obj", "--format", "scn", "--output", root + ".scn"])
    elif(argv[2] == "jps"):
      subprocess.call([argv[1], "tilt", "1", "4", str(psi), root + ".ppm", root + ".obj", root + "-L.ppm"])
      subprocess.call([argv[1], "tilt", "3", "4", str(psi), root + ".ppm", root + ".obj", root + "-R.ppm"])
      subprocess.call(["montage", root + "-R.ppm", root + "-L.ppm", "-geometry", "100%x100%", root + "-stereo.jps"])
      subprocess.call(["montage", root + "-stereo.jps", "-geometry", "100%x100%", root + "-stereo.png"])
    elif(argv[2] == "tilt"):
      for s in range(0, pixels):
        subprocess.call([argv[1], "tilt", str(s), str(pixels), str(psi), root + ".ppm", root + ".obj", root + "-tilt-base-" + str(s) + ".ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-tilt-base-%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + ".mp4"])
    elif(argv[2] == "btilt"):
      for s in range(0, pixels * 2):
        subprocess.call([argv[1], "tilt", "1", "4", str((s - pixels) / float(pixels) * psi), root + ".ppm", root + ".obj", root + "-btilt-base-" + str(s) + ".ppm"])
        subprocess.call(["cp", root + "-btilt-base-" + str(s) + ".ppm", root + "-btilt-base-" + str(pixels * 4 - s - 1) + ".ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-btilt-base-%d.ppm", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-b.mp4"])
    elif(argv[2] == "btilt2"):
      subprocess.call([argv[1], "tilt", "1", "4", str(psi), root + ".ppm", root + ".obj", root + "-btilt2-base-0.ppm"])
      subprocess.call([argv[1], "tilt", "3", "4", str(psi), root + ".ppm", root + ".obj", root + "-btilt2-base-1.ppm"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-btilt2-base-%d.ppm", "-r", "20", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-b2.mp4"])
    elif(argv[2] == "flicker"):
      for s in range(0, pixels):
        subprocess.call([argv[1], "obj", "1", "1", str(s / float(pixels) * zratio), "0", root + "-bump.ppm", root + "-flicker.obj"])
        subprocess.call([argv[1], "tilt", "1", "4", str(psi), root + ".ppm", root + "-flicker.obj", root + "-flicker-base-" + str(s) + "-L.ppm"])
        subprocess.call([argv[1], "tilt", "3", "4", str(psi), root + ".ppm", root + "-flicker.obj", root + "-flicker-base-" + str(s) + "-R.ppm"])
        subprocess.call(["montage", root + "-flicker-base-" + str(s) + "-R.ppm", root + "-flicker-base-" + str(s) + "-L.ppm", "-geometry", "100%x100%", root + "-flicker-base-" + str(s) + ".png"])
        subprocess.call(["cp", root + "-flicker-base-" + str(s) + ".png", root + "-flicker-base-" + str(16 * 2 - s - 1) + ".png"])
      subprocess.call(["ffmpeg", "-loop", "1", "-i", root + "-flicker-base-%d.png", "-r", "6", "-an", "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2", "-vcodec", "libx264", "-pix_fmt", "yuv420p", "-t", "12", root + "-ficker.mp4"])
    elif(argv[2] == "sbox"):
      for s in range(0, pixels):
        subprocess.call([argv[1], argv[2], str(- s * 4 / float(pixels)), "1", root + ".ppm", root + ".obj", root + "-sbox-" + str(pixels - 1 - s) + ".ppm"])
    elif(argv[2] == "demosaic"):
      subprocess.call(["convert", line, "-resize", str(int(10000. * pow(1.5, - float(pixels))) / 100.) + "%", "-compress", "none", root + "-demosaic0.ppm"])
      subprocess.call(["python2", argv[0], argv[1], "enlarge", str(pixels), root + "-demosaic0.ppm"])
    elif(argv[2] == "extend"):
      for tami in range(1, pixels + 1):
        tam = tami / 360. * 60 / pixels
        for s in range(0, 4):
          subprocess.call([argv[1], "tilt", str(s), "4", str(tam), root + ".ppm", root + ".obj", root + "-tilt" + str(s) + ".ppm"])
          subprocess.call([argv[1], "bump", str(bpixels), root + "-tilt" + str(s) + ".ppm", root + "-bumpext" + str(s) + ".ppm"])
          subprocess.call([argv[1], "obj", "1", "1", str(zratio), "0", root + "-bumpext" + str(s) + ".ppm", root + "-bumpext" + str(s) + ".obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext0.obj", "0", "4", str(tam), root + "-bumpextA.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext1.obj", "1", "4", str(tam), root + "-bumpextB.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext2.obj", "2", "4", str(tam), root + "-bumpextC.obj"])
        subprocess.call([argv[1], "habit", root + ".obj", root + "-bumpext3.obj", "3", "4", str(tam), root + "-bumpextD.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextA.obj", root + "-bumpextC.obj", "0", "1", "0", root + "-bumpextAC.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextB.obj", root + "-bumpextD.obj", "0", "1", "0", root + "-bumpextBD.obj"])
        subprocess.call([argv[1], "habit", root + "-bumpextAC.obj", root + "-bumpextBD.obj", "0", "1", "0", root + "-bumpext.obj"])
        subprocess.call(["cp", root + "-bumpext.obj", root + ".obj"])
    elif(argv[2] == "prep"):
      subprocess.call(["convert", line, "-resize", str(pixels) + "@>", root + "-prep.png"])
    elif(argv[2] == "prepsq"):
      subprocess.call(["convert", line, "-resize", str(pixels) + "x" + str(pixels) + "!", root + "-prepsq.png"])
    elif(argv[2] == "mask"):
      subprocess.call(["convert", root + "-mask.png", "-compress", "none", root + "-mask.ppm"])
    elif(argv[2] == "mask0"):
      subprocess.call(["convert", root + ".ppm", "-fill", "black", "-colorize", "100", root + "-mask.png"])

<|endoftext|>