<commit_before>import sys
import os
import unittest
from array import array
from weakref import proxy

from test.test_support import TESTFN, findfile, run_unittest
from UserList import UserList

class AutoFileTests(unittest.TestCase):
    # file tests for which a test file is automatically set up

    def setUp(self):
        self.f = open(TESTFN, 'wb')

    def tearDown(self):
        if self.f:
            self.f.close()
        os.remove(TESTFN)

    def testWeakRefs(self):
        # verify weak references
        p = proxy(self.f)
        p.write('teststring')
        self.assertEquals(self.f.tell(), p.tell())
        self.f.close()
        self.f = None
        self.assertRaises(ReferenceError, getattr, p, 'tell')

    def testAttributes(self):
        # verify expected attributes exist
        f = self.f
        softspace = f.softspace
        f.name     # merely shouldn't blow up
        f.mode     # ditto
        f.closed   # ditto

        # verify softspace is writable
        f.softspace = softspace    # merely shouldn't blow up

        # verify the others aren't
        for attr in 'name', 'mode', 'closed':
            self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops')

    def testReadinto(self):
        # verify readinto
        self.f.write('12')
        self.f.close()
        a = array('c', 'x'*10)
        self.f = open(TESTFN, 'rb')
        n = self.f.readinto(a)
        self.assertEquals('12', a.tostring()[:n])

    def testWritelinesUserList(self):
        # verify writelines with instance sequence
        l = UserList(['1', '2'])
        self.f.writelines(l)
        self.f.close()
        self.f = open(TESTFN, 'rb')
        buf = self.f.read()
        self.assertEquals(buf, '12')

    def testWritelinesIntegers(self):
        # verify writelines with integers
        self.assertRaises(TypeError, self.f.writelines, [1, 2, 3])

    def testWritelinesIntegersUserList(self):
        # verify writelines with integers in UserList
        l = UserList([1,2,3])
        self.assertRaises(TypeError, self.f.writelines, l)

    def testWritelinesNonString(self):
        # verify writelines with non-string object
        class NonString:
            pass

        self.assertRaises(TypeError, self.f.writelines,
                          [NonString(), NonString()])

    def testRepr(self):
        # verify repr works
        self.assert_(repr(self.f).startswith("<open file '" + TESTFN))

    def testErrors(self):
        f = self.f
        self.assertEquals(f.name, TESTFN)
        self.assert_(not f.isatty())
        self.assert_(not f.closed)

        self.assertRaises(TypeError, f.readinto, "")
        f.close()
        self.assert_(f.closed)

    def testMethods(self):
        methods = ['fileno', 'flush', 'isatty', 'next', 'read', 'readinto',
                   'readline', 'readlines', 'seek', 'tell', 'truncate',
                   'write', 'xreadlines', '__iter__']
        if sys.platform.startswith('atheos'):
            methods.remove('truncate')

        self.f.close()

        for methodname in methods:
            method = getattr(self.f, methodname)
            # should raise on closed file
            self.assertRaises(ValueError, method)
        self.assertRaises(ValueError, self.f.writelines, [])

class OtherFileTests(unittest.TestCase):

    def testModeStrings(self):
        # check invalid mode strings
        for mode in ("", "aU", "wU+"):
            try:
                f = open(TESTFN, mode)
            except ValueError:
                pass
            else:
                f.close()
                self.fail('%r is an invalid file mode' % mode)

    def testStdin(self):
        # This causes the interpreter to exit on OSF1 v5.1.
        if sys.platform != 'osf1V5':
            self.assertRaises(IOError, sys.stdin.seek, -1)
        else:
            print >>sys.__stdout__, (
                '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
                ' Test manually.')
        self.assertRaises(IOError, sys.stdin.truncate)

    def testUnicodeOpen(self):
        # verify repr works for unicode too
        f = open(unicode(TESTFN), "w")
        self.assert_(repr(f).startswith("<open file u'" + TESTFN))
        f.close()
        os.unlink(TESTFN)

    def testBadModeArgument(self):
        # verify that we get a sensible error message for bad mode argument
        bad_mode = "qwerty"
        try:
            f = open(TESTFN, bad_mode)
        except ValueError, msg:
            if msg[0] != 0:
                s = str(msg)
                if s.find(TESTFN) != -1 or s.find(bad_mode) == -1:
                    self.fail("bad error message for invalid mode: %s" % s)
            # if msg[0] == 0, we're probably on Windows where there may be
            # no obvious way to discover why open() failed.
        else:
            f.close()
            self.fail("no error for invalid mode: %s" % bad_mode)

    def testSetBufferSize(self):
        # make sure that explicitly setting the buffer size doesn't cause
        # misbehaviour especially with repeated close() calls
        for s in (-1, 0, 1, 512):
            try:
                f = open(TESTFN, 'w', s)
                f.write(str(s))
                f.close()
                f.close()
                f = open(TESTFN, 'r', s)
                d = int(f.read())
                f.close()
                f.close()
            except IOError, msg:
                self.fail('error setting buffer size %d: %s' % (s, str(msg)))
            self.assertEquals(d, s)

    def testTruncateOnWindows(self):
        os.unlink(TESTFN)

        def bug801631():
            # SF bug <http://www.python.org/sf/801631>
            # "file.truncate fault on windows"
            f = open(TESTFN, 'wb')
            f.write('12345678901')   # 11 bytes
            f.close()

            f = open(TESTFN,'rb+')
            data = f.read(5)
            if data != '12345':
                self.fail("Read on file opened for update failed %r" % data)
            if f.tell() != 5:
                self.fail("File pos after read wrong %d" % f.tell())

            f.truncate()
            if f.tell() != 5:
                self.fail("File pos after ftruncate wrong %d" % f.tell())

            f.close()
            size = os.path.getsize(TESTFN)
            if size != 5:
                self.fail("File size after ftruncate wrong %d" % size)

        try:
            bug801631()
        finally:
            os.unlink(TESTFN)

    def testIteration(self):
        # Test the complex interaction when mixing file-iteration and the
        # various read* methods. Ostensibly, the mixture could just be tested
        # to work when it should work according to the Python language,
        # instead of fail when it should fail according to the current CPython
        # implementation.  People don't always program Python the way they
        # should, though, and the implemenation might change in subtle ways,
        # so we explicitly test for errors, too; the test will just have to
        # be updated when the implementation changes.
        dataoffset = 16384
        filler = "ham\n"
        assert not dataoffset % len(filler), \
            "dataoffset must be multiple of len(filler)"
        nchunks = dataoffset // len(filler)
        testlines = [
            "spam, spam and eggs\n",
            "eggs, spam, ham and spam\n",
            "saussages, spam, spam and eggs\n",
            "spam, ham, spam and eggs\n",
            "spam, spam, spam, spam, spam, ham, spam\n",
            "wonderful spaaaaaam.\n"
        ]
        methods = [("readline", ()), ("read", ()), ("readlines", ()),
                   ("readinto", (array("c", " "*100),))]

        try:
            # Prepare the testfile
            bag = open(TESTFN, "w")
            bag.write(filler * nchunks)
            bag.writelines(testlines)
            bag.close()
            # Test for appropriate errors mixing read* and iteration
            for methodname, args in methods:
                f = open(TESTFN)
                if f.next() != filler:
                    self.fail, "Broken testfile"
                meth = getattr(f, methodname)
                try:
                    meth(*args)
                except ValueError:
                    pass
                else:
                    self.fail("%s%r after next() didn't raise ValueError" %
                                     (methodname, args))
                f.close()

            # Test to see if harmless (by accident) mixing of read* and
            # iteration still works. This depends on the size of the internal
            # iteration buffer (currently 8192,) but we can test it in a
            # flexible manner.  Each line in the bag o' ham is 4 bytes
            # ("h", "a", "m", "\n"), so 4096 lines of that should get us
            # exactly on the buffer boundary for any power-of-2 buffersize
            # between 4 and 16384 (inclusive).
            f = open(TESTFN)
            for i in range(nchunks):
                f.next()
            testline = testlines.pop(0)
            try:
                line = f.readline()
            except ValueError:
                self.fail("readline() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if line != testline:
                self.fail("readline() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            testline = testlines.pop(0)
            buf = array("c", "\x00" * len(testline))
            try:
                f.readinto(buf)
            except ValueError:
                self.fail("readinto() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            line = buf.tostring()
            if line != testline:
                self.fail("readinto() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))

            testline = testlines.pop(0)
            try:
                line = f.read(len(testline))
            except ValueError:
                self.fail("read() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if line != testline:
                self.fail("read() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            try:
                lines = f.readlines()
            except ValueError:
                self.fail("readlines() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if lines != testlines:
                self.fail("readlines() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            # Reading after iteration hit EOF shouldn't hurt either
            f = open(TESTFN)
            try:
                for line in f:
                    pass
                try:
                    f.readline()
                    f.readinto(buf)
                    f.read()
                    f.readlines()
                except ValueError:
                    self.fail("read* failed after next() consumed file")
            finally:
                f.close()
        finally:
            os.unlink(TESTFN)

def test_main():
    # Historically, these tests have sloppy about removing TESTFN.  So get
    # rid of it no matter what.
    try:
        run_unittest(AutoFileTests, OtherFileTests)
    finally:
        if os.path.exists(TESTFN):
            os.unlink(TESTFN)

if __name__ == '__main__':
    test_main()
<commit_msg>Fix grammar and reflow<commit_after>import sys
import os
import unittest
from array import array
from weakref import proxy

from test.test_support import TESTFN, findfile, run_unittest
from UserList import UserList

class AutoFileTests(unittest.TestCase):
    # file tests for which a test file is automatically set up

    def setUp(self):
        self.f = open(TESTFN, 'wb')

    def tearDown(self):
        if self.f:
            self.f.close()
        os.remove(TESTFN)

    def testWeakRefs(self):
        # verify weak references
        p = proxy(self.f)
        p.write('teststring')
        self.assertEquals(self.f.tell(), p.tell())
        self.f.close()
        self.f = None
        self.assertRaises(ReferenceError, getattr, p, 'tell')

    def testAttributes(self):
        # verify expected attributes exist
        f = self.f
        softspace = f.softspace
        f.name     # merely shouldn't blow up
        f.mode     # ditto
        f.closed   # ditto

        # verify softspace is writable
        f.softspace = softspace    # merely shouldn't blow up

        # verify the others aren't
        for attr in 'name', 'mode', 'closed':
            self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops')

    def testReadinto(self):
        # verify readinto
        self.f.write('12')
        self.f.close()
        a = array('c', 'x'*10)
        self.f = open(TESTFN, 'rb')
        n = self.f.readinto(a)
        self.assertEquals('12', a.tostring()[:n])

    def testWritelinesUserList(self):
        # verify writelines with instance sequence
        l = UserList(['1', '2'])
        self.f.writelines(l)
        self.f.close()
        self.f = open(TESTFN, 'rb')
        buf = self.f.read()
        self.assertEquals(buf, '12')

    def testWritelinesIntegers(self):
        # verify writelines with integers
        self.assertRaises(TypeError, self.f.writelines, [1, 2, 3])

    def testWritelinesIntegersUserList(self):
        # verify writelines with integers in UserList
        l = UserList([1,2,3])
        self.assertRaises(TypeError, self.f.writelines, l)

    def testWritelinesNonString(self):
        # verify writelines with non-string object
        class NonString:
            pass

        self.assertRaises(TypeError, self.f.writelines,
                          [NonString(), NonString()])

    def testRepr(self):
        # verify repr works
        self.assert_(repr(self.f).startswith("<open file '" + TESTFN))

    def testErrors(self):
        f = self.f
        self.assertEquals(f.name, TESTFN)
        self.assert_(not f.isatty())
        self.assert_(not f.closed)

        self.assertRaises(TypeError, f.readinto, "")
        f.close()
        self.assert_(f.closed)

    def testMethods(self):
        methods = ['fileno', 'flush', 'isatty', 'next', 'read', 'readinto',
                   'readline', 'readlines', 'seek', 'tell', 'truncate',
                   'write', 'xreadlines', '__iter__']
        if sys.platform.startswith('atheos'):
            methods.remove('truncate')

        self.f.close()

        for methodname in methods:
            method = getattr(self.f, methodname)
            # should raise on closed file
            self.assertRaises(ValueError, method)
        self.assertRaises(ValueError, self.f.writelines, [])

class OtherFileTests(unittest.TestCase):

    def testModeStrings(self):
        # check invalid mode strings
        for mode in ("", "aU", "wU+"):
            try:
                f = open(TESTFN, mode)
            except ValueError:
                pass
            else:
                f.close()
                self.fail('%r is an invalid file mode' % mode)

    def testStdin(self):
        # This causes the interpreter to exit on OSF1 v5.1.
        if sys.platform != 'osf1V5':
            self.assertRaises(IOError, sys.stdin.seek, -1)
        else:
            print >>sys.__stdout__, (
                '  Skipping sys.stdin.seek(-1), it may crash the interpreter.'
                ' Test manually.')
        self.assertRaises(IOError, sys.stdin.truncate)

    def testUnicodeOpen(self):
        # verify repr works for unicode too
        f = open(unicode(TESTFN), "w")
        self.assert_(repr(f).startswith("<open file u'" + TESTFN))
        f.close()
        os.unlink(TESTFN)

    def testBadModeArgument(self):
        # verify that we get a sensible error message for bad mode argument
        bad_mode = "qwerty"
        try:
            f = open(TESTFN, bad_mode)
        except ValueError, msg:
            if msg[0] != 0:
                s = str(msg)
                if s.find(TESTFN) != -1 or s.find(bad_mode) == -1:
                    self.fail("bad error message for invalid mode: %s" % s)
            # if msg[0] == 0, we're probably on Windows where there may be
            # no obvious way to discover why open() failed.
        else:
            f.close()
            self.fail("no error for invalid mode: %s" % bad_mode)

    def testSetBufferSize(self):
        # make sure that explicitly setting the buffer size doesn't cause
        # misbehaviour especially with repeated close() calls
        for s in (-1, 0, 1, 512):
            try:
                f = open(TESTFN, 'w', s)
                f.write(str(s))
                f.close()
                f.close()
                f = open(TESTFN, 'r', s)
                d = int(f.read())
                f.close()
                f.close()
            except IOError, msg:
                self.fail('error setting buffer size %d: %s' % (s, str(msg)))
            self.assertEquals(d, s)

    def testTruncateOnWindows(self):
        os.unlink(TESTFN)

        def bug801631():
            # SF bug <http://www.python.org/sf/801631>
            # "file.truncate fault on windows"
            f = open(TESTFN, 'wb')
            f.write('12345678901')   # 11 bytes
            f.close()

            f = open(TESTFN,'rb+')
            data = f.read(5)
            if data != '12345':
                self.fail("Read on file opened for update failed %r" % data)
            if f.tell() != 5:
                self.fail("File pos after read wrong %d" % f.tell())

            f.truncate()
            if f.tell() != 5:
                self.fail("File pos after ftruncate wrong %d" % f.tell())

            f.close()
            size = os.path.getsize(TESTFN)
            if size != 5:
                self.fail("File size after ftruncate wrong %d" % size)

        try:
            bug801631()
        finally:
            os.unlink(TESTFN)

    def testIteration(self):
        # Test the complex interaction when mixing file-iteration and the
        # various read* methods. Ostensibly, the mixture could just be tested
        # to work when it should work according to the Python language,
        # instead of fail when it should fail according to the current CPython
        # implementation.  People don't always program Python the way they
        # should, though, and the implemenation might change in subtle ways,
        # so we explicitly test for errors, too; the test will just have to
        # be updated when the implementation changes.
        dataoffset = 16384
        filler = "ham\n"
        assert not dataoffset % len(filler), \
            "dataoffset must be multiple of len(filler)"
        nchunks = dataoffset // len(filler)
        testlines = [
            "spam, spam and eggs\n",
            "eggs, spam, ham and spam\n",
            "saussages, spam, spam and eggs\n",
            "spam, ham, spam and eggs\n",
            "spam, spam, spam, spam, spam, ham, spam\n",
            "wonderful spaaaaaam.\n"
        ]
        methods = [("readline", ()), ("read", ()), ("readlines", ()),
                   ("readinto", (array("c", " "*100),))]

        try:
            # Prepare the testfile
            bag = open(TESTFN, "w")
            bag.write(filler * nchunks)
            bag.writelines(testlines)
            bag.close()
            # Test for appropriate errors mixing read* and iteration
            for methodname, args in methods:
                f = open(TESTFN)
                if f.next() != filler:
                    self.fail, "Broken testfile"
                meth = getattr(f, methodname)
                try:
                    meth(*args)
                except ValueError:
                    pass
                else:
                    self.fail("%s%r after next() didn't raise ValueError" %
                                     (methodname, args))
                f.close()

            # Test to see if harmless (by accident) mixing of read* and
            # iteration still works. This depends on the size of the internal
            # iteration buffer (currently 8192,) but we can test it in a
            # flexible manner.  Each line in the bag o' ham is 4 bytes
            # ("h", "a", "m", "\n"), so 4096 lines of that should get us
            # exactly on the buffer boundary for any power-of-2 buffersize
            # between 4 and 16384 (inclusive).
            f = open(TESTFN)
            for i in range(nchunks):
                f.next()
            testline = testlines.pop(0)
            try:
                line = f.readline()
            except ValueError:
                self.fail("readline() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if line != testline:
                self.fail("readline() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            testline = testlines.pop(0)
            buf = array("c", "\x00" * len(testline))
            try:
                f.readinto(buf)
            except ValueError:
                self.fail("readinto() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            line = buf.tostring()
            if line != testline:
                self.fail("readinto() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))

            testline = testlines.pop(0)
            try:
                line = f.read(len(testline))
            except ValueError:
                self.fail("read() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if line != testline:
                self.fail("read() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            try:
                lines = f.readlines()
            except ValueError:
                self.fail("readlines() after next() with supposedly empty "
                          "iteration-buffer failed anyway")
            if lines != testlines:
                self.fail("readlines() after next() with empty buffer "
                          "failed. Got %r, expected %r" % (line, testline))
            # Reading after iteration hit EOF shouldn't hurt either
            f = open(TESTFN)
            try:
                for line in f:
                    pass
                try:
                    f.readline()
                    f.readinto(buf)
                    f.read()
                    f.readlines()
                except ValueError:
                    self.fail("read* failed after next() consumed file")
            finally:
                f.close()
        finally:
            os.unlink(TESTFN)

def test_main():
    # Historically, these tests have been sloppy about removing TESTFN.
    # So get rid of it no matter what.
    try:
        run_unittest(AutoFileTests, OtherFileTests)
    finally:
        if os.path.exists(TESTFN):
            os.unlink(TESTFN)

if __name__ == '__main__':
    test_main()
<|endoftext|>


<commit_before>#! /usr/bin/env python3
"""Brute-force test script: test libpqxx against many compilers etc.

This script makes no changes in the source tree; all builds happen in
temporary directories.

To make this possible, you may need to run "make distclean" in the
source tree.  The configure script will refuse to configure otherwise.
"""

# Without this, pocketlint does not yet understand the print function.
from __future__ import print_function

from argparse import ArgumentParser
from contextlib import contextmanager
from os import (
    cpu_count,
    getcwd,
    )
import os.path
from shutil import rmtree
from subprocess import (
    CalledProcessError,
    check_call,
    DEVNULL,
    )
from sys import (
    stderr,
    stdout,
    )
from tempfile import mkdtemp
from textwrap import dedent

CPUS = cpu_count()

GCC_VERSIONS = list(range(7, 12))
GCC = ['g++-%d' % ver for ver in GCC_VERSIONS]
CLANG_VERSIONS = list(range(7, 12))
CLANG = ['clang++-6.0'] + ['clang++-%d' % ver for ver in CLANG_VERSIONS]
CXX = GCC + CLANG

STDLIB = (
    '',
    '-stdlib=libc++',
    )

OPT = ('-O0', '-O3')

LINK = {
    'static': ['--enable-static', '--disable-dynamic'],
    'dynamic': ['--disable-static', '--enable-dynamic'],
}

DEBUG = {
    'plain': [],
    'audit': ['--enable-audit'],
    'maintainer': ['--enable-maintainer-mode'],
    'full': ['--enable-audit', '--enable-maintainer-mode'],
}

# CMake "generators."  Maps a value for cmake's -G option (None for default) to
# a command line to run.
#
# I prefer Ninja if available, because it's fast.  But hey, the default will
# work.
#
# Maps the name of the generator (as used with cmake's -G option) to the
# actual command line needed to do the build.
CMAKE_GENERATORS = {
    'Ninja': ['ninja'],
    None: ['make', '-j%d' % CPUS],
}

class Fail(Exception):
    """A known, well-handled exception.  Doesn't need a traceback."""

def run(cmd, output, cwd=None):
    """Run a command, write output to file-like object."""
    command_line = ' '.join(cmd)
    output.write("%s\n\n" % command_line)
    check_call(cmd, stdout=output, stderr=output, cwd=cwd)

def report(output, message):
    """Report a message to output, and standard output."""
    print(message)
    output.write('\n\n')
    output.write(message)
    output.write('\n')

def report_success(output):
    """Report a succeeded build."""
    report(output, "OK")
    return True

def report_failure(output, error):
    """Report a failed build."""
    report(output, "FAIL: %s" % error)
    return False

def file_contains(path, text):
    """Does the file at path contain text?"""
    with open(path) as stream:
        for line in stream:
            if text in line:
                return True
    return False

def build(configure, output):
    """Perform a full configure-based build."""
    with tmp_dir() as work_dir:
        try:
            run(configure, output, cwd=work_dir)
        except CalledProcessError:
            output.flush()
            if file_contains(output.name, "make distclean"):
                # Looks like that special "configure" error where the source
                # tree is already configured.  Tell the user about this special
                # case without requiring them to dig deeper.
                raise Fail(
                    "Configure failed.  "
                    "Did you remember to 'make distclean' the source tree?")
            return report_failure("configure failed.")

        try:
            run(['make', '-j%d' % CPUS], output, cwd=work_dir)
            run(['make', '-j%d' % CPUS, 'check'], output, cwd=work_dir)
        except CalledProcessError as err:
            return report_failure(output, err)
        else:
            return report_success(output)

@contextmanager
def tmp_dir():
    """Create a temporary directory, and clean it up again."""
    tmp = mkdtemp()
    try:
        yield tmp
    finally:
        rmtree(tmp)

def write_check_code(work_dir):
    """Write a simple C++ program so we can tesst whether we can compile it.

    Returns the file's full path.
    """
    path = os.path.join(work_dir, "check.cxx")
    with open(path, 'w') as source:
        source.write(dedent("""\
            #include <iostream>
            int main()
            {
                std::cout << "Hello world." << std::endl;
            }
            """))

    return path

def check_compiler(work_dir, cxx, stdlib, check, verbose=False):
    """Is the given compiler combo available?"""
    err_file = os.path.join(work_dir, 'stderr.log')
    if verbose:
        err_output = open(err_file, 'w')
    else:
        err_output = DEVNULL
    try:
        command = [cxx, 'check.cxx']
        if stdlib != '':
            command.append(stdlib)
        check_call(command, cwd=work_dir, stderr=err_output)
    except (OSError, CalledProcessError):
        if verbose:
            with open(err_file) as errors:
                stdout.write(errors.read())
            print("Can't build with '%s %s'.  Skipping." % (cxx, stdlib))
        return False
    else:
        return True

def check_compilers(compilers, stdlibs, verbose=False):
    """Check which compiler configurations are viable."""
    with tmp_dir() as work_dir:
        check = write_check_code(work_dir)
        return [
            (cxx, stdlib)
            for stdlib in stdlibs
            for cxx in compilers
            if check_compiler(
                work_dir, cxx, stdlib, check=check, verbose=verbose)
        ]

class Config:
    """A combination of build options."""
    def __init__(self, cxx, opt, stdlib, link, link_opts, debug, debug_opts):
        self.cxx = cxx
        self.opt = opt
        self.stdlib = stdlib
        self.link = link
        self.link_opts = link_opts
        self.debug = debug
        self.debug_opts = debug_opts

def try_build(logs_dir, config):
    """Attempt to build in a given configuration."""
    log = os.path.join(
        logs_dir, 'build-%s.out' % '_'.join([
            config.cxx, config.opt, config.stdlib, config.link, config.debug]))
    print("%s... " % log, end='', flush=True)
    configure = [
        os.path.join(getcwd(), "configure"),
        "CXX=%s" % config.cxx,
        ]

    if config.stdlib == '':
        configure += [
            "CXXFLAGS=%s" % config.opt,
            ]
    else:
        configure += [
            "CXXFLAGS=%s %s" % (config.opt, config.stdlib),
            "LDFLAGS=%s" % config.stdlib,
            ]

    configure += [
        "--disable-documentation",
        ] + config.link_opts + config.debug_opts

    with open(log, 'w') as output:
        build(configure, output)

def prepare_cmake(work_dir, output, verbose=False):
    """Set up a CMake build dir, ready to build.

    Returns the directory, and if successful, the command you need to run in
    order to do the build.
    """
    if verbose:
        print("\nLooking for CMake generator.")
    source_dir = getcwd()
    for gen, cmd in CMAKE_GENERATORS.items():
        name = gen or '<default>'
        cmake = ['cmake', source_dir]
        if gen is not None:
            cmake += ['-G', gen]
        if verbose:
            print("Trying CMake with %s." % name)

        try:
            run(cmake, output=output, cwd=work_dir)
        except FileNotFoundError:
            print("No cmake found.  Skipping.")
            return None
        except CalledProcessError:
            print("CMake generator %s is not available.  Skipping." % name)
        else:
            return cmd

    return None

def build_with_cmake(logs_dir, verbose=False):
    """Build using CMake.  Use the first generator that works."""
    log = os.path.join(logs_dir, "build-cmake.out")
    print("%s... " % log, end='', flush=True)
    with tmp_dir() as work_dir, open(log, 'w') as output:
        try:
            command = prepare_cmake(work_dir, output, verbose)
            if command is None:
                return report_fail(
                    output,
                    "No CMake generators found.  Skipping CMake build.")
            else:
                if verbose:
                    print("Building with CMake and '%s'." % ' '.join(command))
                run(command, output, cwd=work_dir)
                run(['test/runner'], output, cwd=work_dir)
        except CalledProcessError:
            return report_fail(output, "CMake build failed.")
        else:
            return report_success(output)

def parse_args():
    """Parse command-line arguments."""
    parser = ArgumentParser(description=__doc__)
    parser.add_argument('--verbose', '-v', action='store_true')
    parser.add_argument(
        '--compilers', '-c', default=','.join(CXX),
        help="Compilers, separated by commas.  Default is %(default)s.")
    parser.add_argument(
        '--optimize', '-O', default=','.join(OPT),
        help=(
            "Alternative optimisation options, separated by commas.  "
            "Default is %(default)s."))
    parser.add_argument(
        '--stdlibs', '-L', default=','.join(STDLIB),
        help=(
            "Comma-separated options for choosing standard library.  "
            "Defaults to %(default)s."))
    parser.add_argument(
        '--logs', '-l', default='.', metavar='DIRECTORY',
        help="Write build logs to DIRECTORY.")
    parser.add_argument(
        '--jobs', '-j', default=CPUS, metavar='CPUS',
        help=(
            "When running 'make', run up to CPUS concurrent processes.  "
            "Defaults to %(default)s."))
    parser.add_argument(
        '--minimal', '-m', action='store_true',
        help="Make it as short a run as possible.  For testing this script.")
    return parser.parse_args()

def main(args):
    """Do it all."""
    if not os.path.isdir(args.logs):
        raise Fail("Logs location '%s' is not a directory." % args.logs)

    if args.verbose:
        print("\nChecking available compilers.")

    compiler_candidates = args.compilers.split(',')
    compilers = check_compilers(
        compiler_candidates, args.stdlibs.split(','),
        verbose=args.verbose)
    if list(compilers) == []:
        raise Fail(
            "Did not find any viable compilers.  Tried: %s."
            % ', '.join(compiler_candidates))

    opt_levels = args.optimize.split(',')
    link_types = LINK.items()
    debug_mixes = DEBUG.items()

    if args.minimal:
        compilers = compilers[:1]
        opt_levels = opt_levels[:1]
        link_types = list(link_types)[:1]
        debug_mixes = list(debug_mixes)[:1]

    print("\nStarting builds.")
    for opt in sorted(opt_levels):
        for link, link_opts in sorted(link_types):
            for debug, debug_opts in sorted(debug_mixes):
                for cxx, stdlib in compilers:
                    config = Config(
                        opt=opt, link=link, link_opts=link_opts, debug=debug,
                        debug_opts=debug_opts, cxx=cxx, stdlib=stdlib)
                    try_build(logs_dir=args.logs, config=config)

    if args.verbose:
        print("\nBuilding with CMake.")
    build_with_cmake(args.logs, verbose=args.verbose)

if __name__ == '__main__':
    try:
        main(parse_args())
    except Fail as error:
        stderr.write("%s\n" % error)
<commit_msg>`test_all.py`: Turn builds into objects.<commit_after>#! /usr/bin/env python3
"""Brute-force test script: test libpqxx against many compilers etc.

This script makes no changes in the source tree; all builds happen in
temporary directories.

To make this possible, you may need to run "make distclean" in the
source tree.  The configure script will refuse to configure otherwise.
"""

# Without this, pocketlint does not yet understand the print function.
from __future__ import print_function

from abc import (
    ABCMeta,
    abstractmethod,
    )
from argparse import ArgumentParser
from contextlib import contextmanager
from os import (
    cpu_count,
    getcwd,
    )
import os.path
from shutil import rmtree
from subprocess import (
    CalledProcessError,
    check_call,
    DEVNULL,
    )
from sys import (
    stderr,
    stdout,
    )
from tempfile import mkdtemp
from textwrap import dedent

CPUS = cpu_count()

GCC_VERSIONS = list(range(7, 12))
GCC = ['g++-%d' % ver for ver in GCC_VERSIONS]
CLANG_VERSIONS = list(range(7, 12))
CLANG = ['clang++-6.0'] + ['clang++-%d' % ver for ver in CLANG_VERSIONS]
CXX = GCC + CLANG

STDLIB = (
    '',
    '-stdlib=libc++',
    )

OPT = ('-O0', '-O3')

LINK = {
    'static': ['--enable-static', '--disable-dynamic'],
    'dynamic': ['--disable-static', '--enable-dynamic'],
}

DEBUG = {
    'plain': [],
    'audit': ['--enable-audit'],
    'maintainer': ['--enable-maintainer-mode'],
    'full': ['--enable-audit', '--enable-maintainer-mode'],
}

# CMake "generators."  Maps a value for cmake's -G option (None for default) to
# a command line to run.
#
# I prefer Ninja if available, because it's fast.  But hey, the default will
# work.
#
# Maps the name of the generator (as used with cmake's -G option) to the
# actual command line needed to do the build.
CMAKE_GENERATORS = {
    'Ninja': ['ninja'],
    None: ['make', '-j%d' % CPUS],
}

class Fail(Exception):
    """A known, well-handled exception.  Doesn't need a traceback."""

class Skip(Exception):
    """"We're not doing this build.  It's not an error though."""

def run(cmd, output, cwd=None):
    """Run a command, write output to file-like object."""
    command_line = ' '.join(cmd)
    output.write("%s\n\n" % command_line)
    check_call(cmd, stdout=output, stderr=output, cwd=cwd)

def report(output, message):
    """Report a message to output, and standard output."""
    print(message, flush=True)
    output.write('\n\n')
    output.write(message)
    output.write('\n')

def report_success(output):
    """Report a succeeded build."""
    report(output, "OK")
    return True

def report_failure(output, error):
    """Report a failed build."""
    report(output, "FAIL: %s" % error)
    return False

def file_contains(path, text):
    """Does the file at path contain text?"""
    with open(path) as stream:
        for line in stream:
            if text in line:
                return True
    return False

@contextmanager
def tmp_dir():
    """Create a temporary directory, and clean it up again."""
    tmp = mkdtemp()
    try:
        yield tmp
    finally:
        rmtree(tmp)

def write_check_code(work_dir):
    """Write a simple C++ program so we can tesst whether we can compile it.

    Returns the file's full path.
    """
    path = os.path.join(work_dir, "check.cxx")
    with open(path, 'w') as source:
        source.write(dedent("""\
            #include <iostream>
            int main()
            {
                std::cout << "Hello world." << std::endl;
            }
            """))

    return path

def check_compiler(work_dir, cxx, stdlib, check, verbose=False):
    """Is the given compiler combo available?"""
    err_file = os.path.join(work_dir, 'stderr.log')
    if verbose:
        err_output = open(err_file, 'w')
    else:
        err_output = DEVNULL
    try:
        command = [cxx, check]
        if stdlib != '':
            command.append(stdlib)
        check_call(command, cwd=work_dir, stderr=err_output)
    except (OSError, CalledProcessError):
        if verbose:
            with open(err_file) as errors:
                stdout.write(errors.read())
            print("Can't build with '%s %s'.  Skipping." % (cxx, stdlib))
        return False
    else:
        return True

def check_compilers(compilers, stdlibs, verbose=False):
    """Check which compiler configurations are viable."""
    with tmp_dir() as work_dir:
        check = write_check_code(work_dir)
        return [
            (cxx, stdlib)
            for stdlib in stdlibs
            for cxx in compilers
            if check_compiler(
                work_dir, cxx, stdlib, check=check, verbose=verbose)
        ]

class Config:
    """Configuration for a build."""
    __metaclass__ = ABCMeta

    @abstractmethod
    def name(self):
        """Return an identifier for this build configuration."""

    def make_log_name(self):
        """Compose log file name for this build."""
        return "build-%s.out" % self.name()

class Build:
    """A pending or ondoing build, in its own directory.

    Each step returns True for Success, or False for failure.
    """
    def __init__(self, logs_dir, config=None):
        self.config = config
        self.log = open(os.path.join(logs_dir, config.make_log_name()), 'w')
        self.work_dir_context = tmp_dir()
        self.work_dir = self.work_dir_context.__enter__()

    def report(self, stage):
        """Print a message saying what we're about to do."""
        print(
            "%s - %s... " % (self.config.make_log_name(), stage),
            end='', flush=True)

    def clean_up(self):
        """Delete the build tree."""
        self.work_dir_context.__exit__(None, None, None)
        self.log.close()

    @abstractmethod
    def configure(self):
        """Prepare for a build.

        This step is basically single-threaded.
        """

    @abstractmethod
    def build(self):
        """Build the code, including the tests.  Don't run tests though."""

    def test(self):
        """Run tests."""
        try:
            run(
                [os.path.join(os.path.curdir, 'test', 'runner')], self.log,
                cwd=self.work_dir)
        except CalledProcessError:
            return report_failure(self.log, "Tests failed.")
        else:
            return report_success(self.log)

class AutotoolsConfig(Config):
    """A combination of build options for the "configure" script."""
    def __init__(self, cxx, opt, stdlib, link, link_opts, debug, debug_opts):
        self.cxx = cxx
        self.opt = opt
        self.stdlib = stdlib
        self.link = link
        self.link_opts = link_opts
        self.debug = debug
        self.debug_opts = debug_opts

    def name(self):
        return '_'.join([
            self.cxx, self.opt, self.stdlib, self.link, self.debug])

class AutotoolsBuild(Build):
    """Build using the "configure" script."""
    __metaclass__ = ABCMeta

    def configure(self):
        configure = [
            os.path.join(getcwd(), "configure"),
            "CXX=%s" % self.config.cxx,
            ]

        if self.config.stdlib == '':
            configure += [
                "CXXFLAGS=%s" % self.config.opt,
            ]
        else:
            configure += [
                "CXXFLAGS=%s %s" % (self.config.opt, self.config.stdlib),
                "LDFLAGS=%s" % self.config.stdlib,
                ]

        configure += [
            "--disable-documentation",
            ] + self.config.link_opts + self.config.debug_opts

        try:
            run(configure, self.log, cwd=self.work_dir)
        except CalledProcessError:
            self.log.flush()
            if file_contains(self.log.name, "make distclean"):
                # Looks like that special "configure" error where the source
                # tree is already configured.  Tell the user about this
                # special case without requiring them to dig deeper.
                raise Fail(
                    "Configure failed.  "
                    "Did you remember to 'make distclean' the source tree?")
            return report_failure(self.log, "configure failed.")
        else:
            return report_success(self.log)

    def build(self):
        try:
            run(['make', '-j%d' % CPUS], self.log, cwd=self.work_dir)
            # Passing "TESTS=" like this will suppress the actual running of
            # the tests.  We do that in the nest stage.
            run(
                ['make', '-j%d' % CPUS, 'check', 'TESTS='],
                self.log, cwd=self.work_dir)
        except CalledProcessError as err:
            return report_failure(self.log, err)
        else:
            return report_success(self.log)

class CMakeConfig(Config):
    """Configuration for a CMake build."""
    def __init__(self):
        # The CMakeBuild figures this out in the configure stage.
        self.command = None

    def name(self):
        return "cmake"

class CMakeBuild(Build):
    """Build using CMake.

    Ignores the config for now.
    """
    __metaclass__ = ABCMeta

    def configure(self):
        source_dir = getcwd()
        for gen, cmd in CMAKE_GENERATORS.items():
            name = gen or '<default>'
            cmake = ['cmake', source_dir]
            if gen is not None:
                cmake += ['-G', gen]

            try:
                run(cmake, output=self.log, cwd=self.work_dir)
            except FileNotFoundError:
                raise Skip("No cmake found.")
            except CalledProcessError:
                print(
                    "CMake generator %s is not available.  Skipping." % name)
            else:
                self.config.command = cmd
                return report_success(self.log)

        raise Skip("Did not find any working CMake generators.")

    def build(self):
        print("%s... " % self.log.name, end='', flush=True)
        try:
            run(self.config.command, self.log, cwd=self.work_dir)
            run(['test/runner'], self.log, cwd=self.work_dir)
            return report_success(self.log)
        except CalledProcessError:
            return report_failure(self.log, "CMake build failed.")
        else:
            return report_success(self.log)

def run_step(builds, step_name, step):
    """Run `build(step) for each build in builds.  Return passing builds."""
    succeeded = []
    for build in builds:
        try:
            build.report(step_name)
            if step(build):
                succeeded.append(build)
        except Skip as error:
            print("Skipping: %s" % error)
    return succeeded

def parse_args():
    """Parse command-line arguments."""
    parser = ArgumentParser(description=__doc__)
    parser.add_argument('--verbose', '-v', action='store_true')
    parser.add_argument(
        '--compilers', '-c', default=','.join(CXX),
        help="Compilers, separated by commas.  Default is %(default)s.")
    parser.add_argument(
        '--optimize', '-O', default=','.join(OPT),
        help=(
            "Alternative optimisation options, separated by commas.  "
            "Default is %(default)s."))
    parser.add_argument(
        '--stdlibs', '-L', default=','.join(STDLIB),
        help=(
            "Comma-separated options for choosing standard library.  "
            "Defaults to %(default)s."))
    parser.add_argument(
        '--logs', '-l', default='.', metavar='DIRECTORY',
        help="Write build logs to DIRECTORY.")
    parser.add_argument(
        '--jobs', '-j', default=CPUS, metavar='CPUS',
        help=(
            "When running 'make', run up to CPUS concurrent processes.  "
            "Defaults to %(default)s."))
    parser.add_argument(
        '--minimal', '-m', action='store_true',
        help="Make it as short a run as possible.  For testing this script.")
    return parser.parse_args()

def main(args):
    """Do it all."""
    if not os.path.isdir(args.logs):
        raise Fail("Logs location '%s' is not a directory." % args.logs)

    if args.verbose:
        print("\nChecking available compilers.")

    compiler_candidates = args.compilers.split(',')
    compilers = check_compilers(
        compiler_candidates, args.stdlibs.split(','),
        verbose=args.verbose)
    if list(compilers) == []:
        raise Fail(
            "Did not find any viable compilers.  Tried: %s."
            % ', '.join(compiler_candidates))

    opt_levels = args.optimize.split(',')
    link_types = LINK.items()
    debug_mixes = DEBUG.items()

    if args.minimal:
        compilers = compilers[:1]
        opt_levels = opt_levels[:1]
        link_types = list(link_types)[:1]
        debug_mixes = list(debug_mixes)[:1]

    if args.verbose:
        print("\nStarting builds.")

    builds = [
        AutotoolsBuild(
            args.logs,
            AutotoolsConfig(
                opt=opt, link=link, link_opts=link_opts, debug=debug,
                debug_opts=debug_opts, cxx=cxx, stdlib=stdlib))
        for opt in sorted(opt_levels)
        for link, link_opts in sorted(link_types)
        for debug, debug_opts in sorted(debug_mixes)
        for cxx, stdlib in compilers
    ] + [
        CMakeBuild(args.logs, CMakeConfig())
    ]

    try:
        to_build = run_step(
            builds, "configure", lambda build: build.configure(),
            verbose=args.verbose)
        to_test = run_step(
            to_build, "build", lambda build: build.build(),
            verbose=args.verbose)
        done = run_step(
            to_test, "test", lambda build: build.test(),
            verbose=args.verbose)
    finally:
        for build in builds:
            build.clean_up()

    print("Passed %d out of %d builds." % (len(done), len(builds)))

if __name__ == '__main__':
    try:
        main(parse_args())
    except Fail as failure:
        stderr.write("%s\n" % failure)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""The app module, containing the app factory function."""
from __future__ import absolute_import, unicode_literals
import dateutil
import os

import arrow
from flask import Flask, render_template, current_app
from jinja2 import is_undefined
from path import path as ipath
from werkzeug.utils import import_string

from .settings import DevelopmentConfig

class NoContextProcessorException(Exception):
  """Raise if context processor wasn't found."""
  pass

class NoBlueprintException(Exception):
  """Raise if blueprint wasn't found."""
  pass

class NoExtensionException(Exception):
  """Raise if extension wasn't found."""
  pass

def get_imported_stuff_by_path(path):
  """Helper method to import modules from a string path."""
  module_name, object_name = path.rsplit('.', 1)
  module = import_string(module_name)

  return module, object_name

class AppFactory(object):

  """Create a Flask app (Flask Application Factory)."""

  def __init__(self):
    super(AppFactory, self).__init__()

    self.app_config = None
    self.app = None

  def __call__(self, app_name=None, config=None, config_obj=None, **kwargs):
    # initialize the application
    self.app_config = config
    self.app = Flask(app_name or DevelopmentConfig.PROJECT,
                     instance_relative_config=True, **kwargs)

    self._configure_app(config_obj=config_obj)
    self._bind_extensions()
    self._register_blueprints()
    self._configure_logging()
    self._configure_template_filters()
    self._configure_error_handlers()

    return self.app

  def _configure_app(self, config_obj=None):
    """Configure the app in different ways."""
    # http://flask.pocoo.org/docs/api/#configuration
    self.app.config.from_object(config_obj or DevelopmentConfig)

    # user custom config
    # http://flask.pocoo.org/docs/config/#instance-folders
    default_config = os.path.join(self.app.instance_path,
                                  "{}.cfg".format(self.app.name))
    self.app.config.from_pyfile(self.app_config or default_config, silent=True)

  def _bind_extensions(self):
    """Bind extensions to the app dynamically."""
    for ext_path in self.app.config.get('EXTENSIONS', []):
      module, object_name = get_imported_stuff_by_path(ext_path)

      if not hasattr(module, object_name):
        raise NoExtensionException("No {} extension found".format(object_name))

      extension = getattr(module, object_name)

      if getattr(extension, 'init_app', False):
        extension.init_app(self.app)

      else:
        extension(self.app)

  def _register_blueprints(self):
    """Configure blueprints in views."""
    for blueprint_path, url_prefix in self.app.config.get('BLUEPRINTS', []):
      module, object_name = get_imported_stuff_by_path(blueprint_path)

      if hasattr(module, object_name):
        self.app.register_blueprint(getattr(module, object_name),
                                    url_prefix=url_prefix)

      else:
        raise NoBlueprintException("No {} blueprint found".format(object_name))

  def _register_context_processors(self):
    """Register extra contexts for Jinja templates."""
    for processor_path in self.app.config.get('CONTEXT_PROCESSORS', []):
      module, object_name = get_imported_stuff_by_path(processor_path)

      if hasattr(module, object_name):
        self.app.context_processor(getattr(module, object_name))

      else:
        raise NoContextProcessorException("No {} context processor found"
                                          .format(object_name))

  def _configure_logging(self):
    """Configure file(info) and email(error) logging"""
    if self.app.debug or self.app.testing:
      # Skip debug and test mode; just check standard output
      return

    log_folder = self.app.config.get('LOG_FOLDER')
    if log_folder is None:
      default_log_folder = os.path.join(self.app.instance_path, 'logs')
      self.app.config['LOG_FOLDER'] = default_log_folder

    # make sure that all folders are in place
    ipath(self.app.config['LOG_FOLDER']).makedirs_p()

    import logging
    from .log import TlsSMTPHandler

    # Set info level on logger which might be overwritten by handlers
    # Suppress DEBUG messages
    self.app.logger.setLevel(logging.INFO)

    log_file_name = "{}.log".format(self.app.name)
    log_file = os.path.join(self.app.config['LOG_FOLDER'], log_file_name)

    info_log_handler = logging.handlers.RotatingFileHandler(
      log_file, maxBytes=100000, backupCount=10)
    info_log_handler.setLevel(logging.INFO)
    info_log_handler.setFormatter(logging.Formatter(
      '%(asctime)s - %(name)s - %(levelname)s: %(message)s '
      '[in %(pathname)s:%(lineno)d]')
    )
    self.app.logger.addHandler(info_log_handler)

    # also write default Weekzeug log (INFO) to the main log-file
    werkzeug_log = logging.getLogger('werkzeug')
    werkzeug_log.setLevel(logging.INFO)
    werkzeug_log.addHandler(info_log_handler)

    mail_handler = TlsSMTPHandler(
        mailhost=self.app.config['MAIL_SERVER'],
        fromaddr=self.app.config['MAIL_USERNAME'],
        toaddrs=self.app.config['ADMINS'],
        subject="O_ops... {} failed!".format(self.app.name),
        credentials=(self.app.config['MAIL_USERNAME'],
                     self.app.config['MAIL_PASSWORD'])
    )
    mail_handler.setLevel(logging.ERROR)
    mail_handler.setFormatter(logging.Formatter(
      '%(asctime)s - %(name)s - %(levelname)s: %(message)s '
      '[in %(pathname)s:%(lineno)d]')
    )
    self.app.logger.addHandler(mail_handler)
    werkzeug_log.addHandler(mail_handler)

  def _configure_error_handlers(self):
    """Configure error handlers to the corresponding error pages."""
    @self.app.errorhandler(403)
    def forbidden_page(error):
      """Render the forbidden error page on 403 errors."""
      return render_template('errors/forbidden_page.html', error=error), 403

    @self.app.errorhandler(404)
    def page_not_found(error):
      """Render the page not found page on 404 errors."""
      return render_template('errors/page_not_found.html', error=error), 404

    @self.app.errorhandler(500)
    def server_error_page(error):
      """Render the server error page on 500 errors."""
      return render_template('errors/server_error.html', error=error), 500

  def _configure_template_filters(self):
    """Configure custom Jinja2 template filters."""
    @self.app.template_filter()
    def human_date(value):
      time_zone = current_app.config['TIME_ZONE']
      return (arrow.get(value)
                   .replace(tzinfo=dateutil.tz.gettz(time_zone))
                   .humanize())

    @self.app.template_filter()
    def format_date(value, format="%Y-%m-%d"):
      return value.strftime(format)

    @self.app.template_filter()
    def initials(value):
      parts = value.split(' ')
      letters = (part[0] for part in parts)
      return ''.join(letters)

    @self.app.template_filter()
    def client_filter(value):
      """Pass through variable in Jinja2 to client side template.

      The filter tells Jinja2 that a variable is for a client side
      template engine.

      If the variable is undefined, its name will be used in the
      client side template, otherwise, its content will be used.
      """

      if is_undefined(value):
          return '{{{{{}}}}}'.format(value._undefined_name)
      if type(value) is bool:
          value = repr(value).lower()
      return '{{{{{}}}}}'.format(value)
<commit_msg>simplify logging (STDERR) and let wrappers take care of it<commit_after># -*- coding: utf-8 -*-
"""The app module, containing the app factory function."""
from __future__ import absolute_import, unicode_literals
import dateutil
import os

import arrow
from flask import Flask, render_template, current_app
from jinja2 import is_undefined
from path import path as ipath
from werkzeug.utils import import_string

from .settings import DevelopmentConfig

class NoContextProcessorException(Exception):
  """Raise if context processor wasn't found."""
  pass

class NoBlueprintException(Exception):
  """Raise if blueprint wasn't found."""
  pass

class NoExtensionException(Exception):
  """Raise if extension wasn't found."""
  pass

def get_imported_stuff_by_path(path):
  """Helper method to import modules from a string path."""
  module_name, object_name = path.rsplit('.', 1)
  module = import_string(module_name)

  return module, object_name

class AppFactory(object):

  """Create a Flask app (Flask Application Factory)."""

  def __init__(self):
    super(AppFactory, self).__init__()

    self.app_config = None
    self.app = None

  def __call__(self, app_name=None, config=None, config_obj=None, **kwargs):
    # initialize the application
    self.app_config = config
    self.app = Flask(app_name or DevelopmentConfig.PROJECT,
                     instance_relative_config=True, **kwargs)

    self._configure_app(config_obj=config_obj)
    self._bind_extensions()
    self._register_blueprints()
    self._configure_logging()
    self._configure_template_filters()
    self._configure_error_handlers()

    return self.app

  def _configure_app(self, config_obj=None):
    """Configure the app in different ways."""
    # http://flask.pocoo.org/docs/api/#configuration
    self.app.config.from_object(config_obj or DevelopmentConfig)

    # user custom config
    # http://flask.pocoo.org/docs/config/#instance-folders
    default_config = os.path.join(self.app.instance_path,
                                  "{}.cfg".format(self.app.name))
    self.app.config.from_pyfile(self.app_config or default_config, silent=True)

  def _bind_extensions(self):
    """Bind extensions to the app dynamically."""
    for ext_path in self.app.config.get('EXTENSIONS', []):
      module, object_name = get_imported_stuff_by_path(ext_path)

      if not hasattr(module, object_name):
        raise NoExtensionException("No {} extension found".format(object_name))

      extension = getattr(module, object_name)

      if getattr(extension, 'init_app', False):
        extension.init_app(self.app)

      else:
        extension(self.app)

  def _register_blueprints(self):
    """Configure blueprints in views."""
    for blueprint_path, url_prefix in self.app.config.get('BLUEPRINTS', []):
      module, object_name = get_imported_stuff_by_path(blueprint_path)

      if hasattr(module, object_name):
        self.app.register_blueprint(getattr(module, object_name),
                                    url_prefix=url_prefix)

      else:
        raise NoBlueprintException("No {} blueprint found".format(object_name))

  def _register_context_processors(self):
    """Register extra contexts for Jinja templates."""
    for processor_path in self.app.config.get('CONTEXT_PROCESSORS', []):
      module, object_name = get_imported_stuff_by_path(processor_path)

      if hasattr(module, object_name):
        self.app.context_processor(getattr(module, object_name))

      else:
        raise NoContextProcessorException("No {} context processor found"
                                          .format(object_name))

  def _configure_logging(self):
    """Configure file(info) and email(error) logging"""
    if self.app.debug or self.app.testing:
      # Skip debug and test mode; just check standard output
      return

    import logging
    from .log import TlsSMTPHandler

    # Set info level on logger which might be overwritten by handlers
    # Suppress DEBUG messages
    self.app.logger.setLevel(logging.INFO)

    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.INFO)
    stream_handler.setFormatter(logging.Formatter(
      '%(asctime)s - %(name)s - %(levelname)s: %(message)s '
      '[in %(pathname)s:%(lineno)d]')
    )
    self.app.logger.addHandler(stream_handler)

    # also write default Weekzeug log (INFO) to the main log-file
    # note: this is only relevant when not running behind gunicorn
    werkzeug_log = logging.getLogger('werkzeug')
    werkzeug_log.setLevel(logging.INFO)
    werkzeug_log.addHandler(stream_handler)

    mail_handler = TlsSMTPHandler(
        mailhost=self.app.config['MAIL_SERVER'],
        fromaddr=self.app.config['MAIL_USERNAME'],
        toaddrs=self.app.config['ADMINS'],
        subject="O_ops... {} failed!".format(self.app.name),
        credentials=(self.app.config['MAIL_USERNAME'],
                     self.app.config['MAIL_PASSWORD'])
    )
    mail_handler.setLevel(logging.ERROR)
    mail_handler.setFormatter(logging.Formatter(
      '%(asctime)s - %(name)s - %(levelname)s: %(message)s '
      '[in %(pathname)s:%(lineno)d]')
    )
    self.app.logger.addHandler(mail_handler)
    werkzeug_log.addHandler(mail_handler)

  def _configure_error_handlers(self):
    """Configure error handlers to the corresponding error pages."""
    @self.app.errorhandler(403)
    def forbidden_page(error):
      """Render the forbidden error page on 403 errors."""
      return render_template('errors/forbidden_page.html', error=error), 403

    @self.app.errorhandler(404)
    def page_not_found(error):
      """Render the page not found page on 404 errors."""
      return render_template('errors/page_not_found.html', error=error), 404

    @self.app.errorhandler(500)
    def server_error_page(error):
      """Render the server error page on 500 errors."""
      return render_template('errors/server_error.html', error=error), 500

  def _configure_template_filters(self):
    """Configure custom Jinja2 template filters."""
    @self.app.template_filter()
    def human_date(value):
      time_zone = current_app.config['TIME_ZONE']
      return (arrow.get(value)
                   .replace(tzinfo=dateutil.tz.gettz(time_zone))
                   .humanize())

    @self.app.template_filter()
    def format_date(value, format="%Y-%m-%d"):
      return value.strftime(format)

    @self.app.template_filter()
    def initials(value):
      parts = value.split(' ')
      letters = (part[0] for part in parts)
      return ''.join(letters)

    @self.app.template_filter()
    def client_filter(value):
      """Pass through variable in Jinja2 to client side template.

      The filter tells Jinja2 that a variable is for a client side
      template engine.

      If the variable is undefined, its name will be used in the
      client side template, otherwise, its content will be used.
      """

      if is_undefined(value):
          return '{{{{{}}}}}'.format(value._undefined_name)
      if type(value) is bool:
          value = repr(value).lower()
      return '{{{{{}}}}}'.format(value)
<|endoftext|>


<commit_before>IDLE_VERSION = "0.3"
<commit_msg>Version bump awaiting impending new release. (Not much has changed :-( )<commit_after>IDLE_VERSION = "0.4"
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""This script generates a JUnit report from PCP's test runner log file"""
import sys
import re
from xml.sax.saxutils import escape

def read_logfile(no, ext):
    """read a logfile (and return an error message if file could not be found)"""
    fn = str(no) + '.' + ext
    try:
        with open(fn) as f:
            return f.read()
    except IOError as e:
        return 'could not open {}: {}'.format(fn, e)

def parse_testouput_tests():
    """parse tests from testoutput"""
    tests = []
    for line in sys.stdin:
        success_m = re.match(r'^\[\d+%\] (\d+)$', line)
        notrun_m = re.match(r'^\[\d+%\] (\d+) \[not run\] (.+)$', line)
        failed_m = re.match(r'^\[\d+%\] (\d+) (.+)$', line)
        if success_m:
            tests.append({
                "no": success_m.group(1),
                "stdout": read_logfile(success_m.group(1), "out")
            })
        elif notrun_m:
            tests.append({
                "no": notrun_m.group(1),
                "skipped": notrun_m.group(2),
            })
        elif failed_m:
            tests.append({
                "no": failed_m.group(1),
                "failed": failed_m.group(2).replace("- ", ""),
                "failed_reason": "",
                "stdout": read_logfile(failed_m.group(1), "out.bad"),
                "stderr": read_logfile(failed_m.group(1), "full")
            })
        else:
            # if this line doesn't match any regex, it's probably output from the previous test
            if tests and "failed_reason" in tests[-1]:
                tests[-1]["failed_reason"] += line
    return tests

def parse_testouput():
    """parse test output and calculate summary properties"""
    testsuite = {
        "name": "tests",
        "tests": parse_testouput_tests()
    }

    test_durations = {}
    with open('check.time') as f:
        for line in f:
            no, duration = line.strip().split(' ')
            test_durations[no] = int(duration)
    for test in testsuite["tests"]:
        test["time"] = test_durations.get(test["no"], 0)

    testsuite["tests_cnt"] = len(testsuite["tests"])
    testsuite["skipped_cnt"] = len([test for test in testsuite["tests"] if "skipped" in test])
    testsuite["failed_cnt"] = len([test for test in testsuite["tests"] if "failed" in test])
    testsuite["time"] = sum([test["time"] for test in testsuite["tests"]])
    return testsuite

def create_report():
    """generates a JUnit report"""
    testsuites = []
    testsuites.append(parse_testouput())

    print('<?xml version="1.0" encoding="UTF-8"?>')
    print('<testsuites>')

    for testsuite in testsuites:
        print('<testsuite name="{name}" tests="{tests_cnt}" skipped="{skipped_cnt}" '
              'failures="{failed_cnt}" errors="0" time="{time}">'.format(**testsuite))
        for test in testsuite["tests"]:
            print('<testcase classname="{no}" name="{no}" time="{time}">'.format(**test))
            if "skipped" in test:
                print('<skipped message="{skipped}"/>'.format(**test))
            if "failed" in test:
                print('<failure message="{failed}">{failed_reason}</failure>'.format(
                    failed=test["failed"], failed_reason=escape(test["failed_reason"])))
            if "stdout" in test:
                print('<system-out>{stdout}</system-out>'.format(stdout=escape(test["stdout"])))
            if "stderr" in test:
                print('<system-err>{stderr}</system-err>'.format(stderr=escape(test["stderr"])))
            print('</testcase>')
        print('</testsuite>')

    print('</testsuites>')

if __name__ == '__main__':
    create_report()
<commit_msg>qa: fix junit generation script for azure pipelines<commit_after>#!/usr/bin/env python
"""This script generates a JUnit report from PCP's test runner log file"""
import sys
import re
from xml.sax.saxutils import escape

def read_logfile(no, ext):
    """read a logfile (and return an error message if file could not be found)"""
    fn = str(no) + '.' + ext
    try:
        with open(fn) as f:
            return f.read()
    except IOError as e:
        return 'could not open {}: {}'.format(fn, e)

def parse_testouput_tests():
    """parse tests from testoutput"""
    tests = []
    for line in sys.stdin:
        # [xx%] will only be displayed if there are more than 9 tests
        success_m = re.match(r'^(?:\[\d+%\] )?(\d+)$', line)
        notrun_m = re.match(r'^(?:\[\d+%\] )?(\d+) \[not run\] (.+)$', line)
        failed_m = re.match(r'^(?:\[\d+%\] )?(\d+) (.+)$', line)
        if success_m:
            tests.append({
                "no": success_m.group(1),
                "stdout": read_logfile(success_m.group(1), "out")
            })
        elif notrun_m:
            tests.append({
                "no": notrun_m.group(1),
                "skipped": notrun_m.group(2),
            })
        elif failed_m:
            tests.append({
                "no": failed_m.group(1),
                "failed": failed_m.group(2).replace("- ", ""),
                "failed_reason": "",
                "stdout": read_logfile(failed_m.group(1), "out.bad"),
                "stderr": read_logfile(failed_m.group(1), "full")
            })
        else:
            # if this line doesn't match any regex, it's probably output from the previous test
            if tests and "failed_reason" in tests[-1]:
                tests[-1]["failed_reason"] += line
    return tests

def parse_testouput():
    """parse test output and calculate summary properties"""
    testsuite = {
        "name": "tests",
        "tests": parse_testouput_tests()
    }

    test_durations = {}
    with open('check.time') as f:
        for line in f:
            no, duration = line.strip().split(' ')
            test_durations[no] = int(duration)
    for test in testsuite["tests"]:
        test["time"] = test_durations.get(test["no"], 0)

    testsuite["tests_cnt"] = len(testsuite["tests"])
    testsuite["skipped_cnt"] = len([test for test in testsuite["tests"] if "skipped" in test])
    testsuite["failed_cnt"] = len([test for test in testsuite["tests"] if "failed" in test])
    testsuite["time"] = sum([test["time"] for test in testsuite["tests"]])
    return testsuite

def create_report():
    """generates a JUnit report"""
    testsuites = []
    testsuites.append(parse_testouput())

    print('<?xml version="1.0" encoding="UTF-8"?>')
    print('<testsuites>')

    for testsuite in testsuites:
        print('<testsuite name="{name}" tests="{tests_cnt}" skipped="{skipped_cnt}" '
              'failures="{failed_cnt}" errors="0" time="{time}">'.format(**testsuite))
        for test in testsuite["tests"]:
            print('<testcase classname="{no}" name="{no}" time="{time}">'.format(**test))
            if "skipped" in test:
                print('<skipped message="{skipped}"/>'.format(**test))
            if "failed" in test:
                print('<failure message="{failed}">{failed_reason}</failure>'.format(
                    failed=test["failed"], failed_reason=escape(test["failed_reason"])))
            if "stdout" in test:
                print('<system-out>{stdout}</system-out>'.format(stdout=escape(test["stdout"])))
            if "stderr" in test:
                print('<system-err>{stderr}</system-err>'.format(stderr=escape(test["stderr"])))
            print('</testcase>')
        print('</testsuite>')

    print('</testsuites>')

if __name__ == '__main__':
    create_report()
<|endoftext|>


<commit_before>import os

import h5py

from tools import build_transfer_net
from tools.datasets.urban_tribes import load_data

class Session(object):
    """Training session.

    This is a helper class to bundle model and history together and make it
    easy to resume training and recording history.
    """
    def __init__(self, model, history=None):
        self.model = model
        self.history = history

    def dump(self, path):
        W, b = self.model.layers[-1].get_weights()
        with h5py.File(path, 'w') as f:
            f.create_dataset('W', data=W)
            f.create_dataset('b', data=b)
            group = f.create_group('history')
            for k, v in self.history.items():
                group.create_dataset(k, data=v)

    @classmethod
    def load(cls, model, path):
        with h5py.File(path, 'r') as f:
            W = f['W'][:]
            b = f['b'][:]
            history = {}
            for k, v in f['history'].items():
                history[k] = list(v)
        model.layers[-1].set_weights([W, b])
        return cls(model, history=history)

    def train(self, *args, **kwargs):
        new_history = self.model.fit(*args, **kwargs)
        self._record(new_history.history)

    def _record(self, new_history):
        if self.history is None:
            self.history = new_history
        else:
            for key in self.history.keys():
                self.history[key].extend(new_history[key])

def transfer_learn(layer_name, nb_sample, nb_epoch, output_file):
    """Transfer learning for image classification.

    Args:
        layer_name: Transfer layer name.
        nb_sample: Number of samples per categories.
        nb_epoch: Number of epochs to train.
        output_file: Name of the output file to pick history to.
    """
    # Build model
    model = build_transfer_net(output_dim=11,
                               transfer_layer_name=layer_name)

    # Prepare data
    (x_train, y_train), (x_val, y_val), (x_test, y_test) = \
        load_data(images_per_category=nb_sample)

    # Train
    model.compile(optimizer='adadelta', loss='categorical_crossentropy',
                  metrics=['accuracy'])
    if os.path.exists(output_file):
        print('Resuming')
        session = Session.load(model, output_file)
    else:
        print('Starting')
        session = Session(model)
    session.train(x_train, y_train, batch_size=nb_sample, nb_epoch=nb_epoch,
                  validation_data=(x_val, y_val))
    session.dump(output_file)
    return session
<commit_msg>Split functionalities<commit_after>import os

import h5py

from tools import build_transfer_net
from tools.datasets.urban_tribes import load_data

class Session(object):
    """Training session.

    This is a helper class to bundle model and history together and make it
    easy to resume training and recording history.
    """
    def __init__(self, model, history=None):
        self.model = model
        self.history = history

    def dump(self, path):
        W, b = self.model.layers[-1].get_weights()
        self.dump_weights(W, b, path)
        self.dump_history(self.history, path)

    @classmethod
    def load(cls, model, path):
        W, b = cls.load_weights(path)
        history = cls.load_history(path)
        model.layers[-1].set_weights([W, b])
        return cls(model, history=history)

    def train(self, *args, **kwargs):
        new_history = self.model.fit(*args, **kwargs)
        self._record(new_history.history)

    def _record(self, new_history):
        if self.history is None:
            self.history = new_history
        else:
            for key in self.history.keys():
                self.history[key].extend(new_history[key])

    @staticmethod
    def dump_history(history, path):
        with h5py.File(path, 'w') as f:
            group = f.create_group('history')
            for k, v in history.items():
                group.create_dataset(k, data=v)

    @staticmethod
    def load_history(path):
        with h5py.File(path, 'r') as f:
            history = {}
            for k, v in f['history'].items():
                history[k] = list(v)
        return history

    @staticmethod
    def dump_weights(W, b, path):
        with h5py.File(path, 'w') as f:
            f.create_dataset('W', data=W)
            f.create_dataset('b', data=b)

    @staticmethod
    def load_weights(path):
        with h5py.File(path, 'r') as f:
            W = f['W'][:]
            b = f['b'][:]
        return W, b

def transfer_learn(layer_name, nb_sample, nb_epoch, output_file):
    """Transfer learning for image classification.

    Args:
        layer_name: Transfer layer name.
        nb_sample: Number of samples per categories.
        nb_epoch: Number of epochs to train.
        output_file: Name of the output file to pick history to.
    """
    # Build model
    model = build_transfer_net(output_dim=11,
                               transfer_layer_name=layer_name)

    # Prepare data
    (x_train, y_train), (x_val, y_val), (x_test, y_test) = \
        load_data(images_per_category=nb_sample)

    # Train
    model.compile(optimizer='adadelta', loss='categorical_crossentropy',
                  metrics=['accuracy'])
    if os.path.exists(output_file):
        print('Resuming')
        session = Session.load(model, output_file)
    else:
        print('Starting')
        session = Session(model)
    session.train(x_train, y_train, batch_size=nb_sample, nb_epoch=nb_epoch,
                  validation_data=(x_val, y_val))
    session.dump(output_file)
    return session
<|endoftext|>


<commit_before>#
# MansOS web server - mote handler class
#

import serial, subprocess, sys, time
from serial.tools import list_ports
from settings import *

def runSubprocess(args, server):
    retcode = -1
    try:
        print "runSubprocess, args = ", args
        output = subprocess.check_output(args, stderr = subprocess.STDOUT)
        server.uploadCallback(output)
        retcode = 0
    except OSError as e:
        print "runSubprocess OSError:" + str(e)
    except CalledProcessError as e:
        print "runSubprocess CalledProcessError:" + str(e)
        retcode = e.returncode
    except Exception as e:
        print "runSubprocess exception:" + str(e)
    finally:
        return retcode

def runSubprocess1(args, server):
    retcode = -1
    try:
        print "runSubprocess1, args = ", args
        proc = subprocess.Popen(args, stderr = subprocess.STDOUT, stdout = subprocess.PIPE, shell = False)
        while proc.poll() is None:
            line = proc.stdout.readline()
            if line:
                if not server.uploadCallback(line):
                    break
            time.sleep(0.01)
        proc.wait()
        retcode = proc.returncode
    except OSError as e:
        print "runSubprocess1 OSError:" + str(e)
    except CalledProcessError as e:
        print "runSubprocess1 CalledProcessError:" + str(e)
        retcode = e.returncode
    except Exception as e:
        print "runSubprocess1 exception:" + str(e)
    finally:
        return retcode

def simpleCallback(line):
    print "simpleCallback: got", line
    return True

class Mote(object):
    counter = 0
    def __init__(self, portName):
        self.number = Mote.counter
        Mote.counter += 1
        self.portName = portName
        self.port = None
        self.isSelected = False
        self.isStatic = False
        self.buffer = ""
        self.writebuffer = ""
        self.platform = "telosb"

    def openSerial(self):
        if not self.isSelected: return
        try:
            baudrate = settingsInstance.getCfgValueAsInt("baudrate", SERIAL_BAUDRATE)
            self.port = serial.Serial(self.portName,
                                      baudrate,
                                      timeout=1,
                                      parity=serial.PARITY_NONE)
            self.port.flushInput()
            self.port.flushOutput()
            if self.platform not in ['xm1000', 'z1'] :
                # make sure reset pin is low for the platforms that need it
                self.port.setDTR(0)
                self.port.setRTS(0)
        except Exception as e:
            print("\nSerial exception:\n\t" + str(e))
            self.port = None
            return

        print("Listening to serial port: " + self.port.portstr + ", rate: " + str(baudrate))

    def closeSerial(self):
        if self.port:
            self.port.close()
            self.port = None

    def tryToOpenSerial(self, makeSelected):
        if not self.port:
            if makeSelected: self.isSelected = True
            self.openSerial()

    def tryRead(self, binaryToo):
        numRead = 0
        if self.port:
            try:
              while self.port.inWaiting():
                c = self.port.read(1)

                # save to file if required
                if settingsInstance.cfg.saveToFilename \
                        and not settingsInstance.cfg.saveProcessedData:
                    filename = settingsInstance.cfg.dataDirectory + "/" \
                        + settingsInstance.cfg.saveToFilename
                    with open(filename, "a") as f:
                        f.write(c)
                        f.close()

                # add to the local buffer
                if binaryToo or isascii(c):
                    self.buffer += c
                    numRead += 1
            except Exception as e:
              print("\nserial read exception:\t" + str(e))
              self.port.close()
              self.port = None

        return numRead

    def tryToUpload(self, server, filename):
        self.tryToOpenSerial(False)
        if not self.port: return 1

        if self.platform == "telosb":
            bsl = "mos/make/scripts/tos-bsl"
            platformArgs = ["--telosb"]
        elif self.platform == "xm1000":
            bsl = "mos/make/scripts/xm1000-bsl"
            platformArgs = ["--telosb"] # "telosb" is still the right arg
        elif self.platform == "z1":
            bsl = "mos/make/scripts/z1-bsl-nopic"
            platformArgs = ["--z1"]
        else:
            bsl = "mos/make/scripts/bsl.py"
            platformArgs = ["--invert-reset", "--invert-test"]

        bsl = os.path.join(settingsInstance.getCfgValue("pathToMansOS"), bsl)
        arglist = ["python", bsl, "-c", self.port.portstr, "-r", "-e", "-I", "-p", filename]
        argist.extend(platformArgs)
        if settingsInstance.getCfgValueAsInt("slowUpload"):
            arglist.append("--slow")

        retcode = runSubprocess1(arglist, server)

        return retcode

    def tryToCompileAndUpload(self, server, filename):
        self.tryToOpenSerial(False)
        if not self.port: return 1

        arglist = ["make", self.platform, "upload"]
        retcode = runSubprocess1(arglist, server)

        return retcode

class MoteCollection(object):
    def __init__(self):
        self.motes = []

    def storeSelected(self):
        selected = []
        platforms = []
        for m in self.motes:
            if m.isSelected:
                selected.append(m.portName)
            platforms.append(m.portName + ":" + m.platform)
        settingsInstance.setCfgValue("selectedMotes", selected)
        settingsInstance.setCfgValue("motePlatforms", platforms)
        settingsInstance.save()

    def retrieveSelected(self):
        selected = settingsInstance.getCfgValue("selectedMotes")
        platforms = settingsInstance.getCfgValue("motePlatforms")
        for m in self.motes:
            m.isSelected = m.portName in selected
        for p in platforms:
            try:
                (portName, platform) = p.split(':')
                for m in self.motes:
                    if m.portName == portName:
                        m.platform = platform
            except:
                pass

    def addAll(self):
        self.motes = []
        staticPorts = set(settingsInstance.getCfgValue("motes"))

        dynamicPorts = set()
        for x in list_ports.comports():
            # skip ports that are not connected
            if x[2] == "n/a": continue
            portName = x[0]
            if not ("USB" in portName or "ACM" in portName): continue
            dynamicPorts.add(portName)

        allPorts = dynamicPorts.union(staticPorts)
        for port in allPorts:
            self.motes.append(Mote(port))

        self.retrieveSelected()

    def getMotes(self):
        return self.motes

    def getMote(self, num):
        return self.motes[num]

    def isEmpty(self):
        return len(self.motes) == 0

    def anySelected(self):
        for m in self.motes:
            if m.isSelected: return True
        return False

    def closeAll(self):
        for m in self.motes:
            m.closeSerial()
<commit_msg>comment out dynamic ports<commit_after>#
# MansOS web server - mote handler class
#

import serial, subprocess, sys, time
#from serial.tools import list_ports
from settings import *

def runSubprocess(args, server):
    retcode = -1
    try:
        print "runSubprocess, args = ", args
        output = subprocess.check_output(args, stderr = subprocess.STDOUT)
        server.uploadCallback(output)
        retcode = 0
    except OSError as e:
        print "runSubprocess OSError:" + str(e)
    except CalledProcessError as e:
        print "runSubprocess CalledProcessError:" + str(e)
        retcode = e.returncode
    except Exception as e:
        print "runSubprocess exception:" + str(e)
    finally:
        return retcode

def runSubprocess1(args, server):
    retcode = -1
    try:
        print "runSubprocess1, args = ", args
        proc = subprocess.Popen(args, stderr = subprocess.STDOUT, stdout = subprocess.PIPE, shell = False)
        while proc.poll() is None:
            line = proc.stdout.readline()
            if line:
                if not server.uploadCallback(line):
                    break
            time.sleep(0.01)
        proc.wait()
        retcode = proc.returncode
    except OSError as e:
        print "runSubprocess1 OSError:" + str(e)
    except CalledProcessError as e:
        print "runSubprocess1 CalledProcessError:" + str(e)
        retcode = e.returncode
    except Exception as e:
        print "runSubprocess1 exception:" + str(e)
    finally:
        return retcode

def simpleCallback(line):
    print "simpleCallback: got", line
    return True

class Mote(object):
    counter = 0
    def __init__(self, portName):
        self.number = Mote.counter
        Mote.counter += 1
        self.portName = portName
        self.port = None
        self.isSelected = False
        self.isStatic = False
        self.buffer = ""
        self.writebuffer = ""
        self.platform = "telosb"

    def openSerial(self):
        if not self.isSelected: return
        try:
            baudrate = settingsInstance.getCfgValueAsInt("baudrate", SERIAL_BAUDRATE)
            self.port = serial.Serial(self.portName,
                                      baudrate,
                                      timeout=1,
                                      parity=serial.PARITY_NONE)
            self.port.flushInput()
            self.port.flushOutput()
            if self.platform not in ['xm1000', 'z1'] :
                # make sure reset pin is low for the platforms that need it
                self.port.setDTR(0)
                self.port.setRTS(0)
        except Exception as e:
            print("\nSerial exception:\n\t" + str(e))
            self.port = None
            return

        print("Listening to serial port: " + self.port.portstr + ", rate: " + str(baudrate))

    def closeSerial(self):
        if self.port:
            self.port.close()
            self.port = None

    def tryToOpenSerial(self, makeSelected):
        if not self.port:
            if makeSelected: self.isSelected = True
            self.openSerial()

    def tryRead(self, binaryToo):
        numRead = 0
        if self.port:
            try:
              while self.port.inWaiting():
                c = self.port.read(1)

                # save to file if required
                if settingsInstance.cfg.saveToFilename \
                        and not settingsInstance.cfg.saveProcessedData:
                    filename = settingsInstance.cfg.dataDirectory + "/" \
                        + settingsInstance.cfg.saveToFilename
                    with open(filename, "a") as f:
                        f.write(c)
                        f.close()

                # add to the local buffer
                if binaryToo or isascii(c):
                    self.buffer += c
                    numRead += 1
            except Exception as e:
              print("\nserial read exception:\t" + str(e))
              self.port.close()
              self.port = None

        return numRead

    def tryToUpload(self, server, filename):
        self.tryToOpenSerial(False)
        if not self.port: return 1

        if self.platform == "telosb":
            bsl = "mos/make/scripts/tos-bsl"
            platformArgs = ["--telosb"]
        elif self.platform == "xm1000":
            bsl = "mos/make/scripts/xm1000-bsl"
            platformArgs = ["--telosb"] # "telosb" is still the right arg
        elif self.platform == "z1":
            bsl = "mos/make/scripts/z1-bsl-nopic"
            platformArgs = ["--z1"]
        else:
            bsl = "mos/make/scripts/bsl.py"
            platformArgs = ["--invert-reset", "--invert-test"]

        bsl = os.path.join(settingsInstance.getCfgValue("pathToMansOS"), bsl)
        arglist = ["python", bsl, "-c", self.port.portstr, "-r", "-e", "-I", "-p", filename]
        argist.extend(platformArgs)
        if settingsInstance.getCfgValueAsInt("slowUpload"):
            arglist.append("--slow")

        retcode = runSubprocess1(arglist, server)

        return retcode

    def tryToCompileAndUpload(self, server, filename):
        self.tryToOpenSerial(False)
        if not self.port: return 1

        arglist = ["make", self.platform, "upload"]
        retcode = runSubprocess1(arglist, server)

        return retcode

class MoteCollection(object):
    def __init__(self):
        self.motes = []

    def storeSelected(self):
        selected = []
        platforms = []
        for m in self.motes:
            if m.isSelected:
                selected.append(m.portName)
            platforms.append(m.portName + ":" + m.platform)
        settingsInstance.setCfgValue("selectedMotes", selected)
        settingsInstance.setCfgValue("motePlatforms", platforms)
        settingsInstance.save()

    def retrieveSelected(self):
        selected = settingsInstance.getCfgValue("selectedMotes")
        platforms = settingsInstance.getCfgValue("motePlatforms")
        for m in self.motes:
            m.isSelected = m.portName in selected
        for p in platforms:
            try:
                (portName, platform) = p.split(':')
                for m in self.motes:
                    if m.portName == portName:
                        m.platform = platform
            except:
                pass

    def addAll(self):
        self.motes = []
        staticPorts = set(settingsInstance.getCfgValue("motes"))

        dynamicPorts = set()
#        for x in list_ports.comports():
#            # skip ports that are not connected
#            if x[2] == "n/a": continue
#            portName = x[0]
#            if not ("USB" in portName or "ACM" in portName): continue
#            dynamicPorts.add(portName)

        allPorts = dynamicPorts.union(staticPorts)
        for port in allPorts:
            self.motes.append(Mote(port))

        self.retrieveSelected()

    def getMotes(self):
        return self.motes

    def getMote(self, num):
        return self.motes[num]

    def isEmpty(self):
        return len(self.motes) == 0

    def anySelected(self):
        for m in self.motes:
            if m.isSelected: return True
        return False

    def closeAll(self):
        for m in self.motes:
            m.closeSerial()
<|endoftext|>