<commit_before># -*- coding: utf-8 -*-
import unittest
from nose.tools import assert_equals, assert_false, assert_true
from splinter.element_list import ElementList

class Person(object):
    """Very simple class, just for tests"""

    def __init__(self):
        self.current_action = None

    def walk(self):
        self.current_action = "walking"

class ElementListTest(unittest.TestCase):

    def test_method_that_verifies_if_the_list_is_empty(self):
        "should verify if the list is empty"
        the_list = ElementList([1, 2, 3])
        assert_false(the_list.is_empty())
        assert_true(ElementList([]).is_empty())

    def test_call_method_on_first_element(self):
        "when some method is missing on ElementList and is present in element, it should be passed"
        the_list = ElementList([Person(), Person(), Person()])
        the_list.walk()
        the_person = the_list.first
        assert_equals("walking", the_person.current_action)
<commit_msg>More tests for ElementList<commit_after># -*- coding: utf-8 -*-
import unittest
from nose.tools import assert_equals, assert_false, assert_true, raises
from splinter.element_list import ElementList
from splinter.exceptions import ElementDoesNotExist

class Person(object):
    """Very simple class, just for tests"""

    def __init__(self):
        self.current_action = None

    def walk(self):
        self.current_action = "walking"

class ElementListTest(unittest.TestCase):

    def test_method_that_verifies_if_the_list_is_empty(self):
        "should verify if the list is empty"
        the_list = ElementList([1, 2, 3])
        assert_false(the_list.is_empty())
        assert_true(ElementList([]).is_empty())

    def test_property_first_and_last(self):
        "should provide a \"first\" and a \"last\" properties which returns the first and last element"
        the_list = ElementList([1, 2, 3])
        assert_equals(the_list[0], the_list.first)
        assert_equals(the_list[2], the_list.last)

    def test_call_method_on_first_element(self):
        "when some method is missing on ElementList and is present in element, it should be passed"
        the_list = ElementList([Person(), Person(), Person()])
        the_list.walk()
        the_person = the_list.first
        assert_equals("walking", the_person.current_action)

    @raises(ElementDoesNotExist)
    def test_raise_exception_on_indexerror(self):
        "should raise ElementDoesNotExist exception on IndexError"
        ElementList([]).first
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    Pygments tests with example files
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: 2006-2007 by Georg Brandl.
    :license: BSD, see LICENSE for more details.
"""

import os
import unittest

from pygments import highlight
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error

class ExampleFileTest(unittest.TestCase):
    pass

lfd = 0

# generate methods
for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
    absfn = os.path.join(testdir, 'examplefiles', fn)
    if not os.path.isfile(absfn):
        continue

    try:
        lx = get_lexer_for_filename(absfn)
    except ValueError:
        try:
            name, rest = fn.split("_", 1)
            lx = get_lexer_by_name(name)
        except ValueError:
            raise AssertionError('no lexer found for file %r' % fn)

    def test(self, lx=lx, absfn=absfn):
        text = file(absfn, 'U').read()
        text = text.strip('\n') + '\n'
        text = text.decode('latin1')
        ntext = []
        for type, val in lx.get_tokens(text):
            ntext.append(val)
            self.failIf(type == Error, 'lexer generated error token for '+absfn)
        if u''.join(ntext) != text:
            self.fail('round trip failed for '+absfn)

    setattr(ExampleFileTest, 'test_%i' % lfd, test)
    lfd += 1
<commit_msg>Use specific exception.<commit_after># -*- coding: utf-8 -*-
"""
    Pygments tests with example files
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: 2006-2008 by Georg Brandl.
    :license: BSD, see LICENSE for more details.
"""

import os
import unittest

from pygments import highlight
from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
from pygments.token import Error
from pygments.util import ClassNotFound

class ExampleFileTest(unittest.TestCase):
    pass

lfd = 0

# generate methods
for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
    absfn = os.path.join(testdir, 'examplefiles', fn)
    if not os.path.isfile(absfn):
        continue

    try:
        lx = get_lexer_for_filename(absfn)
    except ClassNotFound:
        try:
            name, rest = fn.split("_", 1)
            lx = get_lexer_by_name(name)
        except ClassNotFound:
            raise AssertionError('no lexer found for file %r' % fn)

    def test(self, lx=lx, absfn=absfn):
        text = file(absfn, 'U').read()
        text = text.strip('\n') + '\n'
        text = text.decode('latin1')
        ntext = []
        for type, val in lx.get_tokens(text):
            ntext.append(val)
            self.failIf(type == Error, 'lexer generated error token for '+absfn)
        if u''.join(ntext) != text:
            self.fail('round trip failed for '+absfn)

    setattr(ExampleFileTest, 'test_%i' % lfd, test)
    lfd += 1
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
test examples given in paper by comparison against reference solution.

the actual comparison is done by checking the equality of the interaction-term
with a numerical solution (generated with numerical_evaluation.py)

"""

import unittest
import numpy as np

import sys

#sys.path.append(os.path.join('..', 'mintrend'))
sys.path.append('..')
from rt1.rt1 import RT1
from rt1.volume import Rayleigh, HenyeyGreenstein
from rt1.surface import Isotropic, CosineLobe

class TestExamples(unittest.TestCase):

    def setUp(self):
        # read reference solutions for backscattering case
        fname1 = 'example1_int.csv'
        x1 = np.loadtxt('tests/' + fname1, delimiter=',',skiprows=0)
        self.inc1 = x1[:,0]
        self.int_num_1 = x1[:,1]

        fname2 = 'example2_int.csv'
        x2 = np.loadtxt('tests/' + fname2, delimiter=',',skiprows=0)
        self.inc2 = x2[:,0]
        self.int_num_2 = x2[:,1]

    def test_example_1_int(self):
        print('Testing Example 1 ...')

        inc = self.inc1

        # initialize output fields for faster processing
        Itot = np.ones_like(inc)*np.nan
        Isurf = np.ones_like(inc)*np.nan
        Ivol = np.ones_like(inc)*np.nan
        Iint = np.ones_like(inc)*np.nan

        # ---- evaluation of first example
        V = Rayleigh(tau=0.7, omega=0.3)
        SRF = CosineLobe(ncoefs=11, i=5) # 11 instead of 10 coefficients used to assure 7 digit precision

        fn = None
        for i in xrange(len(inc)):
            # set geometries
            mu_0 = np.cos(np.deg2rad(inc[i]))
            phi_0 = np.deg2rad(0.)
            phi_ex = phi_0 + np.pi

            R = RT1(1., mu_0, mu_0, phi_0, phi_ex, RV=V, SRF=SRF, fn=fn, geometry='mono')
            fn = R.fn  # store coefficients for faster itteration
            Itot[i], Isurf[i], Ivol[i], Iint[i] = R.calc()

        #for i in range(0,len(self.inc1)):
            self.assertAlmostEqual(self.int_num_1[i],Iint[i])

    def test_example_2_int(self):
        print('Testing Example 2 ...')

        inc = self.inc2

        # initialize output fields for faster processing
        Itot = np.ones_like(inc)*np.nan
        Isurf = np.ones_like(inc)*np.nan
        Ivol = np.ones_like(inc)*np.nan
        Iint = np.ones_like(inc)*np.nan

        # ---- evaluation of second example
        V = HenyeyGreenstein(tau=0.7, omega=0.3, t=0.7, ncoefs=20)
        SRF = CosineLobe(ncoefs=11, i=5) # 11 instead of 10 coefficients used to assure 7 digit precision

        fn = None
        for i in xrange(len(inc)):
            # set geometries
            mu_0 = np.cos(np.deg2rad(inc[i]))
            phi_0 = np.deg2rad(0.)
            phi_ex = phi_0 + np.pi

            R = RT1(1., mu_0, mu_0, phi_0, phi_ex, RV=V, SRF=SRF, fn=fn, geometry='mono')
            fn = R.fn  # store coefficients for faster itteration
            Itot[i], Isurf[i], Ivol[i], Iint[i] = R.calc()

        #for i in range(0,len(self.inc1)):
            self.assertAlmostEqual(self.int_num_2[i],Iint[i])

if __name__ == "__main__":
    unittest.main()

<commit_msg>fixed minor path problem for example tests<commit_after># -*- coding: utf-8 -*-

"""
test examples given in paper by comparison against reference solution.

the actual comparison is done by checking the equality of the interaction-term
with a numerical solution (generated with numerical_evaluation.py)

"""

import unittest
import numpy as np

import os
import sys

#sys.path.append(os.path.join('..', 'mintrend'))
sys.path.append('..')
from rt1.rt1 import RT1
from rt1.volume import Rayleigh, HenyeyGreenstein
from rt1.surface import Isotropic, CosineLobe

class TestExamples(unittest.TestCase):

    def setUp(self):
        # read reference solutions for backscattering case
        fname1 = os.path.dirname(__file__) + os.sep + 'example1_int.csv'
        x1 = np.loadtxt(fname1, delimiter=',',skiprows=0)
        self.inc1 = x1[:,0]
        self.int_num_1 = x1[:,1]

        fname2 = os.path.dirname(__file__) + os.sep + 'example2_int.csv'
        x2 = np.loadtxt(fname2, delimiter=',',skiprows=0)
        self.inc2 = x2[:,0]
        self.int_num_2 = x2[:,1]

    def test_example_1_int(self):
        print('Testing Example 1 ...')

        inc = self.inc1

        # initialize output fields for faster processing
        Itot = np.ones_like(inc)*np.nan
        Isurf = np.ones_like(inc)*np.nan
        Ivol = np.ones_like(inc)*np.nan
        Iint = np.ones_like(inc)*np.nan

        # ---- evaluation of first example
        V = Rayleigh(tau=0.7, omega=0.3)
        SRF = CosineLobe(ncoefs=11, i=5) # 11 instead of 10 coefficients used to assure 7 digit precision

        fn = None
        for i in xrange(len(inc)):
            # set geometries
            mu_0 = np.cos(np.deg2rad(inc[i]))
            phi_0 = np.deg2rad(0.)
            phi_ex = phi_0 + np.pi

            R = RT1(1., mu_0, mu_0, phi_0, phi_ex, RV=V, SRF=SRF, fn=fn, geometry='mono')
            fn = R.fn  # store coefficients for faster itteration
            Itot[i], Isurf[i], Ivol[i], Iint[i] = R.calc()

        #for i in range(0,len(self.inc1)):
            self.assertAlmostEqual(self.int_num_1[i],Iint[i])

    def test_example_2_int(self):
        print('Testing Example 2 ...')

        inc = self.inc2

        # initialize output fields for faster processing
        Itot = np.ones_like(inc)*np.nan
        Isurf = np.ones_like(inc)*np.nan
        Ivol = np.ones_like(inc)*np.nan
        Iint = np.ones_like(inc)*np.nan

        # ---- evaluation of second example
        V = HenyeyGreenstein(tau=0.7, omega=0.3, t=0.7, ncoefs=20)
        SRF = CosineLobe(ncoefs=11, i=5) # 11 instead of 10 coefficients used to assure 7 digit precision

        fn = None
        for i in xrange(len(inc)):
            # set geometries
            mu_0 = np.cos(np.deg2rad(inc[i]))
            phi_0 = np.deg2rad(0.)
            phi_ex = phi_0 + np.pi

            R = RT1(1., mu_0, mu_0, phi_0, phi_ex, RV=V, SRF=SRF, fn=fn, geometry='mono')
            fn = R.fn  # store coefficients for faster itteration
            Itot[i], Isurf[i], Ivol[i], Iint[i] = R.calc()

        #for i in range(0,len(self.inc1)):
            self.assertAlmostEqual(self.int_num_2[i],Iint[i])

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Mapmask.py
#
#   Copyright (C) 2008 STFC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# 23rd December 2008
#
# A wrapper for the CCP4 program mapmask
#

import os
import sys

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                             'Python'))

from Driver.DriverFactory import DriverFactory
from Decorators.DecoratorFactory import DecoratorFactory

def Mapmask(DriverType = None):
    '''Create a Mapmask instance based on the passed in Driver type.'''

    DriverInstance = DriverFactory.Driver(DriverType)
    CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')

    class MapmaskWrapper(CCP4DriverInstance.__class__):
        '''A wrapper class for converting reflections to mtz.'''

        def __init__(self):
            CCP4DriverInstance.__class__.__init__(self)

            self.set_executable(os.path.join(
                os.environ.get('CBIN', ''), 'mapmask'))

            self._symmetry = None

            return

        def set_symmetry(self, symmetry):
            self._symetry = symmetry
            return

        def mask_asu(self):
            self.check_mapin()
            self.check_mapout()

            self.start()

            if self._symmetry:
                self.input('symmetry %s' % str(self._symmetry))

            self.input('xyzlim asu')

            self.close_wait()

            return self.get_ccp4_status()

        def mask_xyzin(self):
            self.check_mapin()
            self.check_xyzin()

            self.check_mapout()

            self.start()

            if self._symmetry:
                self.input('symmetry %s' % str(self._symmetry))

            self.input('xyzlim asu')

            self.close_wait()

            return self.get_ccp4_status()

    return MapmaskWrapper()

if __name__ == '__main__':

    mapmask = Mapmask()

    mapmask.set_mapin(sys.argv[1])
    mapmask.set_mapout(sys.argv[2])

    print mapmask.mask_asu()
<commit_msg>Moved more things from the attic at last<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# Cellparm.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# A wrapper for the XDS jiffy program CELLPARM for computing average unit
# cell parameters, from the results of multiple CORRECT runs for input into
# a single XSCALE run.
#

import os
import sys
import math

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

if not os.environ.has_key('XIA2_ROOT'):
    raise RuntimeError, 'XIA2_ROOT not defined'

if not os.path.join(os.environ['XIA2CORE_ROOT'],
                    'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                                 'Python'))

if not os.environ['XIA2_ROOT'] in sys.path:
    sys.path.append(os.environ['XIA2_ROOT'])

from Driver.DriverFactory import DriverFactory

def Cellparm(DriverType = None):

    DriverInstance = DriverFactory.Driver(DriverType)

    class CellparmWrapper(DriverInstance.__class__):
        '''A wrapper for wrapping CELLPARM.'''

        def __init__(self):

            # set up the object ancestors...

            DriverInstance.__class__.__init__(self)

            self.set_executable('cellparm')

            # input parameters
            self._cells = []
            self._n_refs = []

            return

        def add_cell(self, cell, n_ref):
            '''Add a unit cell which belongs to n_ref reflections.'''

            self._cells.append(cell)
            self._n_refs.append(n_ref)

            return

        def get_cell(self):
            '''Compute an average cell.'''

            if len(self._cells) < 1:
                raise RuntimeError, 'no input unit cell parameters'

            # check that the input cells are reasonably uniform -
            # be really relaxed and allow 5% variation!

            average_cell = [self._cells[0][j] for j in range(6)]
            number_cells = 1

            for j in range(1, len(self._cells)):
                cell = self._cells[j]
                for k in range(6):
                    average = average_cell[k] / number_cells
                    if math.fabs((cell[k] - average) / average) > 0.05:
                        raise RuntimeError, 'incompatible unit cells'

                # it was ok to remember for later on..
                for k in range(6):
                    average_cell[k] += cell[k]
                number_cells += 1

            cellparm_inp = open(os.path.join(
                self.get_working_directory(), 'CELLPARM.INP'), 'w')

            for j in range(len(self._cells)):
                cell = self._cells[j]
                n_ref = self._n_refs[j]
                cellparm_inp.write('UNIT_CELL_CONSTANTS=')
                cellparm_inp.write(
                    '%.3f %.3f %.3f %.3f %.3f %.3f WEIGHT=%d\n' % \
                    (cell[0], cell[1], cell[2], cell[3], cell[4], cell[5],
                     n_ref))

            cellparm_inp.close()

            self.start()

            self.close_wait()

            # FIXME need to look for errors in here

            cellparm_lp = open(os.path.join(
                self.get_working_directory(), 'CELLPARM.LP'), 'r')
            data = cellparm_lp.readlines()

            return map(float, data[-1].split()[:6])

    return CellparmWrapper()

if __name__ == '__main__':
    cellparm = Cellparm()

    cellparm.add_cell(
        (64.585, 30.911, 34.886, 90.000, 105.822, 90.000), 284516)

    cellparm.add_cell(
        (64.584, 30.932, 34.879, 90.000, 105.832, 90.000), 32106)

    cellparm.add_cell(
        (64.658, 30.972, 34.918, 90.000, 105.823, 90.000), 1966)

    print cellparm.get_cell()
<commit_msg>Add use of relax flag to this - will now allow 20% difference in unit cell (!) if you specify -relax...<commit_after>#!/usr/bin/env python
# Cellparm.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# A wrapper for the XDS jiffy program CELLPARM for computing average unit
# cell parameters, from the results of multiple CORRECT runs for input into
# a single XSCALE run.
#

import os
import sys
import math

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

if not os.environ.has_key('XIA2_ROOT'):
    raise RuntimeError, 'XIA2_ROOT not defined'

if not os.path.join(os.environ['XIA2CORE_ROOT'],
                    'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                                 'Python'))

if not os.environ['XIA2_ROOT'] in sys.path:
    sys.path.append(os.environ['XIA2_ROOT'])

from Driver.DriverFactory import DriverFactory

from Handlers.Flags import Flags

def Cellparm(DriverType = None):

    DriverInstance = DriverFactory.Driver(DriverType)

    class CellparmWrapper(DriverInstance.__class__):
        '''A wrapper for wrapping CELLPARM.'''

        def __init__(self):

            # set up the object ancestors...

            DriverInstance.__class__.__init__(self)

            self.set_executable('cellparm')

            # input parameters
            self._cells = []
            self._n_refs = []

            return

        def add_cell(self, cell, n_ref):
            '''Add a unit cell which belongs to n_ref reflections.'''

            self._cells.append(cell)
            self._n_refs.append(n_ref)

            return

        def get_cell(self):
            '''Compute an average cell.'''

            if len(self._cells) < 1:
                raise RuntimeError, 'no input unit cell parameters'

            # check that the input cells are reasonably uniform -
            # be really relaxed and allow 5% variation!

            average_cell = [self._cells[0][j] for j in range(6)]
            number_cells = 1

            for j in range(1, len(self._cells)):
                cell = self._cells[j]
                for k in range(6):
                    average = average_cell[k] / number_cells
                    if math.fabs((cell[k] - average) / average) > 0.05 \
                           and not Flags.get_relax():
                        raise RuntimeError, 'incompatible unit cells'
                    average = average_cell[k] / number_cells
                    if math.fabs((cell[k] - average) / average) > 0.2:
                        raise RuntimeError, 'very incompatible unit cells'

                # it was ok to remember for later on..
                for k in range(6):
                    average_cell[k] += cell[k]
                number_cells += 1

            cellparm_inp = open(os.path.join(
                self.get_working_directory(), 'CELLPARM.INP'), 'w')

            for j in range(len(self._cells)):
                cell = self._cells[j]
                n_ref = self._n_refs[j]
                cellparm_inp.write('UNIT_CELL_CONSTANTS=')
                cellparm_inp.write(
                    '%.3f %.3f %.3f %.3f %.3f %.3f WEIGHT=%d\n' % \
                    (cell[0], cell[1], cell[2], cell[3], cell[4], cell[5],
                     n_ref))

            cellparm_inp.close()

            self.start()

            self.close_wait()

            # FIXME need to look for errors in here

            cellparm_lp = open(os.path.join(
                self.get_working_directory(), 'CELLPARM.LP'), 'r')
            data = cellparm_lp.readlines()

            return map(float, data[-1].split()[:6])

    return CellparmWrapper()

if __name__ == '__main__':
    cellparm = Cellparm()

    cellparm.add_cell(
        (64.585, 30.911, 34.886, 90.000, 105.822, 90.000), 284516)

    cellparm.add_cell(
        (64.584, 30.932, 34.879, 90.000, 105.832, 90.000), 32106)

    cellparm.add_cell(
        (64.658, 30.972, 34.918, 90.000, 105.823, 90.000), 1966)

    print cellparm.get_cell()
<|endoftext|>


<commit_before>import os
import sys
import unittest

try:
    #sys.path.insert(0, os.path.abspath('..'))
    from sheetmaker import html_builder
    from data import test_html_constants
except:
    import html_builder
    from tests.data import test_html_constants

class HtmlBuilderTestCase(unittest.TestCase):
    """Tests for `html_builder.py`."""

    def test_create_empty_sheet(self):
        """Is empty sheet html created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.create_empty_sheet()
        self.assertEqual(test_html[0], test_html_constants.TEST_EMPTY_SHEET)
        self.assertEqual(test_html[1], None)

    def test_set_style(self):
        """Is color style created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.set_style(1)
        self.assertEqual(test_html[0], test_html_constants.TEST_COLOR_STYLE)
        self.assertEqual(test_html[1], "<!-- css -->")

    def test_build_columns(self):
        """Are columns created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_columns(3)
        self.assertEqual(test_html[0], test_html_constants.TEST_COLUMNS)
        self.assertEqual(test_html[1], "<!-- columns -->")

    def test_build_header(self):
        """Is header created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_header("author")
        self.assertEqual(test_html[0], test_html_constants.TEST_HEADER)
        self.assertEqual(test_html[1], "<!-- header -->")

    def test_build_footer(self):
        """Is footer created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None, "author")
        test_html = empty_sheet.build_footer("author.png", "http://author.com", "sponsor", "http://sponsor.com")
        self.assertEqual(test_html[0], test_html_constants.TEST_FOOTER)
        self.assertEqual(test_html[1], "<!-- footer -->")

    def test_build_rows_block(self):
        """Is rows block created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_rows_block(1, "block title", 2, ["row1", "row2"])
        self.assertEqual(test_html[0], test_html_constants.TEST_ROWS_BLOCK)
        self.assertEqual(test_html[1], "<!-- column1 -->")

    def test_build_text_block(self):
        """Is text block created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_text_block(2, "block title", "text text text")
        self.assertEqual(test_html[0], test_html_constants.TEST_TEXT_BLOCK)
        self.assertEqual(test_html[1], "<!-- column2 -->")

    def tearDown(self):
        """Removes  extra files generated during tests

        Args:
            file_name (str): File to be removed.
        """
        try:
            #os.remove(self.test_file_name + ".html")
            pass
        except OSError as oserr:
            print(oserr)

if __name__ == '__main__':
    unittest.main()

<commit_msg>Path changes<commit_after>import os
import sys
import unittest

try:
    sys.path.insert(0, os.path.abspath('..'))
    #from sheetmaker import html_builder
    from sheetmaker.html_builder import *
    from data import test_html_constants
except:
    from sheetmaker.html_builder import *
    from tests.data import test_html_constants

class HtmlBuilderTestCase(unittest.TestCase):
    """Tests for `html_builder.py`."""

    def test_create_empty_sheet(self):
        """Is empty sheet html created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.create_empty_sheet()
        self.assertEqual(test_html[0], test_html_constants.TEST_EMPTY_SHEET)
        self.assertEqual(test_html[1], None)

    def test_set_style(self):
        """Is color style created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.set_style(1)
        self.assertEqual(test_html[0], test_html_constants.TEST_COLOR_STYLE)
        self.assertEqual(test_html[1], "<!-- css -->")

    def test_build_columns(self):
        """Are columns created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_columns(3)
        self.assertEqual(test_html[0], test_html_constants.TEST_COLUMNS)
        self.assertEqual(test_html[1], "<!-- columns -->")

    def test_build_header(self):
        """Is header created succesfully?"""
        self.title = "title"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_header("author")
        self.assertEqual(test_html[0], test_html_constants.TEST_HEADER)
        self.assertEqual(test_html[1], "<!-- header -->")

    def test_build_footer(self):
        """Is footer created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None, "author")
        test_html = empty_sheet.build_footer("author.png", "http://author.com", "sponsor", "http://sponsor.com")
        self.assertEqual(test_html[0], test_html_constants.TEST_FOOTER)
        self.assertEqual(test_html[1], "<!-- footer -->")

    def test_build_rows_block(self):
        """Is rows block created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_rows_block(1, "block title", 2, ["row1", "row2"])
        self.assertEqual(test_html[0], test_html_constants.TEST_ROWS_BLOCK)
        self.assertEqual(test_html[1], "<!-- column1 -->")

    def test_build_text_block(self):
        """Is text block created succesfully?"""
        self.title = "test"
        empty_sheet = html_builder.HtmlSheet(self.title, None)
        test_html = empty_sheet.build_text_block(2, "block title", "text text text")
        self.assertEqual(test_html[0], test_html_constants.TEST_TEXT_BLOCK)
        self.assertEqual(test_html[1], "<!-- column2 -->")

    def tearDown(self):
        """Removes  extra files generated during tests

        Args:
            file_name (str): File to be removed.
        """
        try:
            #os.remove(self.test_file_name + ".html")
            pass
        except OSError as oserr:
            print(oserr)

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before># Copyright (c) 2016 Tigera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys

import netaddr
from netaddr import AddrFormatError, IPAddress
from pycalico.datastore_datatypes import IPPool
from pycalico.ipam import IPAMClient
from pycalico.util import get_host_ips, validate_asn

DEFAULT_IPV4_POOL = IPPool("192.168.0.0/16")
DEFAULT_IPV6_POOL = IPPool("fd80:24e2:f998:72d6::/64")

def _find_pool(ip_addr, ipv4_pools):
    """
    Find the pool containing the given IP.

    :param ip_addr:  IP address to find.
    :param ipv4_pools:  iterable containing IPPools.
    :return: The pool, or None if not found
    """
    for pool in ipv4_pools:
        if ip_addr in pool.cidr:
            return pool
    else:
        return None

def _ensure_host_tunnel_addr(ipv4_pools, ipip_pools):
    """
    Ensure the host has a valid IP address for its IPIP tunnel device.

    This must be an IP address claimed from one of the IPIP pools.
    Handles re-allocating the address if it finds an existing address
    that is not from an IPIP pool.

    :param ipv4_pools: List of all IPv4 pools.
    :param ipip_pools: List of IPIP-enabled pools.
    """
    ip_addr = _get_host_tunnel_ip()
    if ip_addr:
        # Host already has a tunnel IP assigned, verify that it's still valid.
        pool = _find_pool(ip_addr, ipv4_pools)
        if pool and not pool.ipip:
            # No longer an IPIP pool. Release the IP, it's no good to us.
            client.release_ips({ip_addr})
            ip_addr = None
        elif not pool:
            # Not in any IPIP pool.  IP must be stale.  Since it's not in any
            # pool, we can't release it.
            ip_addr = None
    if not ip_addr:
        # Either there was no IP or the IP needs to be replaced.  Try to
        # get an IP from one of the IPIP-enabled pools.
        _assign_host_tunnel_addr(ipip_pools)

def _assign_host_tunnel_addr(ipip_pools):
    """
    Claims an IPIP-enabled IP address from the first pool with some
    space.

    Stores the result in the host's config as its tunnel address.

    Exits on failure.
    :param ipip_pools:  List of IPPools to search for an address.
    """
    for ipip_pool in ipip_pools:
        v4_addrs, _ = client.auto_assign_ips(
            num_v4=1, num_v6=0,
            handle_id=None,
            attributes={},
            pool=(ipip_pool, None),
            host=hostname
        )
        if v4_addrs:
            # Successfully allocated an address.  Unpack the list.
            [ip_addr] = v4_addrs
            break
    else:
        # Failed to allocate an address, the pools must be full.
        print "Failed to allocate an IP address from an IPIP-enabled pool " \
            "for the host's IPIP tunnel device.  Pools are likely " \
            "exhausted."

        sys.exit(1)
    # If we get here, we've allocated a new IPIP-enabled address,
    # Store it in etcd so that Felix will pick it up.
    client.set_per_host_config(hostname, "IpInIpTunnelAddr",
                               str(ip_addr))

def _remove_host_tunnel_addr():
    """
    Remove any existing IP address for this host's IPIP tunnel device.

    Idempotent; does nothing if there is no IP assigned.  Releases the
    IP from IPAM.
    """
    ip_addr = _get_host_tunnel_ip()
    if ip_addr:
        client.release_ips({ip_addr})
    client.remove_per_host_config(hostname, "IpInIpTunnelAddr")

def _get_host_tunnel_ip():
    """
    :return: The IPAddress of the host's IPIP tunnel or None if not
             present/invalid.
    """
    raw_addr = client.get_per_host_config(hostname, "IpInIpTunnelAddr")
    try:
        ip_addr = IPAddress(raw_addr)
    except (AddrFormatError, ValueError, TypeError):
        # Either there's no address or the data is bad.  Treat as missing.
        ip_addr = None
    return ip_addr

def error_if_bgp_ip_conflict(ip, ip6):
    """
    Prints an error message and exits if either of the IPv4 or IPv6 addresses
    is already in use by another calico BGP host.

    :param ip: User-provided IPv4 address to start this node with.
    :param ip6: User-provided IPv6 address to start this node with.
    :return: Nothing
    """
    ip_list = []
    if ip:
        ip_list.append(ip)
    if ip6:
        ip_list.append(ip6)
    try:
        # Get hostname of host that already uses the given IP, if it exists
        ip_conflicts = client.get_hostnames_from_ips(ip_list)
    except KeyError:
        # No hosts have been configured in etcd, so there cannot be a conflict
        return

    if ip_conflicts.keys():
        ip_error = "ERROR: IP address %s is already in use by host %s. " \
                   "Calico requires each compute host to have a unique IP. " \
                   "If this is your first time running the Calico node on " \
                   "this host, ensure that another host is not already using " \
                   "the same IP address."
        try:
            if ip_conflicts[ip] != hostname:
                ip_error = ip_error % (ip, str(ip_conflicts[ip]))
                print ip_error
                sys.exit(1)
        except KeyError:
            # IP address was not found in ip-host dictionary
            pass
        try:
            if ip6 and ip_conflicts[ip6] != hostname:
                ip_error = ip_error % (ip6, str(ip_conflicts[ip6]))
                print ip_error
                sys.exit(1)
        except KeyError:
            # IP address was not found in ip-host dictionary
            pass

def warn_if_unknown_ip(ip, ip6):
    """
    Prints a warning message if the IP addresses are not assigned to interfaces
    on the current host.

    :param ip: IPv4 address which should be present on the host.
    :param ip6: IPv6 address which should be present on the host.
    :return: None
    """
    if ip and IPAddress(ip) not in get_host_ips(version=4, exclude=["docker0"]):
        print "WARNING: Could not confirm that the provided IPv4 address is" \
              " assigned to this host."

    if ip6 and IPAddress(ip6) not in get_host_ips(version=6,
                                                  exclude=["docker0"]):
        print "WARNING: Could not confirm that the provided IPv6 address is" \
              " assigned to this host."

def warn_if_hostname_conflict(ip):
    """
    Prints a warning message if it seems like an existing host is already running
    calico using this hostname.

    :param ip: User-provided or detected IP address to start this node with.
    :return: Nothing
    """
    try:
        current_ipv4, _ = client.get_host_bgp_ips(hostname)
    except KeyError:
        # No other machine has registered configuration under this hostname.
        # This must be a new host with a unique hostname, which is the
        # expected behavior.
        pass
    else:
        if current_ipv4 != "" and current_ipv4 != ip:
            hostname_warning = "WARNING: Hostname '%s' is already in use " \
                               "with IP address %s. Calico requires each " \
                               "compute host to have a unique hostname. " \
                               "If this is your first time running " \
                               "the Calico node on this host, ensure " \
                               "that another host is not already using the " \
                               "same hostname." % (hostname, current_ipv4)
            print hostname_warning

def main():
    ip = os.getenv("IP")
    ip = ip or None
    if ip and not netaddr.valid_ipv4(ip):
        print "IP environment (%s) is not a valid IPv4 address." % ip
        sys.exit(1)

    ip6 = os.getenv("IP6")
    ip6 = ip6 or None
    if ip6 and not netaddr.valid_ipv6(ip6):
        print "IP6 environment (%s) is not a valid IPv6 address." % ip6
        sys.exit(1)

    as_num = os.getenv("AS")
    as_num = as_num or None
    if as_num and not validate_asn(as_num):
        print "AS environment (%s) is not a AS number." % as_num
        sys.exit(1)

    # Get IP address of host, if none was specified
    if not ip:
        ips = get_host_ips(exclude=["^docker.*", "^cbr.*",
                                    "virbr.*", "lxcbr.*", "veth.*",
                                    "cali.*", "tunl.*", "flannel.*"])
        try:
            ip = str(ips.pop())
        except IndexError:
            print "Couldn't autodetect a management IP address. Please " \
                  "provide an IP address by rerunning the container with the" \
                  " IP environment variable set."
            sys.exit(1)
        else:
            print "No IP provided. Using detected IP: %s" % ip

    # Write a startup environment file containing the IP address that may have
    # just been detected.
    # This is required because the confd templates expect to be able to fill in
    #  the IP by fetching it from the environment.
    with open('startup.env', 'w') as f:
        f.write("IP=" + ip)

    warn_if_hostname_conflict(ip)

    # Verify that IPs are not already in use by another host.
    error_if_bgp_ip_conflict(ip, ip6)

    # Verify that the chosen IP exists on the current host
    warn_if_unknown_ip(ip, ip6)

    if os.getenv("NO_DEFAULT_POOLS", "").lower() != "true":
        # Set up etcd
        ipv4_pools = client.get_ip_pools(4)
        ipv6_pools = client.get_ip_pools(6)

        # Create default pools if required
        if not ipv4_pools:
            client.add_ip_pool(4, DEFAULT_IPV4_POOL)

        # If the OS has not been built with IPv6 then the /proc config for IPv6
        # will not be present.
        if not ipv6_pools and os.path.exists('/proc/sys/net/ipv6'):
            client.add_ip_pool(6, DEFAULT_IPV6_POOL)

    client.ensure_global_config()
    client.create_host(hostname, ip, ip6, as_num)

    # If IPIP is enabled, the host requires an IP address for its tunnel
    # device, which is in an IPIP pool.  Without this, a host can't originate
    # traffic to a pool address because the response traffic would not be
    # routed via the tunnel (likely being dropped by RPF checks in the fabric).
    ipv4_pools = client.get_ip_pools(4)
    ipip_pools = [p for p in ipv4_pools if p.ipip]

    if ipip_pools:
        # IPIP is enabled, make sure the host has an address for its tunnel.
        _ensure_host_tunnel_addr(ipv4_pools, ipip_pools)
    else:
        # No IPIP pools, clean up any old address.
        _remove_host_tunnel_addr()

hostname = os.getenv("HOSTNAME")
client = IPAMClient()

if __name__ == "__main__":
    main()
<commit_msg>Default to socket.gethostname()<commit_after># Copyright (c) 2016 Tigera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import socket

import netaddr
from netaddr import AddrFormatError, IPAddress
from pycalico.datastore_datatypes import IPPool
from pycalico.ipam import IPAMClient
from pycalico.util import get_host_ips, validate_asn

DEFAULT_IPV4_POOL = IPPool("192.168.0.0/16")
DEFAULT_IPV6_POOL = IPPool("fd80:24e2:f998:72d6::/64")

def _find_pool(ip_addr, ipv4_pools):
    """
    Find the pool containing the given IP.

    :param ip_addr:  IP address to find.
    :param ipv4_pools:  iterable containing IPPools.
    :return: The pool, or None if not found
    """
    for pool in ipv4_pools:
        if ip_addr in pool.cidr:
            return pool
    else:
        return None

def _ensure_host_tunnel_addr(ipv4_pools, ipip_pools):
    """
    Ensure the host has a valid IP address for its IPIP tunnel device.

    This must be an IP address claimed from one of the IPIP pools.
    Handles re-allocating the address if it finds an existing address
    that is not from an IPIP pool.

    :param ipv4_pools: List of all IPv4 pools.
    :param ipip_pools: List of IPIP-enabled pools.
    """
    ip_addr = _get_host_tunnel_ip()
    if ip_addr:
        # Host already has a tunnel IP assigned, verify that it's still valid.
        pool = _find_pool(ip_addr, ipv4_pools)
        if pool and not pool.ipip:
            # No longer an IPIP pool. Release the IP, it's no good to us.
            client.release_ips({ip_addr})
            ip_addr = None
        elif not pool:
            # Not in any IPIP pool.  IP must be stale.  Since it's not in any
            # pool, we can't release it.
            ip_addr = None
    if not ip_addr:
        # Either there was no IP or the IP needs to be replaced.  Try to
        # get an IP from one of the IPIP-enabled pools.
        _assign_host_tunnel_addr(ipip_pools)

def _assign_host_tunnel_addr(ipip_pools):
    """
    Claims an IPIP-enabled IP address from the first pool with some
    space.

    Stores the result in the host's config as its tunnel address.

    Exits on failure.
    :param ipip_pools:  List of IPPools to search for an address.
    """
    for ipip_pool in ipip_pools:
        v4_addrs, _ = client.auto_assign_ips(
            num_v4=1, num_v6=0,
            handle_id=None,
            attributes={},
            pool=(ipip_pool, None),
            host=hostname
        )
        if v4_addrs:
            # Successfully allocated an address.  Unpack the list.
            [ip_addr] = v4_addrs
            break
    else:
        # Failed to allocate an address, the pools must be full.
        print "Failed to allocate an IP address from an IPIP-enabled pool " \
            "for the host's IPIP tunnel device.  Pools are likely " \
            "exhausted."

        sys.exit(1)
    # If we get here, we've allocated a new IPIP-enabled address,
    # Store it in etcd so that Felix will pick it up.
    client.set_per_host_config(hostname, "IpInIpTunnelAddr",
                               str(ip_addr))

def _remove_host_tunnel_addr():
    """
    Remove any existing IP address for this host's IPIP tunnel device.

    Idempotent; does nothing if there is no IP assigned.  Releases the
    IP from IPAM.
    """
    ip_addr = _get_host_tunnel_ip()
    if ip_addr:
        client.release_ips({ip_addr})
    client.remove_per_host_config(hostname, "IpInIpTunnelAddr")

def _get_host_tunnel_ip():
    """
    :return: The IPAddress of the host's IPIP tunnel or None if not
             present/invalid.
    """
    raw_addr = client.get_per_host_config(hostname, "IpInIpTunnelAddr")
    try:
        ip_addr = IPAddress(raw_addr)
    except (AddrFormatError, ValueError, TypeError):
        # Either there's no address or the data is bad.  Treat as missing.
        ip_addr = None
    return ip_addr

def error_if_bgp_ip_conflict(ip, ip6):
    """
    Prints an error message and exits if either of the IPv4 or IPv6 addresses
    is already in use by another calico BGP host.

    :param ip: User-provided IPv4 address to start this node with.
    :param ip6: User-provided IPv6 address to start this node with.
    :return: Nothing
    """
    ip_list = []
    if ip:
        ip_list.append(ip)
    if ip6:
        ip_list.append(ip6)
    try:
        # Get hostname of host that already uses the given IP, if it exists
        ip_conflicts = client.get_hostnames_from_ips(ip_list)
    except KeyError:
        # No hosts have been configured in etcd, so there cannot be a conflict
        return

    if ip_conflicts.keys():
        ip_error = "ERROR: IP address %s is already in use by host %s. " \
                   "Calico requires each compute host to have a unique IP. " \
                   "If this is your first time running the Calico node on " \
                   "this host, ensure that another host is not already using " \
                   "the same IP address."
        try:
            if ip_conflicts[ip] != hostname:
                ip_error = ip_error % (ip, str(ip_conflicts[ip]))
                print ip_error
                sys.exit(1)
        except KeyError:
            # IP address was not found in ip-host dictionary
            pass
        try:
            if ip6 and ip_conflicts[ip6] != hostname:
                ip_error = ip_error % (ip6, str(ip_conflicts[ip6]))
                print ip_error
                sys.exit(1)
        except KeyError:
            # IP address was not found in ip-host dictionary
            pass

def warn_if_unknown_ip(ip, ip6):
    """
    Prints a warning message if the IP addresses are not assigned to interfaces
    on the current host.

    :param ip: IPv4 address which should be present on the host.
    :param ip6: IPv6 address which should be present on the host.
    :return: None
    """
    if ip and IPAddress(ip) not in get_host_ips(version=4, exclude=["docker0"]):
        print "WARNING: Could not confirm that the provided IPv4 address is" \
              " assigned to this host."

    if ip6 and IPAddress(ip6) not in get_host_ips(version=6,
                                                  exclude=["docker0"]):
        print "WARNING: Could not confirm that the provided IPv6 address is" \
              " assigned to this host."

def warn_if_hostname_conflict(ip):
    """
    Prints a warning message if it seems like an existing host is already running
    calico using this hostname.

    :param ip: User-provided or detected IP address to start this node with.
    :return: Nothing
    """
    try:
        current_ipv4, _ = client.get_host_bgp_ips(hostname)
    except KeyError:
        # No other machine has registered configuration under this hostname.
        # This must be a new host with a unique hostname, which is the
        # expected behavior.
        pass
    else:
        if current_ipv4 != "" and current_ipv4 != ip:
            hostname_warning = "WARNING: Hostname '%s' is already in use " \
                               "with IP address %s. Calico requires each " \
                               "compute host to have a unique hostname. " \
                               "If this is your first time running " \
                               "the Calico node on this host, ensure " \
                               "that another host is not already using the " \
                               "same hostname." % (hostname, current_ipv4)
            print hostname_warning

def main():
    ip = os.getenv("IP")
    ip = ip or None
    if ip and not netaddr.valid_ipv4(ip):
        print "IP environment (%s) is not a valid IPv4 address." % ip
        sys.exit(1)

    ip6 = os.getenv("IP6")
    ip6 = ip6 or None
    if ip6 and not netaddr.valid_ipv6(ip6):
        print "IP6 environment (%s) is not a valid IPv6 address." % ip6
        sys.exit(1)

    as_num = os.getenv("AS")
    as_num = as_num or None
    if as_num and not validate_asn(as_num):
        print "AS environment (%s) is not a AS number." % as_num
        sys.exit(1)

    # Get IP address of host, if none was specified
    if not ip:
        ips = get_host_ips(exclude=["^docker.*", "^cbr.*",
                                    "virbr.*", "lxcbr.*", "veth.*",
                                    "cali.*", "tunl.*", "flannel.*"])
        try:
            ip = str(ips.pop())
        except IndexError:
            print "Couldn't autodetect a management IP address. Please " \
                  "provide an IP address by rerunning the container with the" \
                  " IP environment variable set."
            sys.exit(1)
        else:
            print "No IP provided. Using detected IP: %s" % ip

    # Write a startup environment file containing the IP address that may have
    # just been detected.
    # This is required because the confd templates expect to be able to fill in
    # some templates by fetching them from the environment.
    with open('startup.env', 'w') as f:
        f.write("IP=%s\n" % ip)
        f.write("HOSTNAME=%s\n" % hostname)

    warn_if_hostname_conflict(ip)

    # Verify that IPs are not already in use by another host.
    error_if_bgp_ip_conflict(ip, ip6)

    # Verify that the chosen IP exists on the current host
    warn_if_unknown_ip(ip, ip6)

    if os.getenv("NO_DEFAULT_POOLS", "").lower() != "true":
        # Set up etcd
        ipv4_pools = client.get_ip_pools(4)
        ipv6_pools = client.get_ip_pools(6)

        # Create default pools if required
        if not ipv4_pools:
            client.add_ip_pool(4, DEFAULT_IPV4_POOL)

        # If the OS has not been built with IPv6 then the /proc config for IPv6
        # will not be present.
        if not ipv6_pools and os.path.exists('/proc/sys/net/ipv6'):
            client.add_ip_pool(6, DEFAULT_IPV6_POOL)

    client.ensure_global_config()
    client.create_host(hostname, ip, ip6, as_num)

    # If IPIP is enabled, the host requires an IP address for its tunnel
    # device, which is in an IPIP pool.  Without this, a host can't originate
    # traffic to a pool address because the response traffic would not be
    # routed via the tunnel (likely being dropped by RPF checks in the fabric).
    ipv4_pools = client.get_ip_pools(4)
    ipip_pools = [p for p in ipv4_pools if p.ipip]

    if ipip_pools:
        # IPIP is enabled, make sure the host has an address for its tunnel.
        _ensure_host_tunnel_addr(ipv4_pools, ipip_pools)
    else:
        # No IPIP pools, clean up any old address.
        _remove_host_tunnel_addr()

# Try the HOSTNAME environment variable, but default to
# the socket.gethostname() value if unset.
hostname = os.getenv("HOSTNAME")
if not hostname:
    hostname = socket.gethostname()

client = IPAMClient()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>try:
    from http.server import SimpleHTTPRequestHandler, HTTPServer
except ImportError:  # Python 2
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    from BaseHTTPServer import HTTPServer

import os
from multiprocessing import Process

import http_crawler

serving = False

def serve():
    global serving

    if serving:
        return

    serving = True

    def _serve(dir, port):
        base_dir = os.path.join('tests', dir)
        os.chdir(base_dir)
        server = HTTPServer(('', port), SimpleHTTPRequestHandler)
        server.serve_forever()

    Process(target=_serve, args=('site', 8000), daemon=True).start()
    Process(target=_serve, args=('external-site', 8001), daemon=True).start()

def test_crawl():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/'))

    assert len(rsps) == 13

    urls = [rsp.url for rsp in rsps]

    assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
        'http://localhost:8001/pages/page-1/',
    }

def test_crawl_follow_external_links_false():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/',
                follow_external_links=False))

    assert len(rsps) == 12

    urls = [rsp.url for rsp in rsps]

    assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
    }

def test_crawl_ignore_fragments_false():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/',
                ignore_fragments=False))

    # assert len(rsps) == 14

    urls = [rsp.url for rsp in rsps]

    # assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-1/#anchor',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-2/#anchor',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/pages/page-3/#anchor',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
        'http://localhost:8001/pages/page-1/',
        'http://localhost:8001/pages/page-1/#anchor',
    }

def test_extract_urls_from_html():
    with open(os.path.join('tests', 'site', 'index.html')) as f:
        content = f.read()

    urls = http_crawler.extract_urls_from_html(content)

    assert len(urls) == 13
    assert set(urls) == {
        '/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-1/#anchor',
        'http://localhost:8001/pages/page-1/',
        'http://localhost:8001/pages/page-1/#anchor',
        '/pages/page-2/',
        '/pages/page-2/#anchor',
        'pages/page-3/',
        'pages/page-3/#anchor',
        '/assets/styles.css',
        '/assets/image.jpg',
        '/assets/script.js',
        'mailto:example@example.org',
    }

def test_extract_urls_from_css():
    with open(os.path.join('tests', 'site', 'assets', 'styles.css')) as f:
        content = f.read()

    urls = http_crawler.extract_urls_from_css(content)

    assert len(urls) == 5
    assert set(urls) == {
        '/assets/styles-2.css',
        '/assets/tile-1.jpg',
        '/assets/somefont.eot',
        '/assets/somefont.ttf',
    }
<commit_msg>The daemon argument was only added in Python 3.3<commit_after>try:
    from http.server import SimpleHTTPRequestHandler, HTTPServer
except ImportError:  # Python 2
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    from BaseHTTPServer import HTTPServer

import os
from multiprocessing import Process

import http_crawler

serving = False

def serve():
    global serving

    if serving:
        return

    serving = True

    def _serve(dir, port):
        base_dir = os.path.join('tests', dir)
        os.chdir(base_dir)
        server = HTTPServer(('', port), SimpleHTTPRequestHandler)
        server.serve_forever()

    proc_site = Process(target=_serve, args=('site', 8000))
    proc_site.daemon = True
    proc_site.start()

    proc_external_site = Process(target=_serve, args=('external-site', 8001))
    proc_external_site.daemon = True
    proc_external_site.start()

def test_crawl():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/'))

    assert len(rsps) == 13

    urls = [rsp.url for rsp in rsps]

    assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
        'http://localhost:8001/pages/page-1/',
    }

def test_crawl_follow_external_links_false():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/',
                follow_external_links=False))

    assert len(rsps) == 12

    urls = [rsp.url for rsp in rsps]

    assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
    }

def test_crawl_ignore_fragments_false():
    serve()

    rsps = list(http_crawler.crawl('http://localhost:8000/',
                ignore_fragments=False))

    # assert len(rsps) == 14

    urls = [rsp.url for rsp in rsps]

    # assert len(urls) == len(set(urls))
    assert set(urls) == {
        'http://localhost:8000/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-1/#anchor',
        'http://localhost:8000/pages/page-2/',
        'http://localhost:8000/pages/page-2/#anchor',
        'http://localhost:8000/pages/page-3/',
        'http://localhost:8000/pages/page-3/#anchor',
        'http://localhost:8000/assets/styles.css',
        'http://localhost:8000/assets/styles-2.css',
        'http://localhost:8000/assets/image.jpg',
        'http://localhost:8000/assets/script.js',
        'http://localhost:8000/assets/tile-1.jpg',
        'http://localhost:8000/assets/tile-2.jpg',
        'http://localhost:8000/assets/somefont.eot',
        'http://localhost:8000/assets/somefont.ttf',
        'http://localhost:8001/pages/page-1/',
        'http://localhost:8001/pages/page-1/#anchor',
    }

def test_extract_urls_from_html():
    with open(os.path.join('tests', 'site', 'index.html')) as f:
        content = f.read()

    urls = http_crawler.extract_urls_from_html(content)

    assert len(urls) == 13
    assert set(urls) == {
        '/',
        'http://localhost:8000/pages/page-1/',
        'http://localhost:8000/pages/page-1/#anchor',
        'http://localhost:8001/pages/page-1/',
        'http://localhost:8001/pages/page-1/#anchor',
        '/pages/page-2/',
        '/pages/page-2/#anchor',
        'pages/page-3/',
        'pages/page-3/#anchor',
        '/assets/styles.css',
        '/assets/image.jpg',
        '/assets/script.js',
        'mailto:example@example.org',
    }

def test_extract_urls_from_css():
    with open(os.path.join('tests', 'site', 'assets', 'styles.css')) as f:
        content = f.read()

    urls = http_crawler.extract_urls_from_css(content)

    assert len(urls) == 5
    assert set(urls) == {
        '/assets/styles-2.css',
        '/assets/tile-1.jpg',
        '/assets/somefont.eot',
        '/assets/somefont.ttf',
    }
<|endoftext|>


<commit_before>"""
ZenCODE's Music Library Checker
===============================

This module checks that the specified albums has properly structured music
albums using the following conventions:

    <Artists>\<Album>\<Track> - <Title>.<ext>
"""
__author__ = 'Richard Larkin a.k.a. ZenCODE'

from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.lang import Builder
from os import sep, listdir, path, walk
from kivy.uix.image import Image
from kivy.uix.label import Label
from kivy.properties import (NumericProperty, ListProperty, ObjectProperty,
                             BooleanProperty)
from kivy.event import EventDispatcher

Builder.load_file('lib_checker.kv')

class MusicLib(EventDispatcher):
    """
    This class houses metadata about our music collection.
    """
    source = u'/media/ZenOne/Zen/Music/CD'  # PC. Linux
    # source = u'/media/richard/ZenUno/Zen/Music/MP3'  Laptop, linux
    # source = r"d:\Zen\Music\MP3"  # PC, Windows

    albums = ListProperty([])
    '''
    A list of dictionaries containing data for each album found. Entries are as
    follows:

        'folder': the full folder path

    This is the minimum it will contain. Once processed it contains:

        'artist': the artist
        'album': the album
        'tracks': a list of sorted file names
        'images': a list of images found
        'warning': a list list of warnings found
    '''
    max_albums = 10

    def __init__(self, **kwargs):
        """ The class constructor. """
        super(MusicLib, self).__init__(**kwargs)
        self.albums = MusicLib._get_albums(MusicLib.source,
                                            [],
                                            MusicLib.max_albums)

    @staticmethod
    def _get_albums(folder, albums, max_albums):
        """
        Process the *folder*, appending to the *albums* list adding only
        *max_albums* + 1 albums.
        """
        if len(albums) > max_albums:
            return albums

        def is_not_in_albums(_folder, _albums):
            return all([_folder != album['folder'] for album in _albums])

        for root, sub_albums, files in walk(folder):
            for i in sub_albums:
                MusicLib._get_albums(path.join(folder, i), albums, max_albums)

            if len(sub_albums) == 0 and len(files) > 0:
                if is_not_in_albums(root, albums):
                    albums.append({'folder': root})

                if len(albums) > max_albums:
                    return albums
        return albums

    @staticmethod
    def _populate_album(album):
        """
        Populates the indexed *folders" dictionary with album info and data.
        """
        folder = album['folder']
        parts = folder.split(sep)
        artist, album_name = parts[-2], parts[-1]
        images, tracks, warnings = [], [], []
        files = [file_name for file_name in listdir(folder)]

        for my_file in sorted(files):
            ext = my_file[-4:]
            if ext in [".jpg", ".png", ".gif", "jpeg"]:
                images.append(path.join(folder, my_file))
            elif ext in [".mp3", ".ogg"]:
                tracks.append(my_file)
            else:
                warnings.append("Unrecognized file: {0}".format(my_file))

        album.update({'artist': artist,
                      'album': album_name,
                      'tracks': tracks,
                      'images': images,
                      'warnings': warnings})

    def get_row_item(self, index, controller):
        """
        Build and return a formatted DisplayItem for the folder.
        """
        # Initialize
        album = self.albums[index]

        if 'tracks' not in album.keys():
            self._populate_album(album)
        di = DisplayItem()
        add_label = di.ids.labels.add_widget

        # Add header
        add_label(Label(
            text=u"[b][color=#FFFF00]{0} : {1}[/color][/b]".format(
                 album['artist'], album['album']),
            markup=True))

        # Add images + warnings
        if len(album['images']) == 0:
            di.ids.images.add_widget(Image(source="images/album.png"))
        else:
            [di.ids.images.add_widget(Image(source=image, allow_stretch=True))
             for image in album['images']]
        [add_label(Label(
            text=u"[color=#FF0000]{0}[/color]".format(warn)))
            for warn in album['warnings']]

        # Add tracks
        for k, track in enumerate(album['tracks']):
            playing = bool(
                controller.playing_album == controller.album_index and
                controller.playing_track == k)
            pl_label = PlaylistLabel(
                controller=controller,
                text=track,
                album_index=index,
                track_index=k)
            add_label(pl_label)
            if playing:
                controller.set_selected(pl_label)

        return di

class DisplayItem(BoxLayout):
    """ This class represent an individual album found in the search. """

class PlaylistLabel(Label):
    """
    This class is used to represent each playlist item.
    """
    album_index = NumericProperty()
    track_index = NumericProperty()
    playing = BooleanProperty(False)
    controller = ObjectProperty()

    back_colour = ListProperty([0, 0, 0, 0])

    def on_playing(self, widget, value):
        """ Respond to the change in state. """
        self.back_colour = [0.5, 0.5, 1, 0.3] if value else [0, 0, 0, 0]

    def on_touch_down(self, touch):
        """ Handle the event. """
        if self.collide_point(*touch.pos):
            touch.grab(self)

    def on_touch_up(self, touch):
        """ Handle the event. """
        if touch.grab_current is self:
            touch.ungrab(self)
            if self.collide_point(*touch.pos):
                self.controller.play_track(self)

class Controller(EventDispatcher):
    """
    This class houses the logic and management of the current album and
    track index as well as the management of the audio.
    """
    album_index = NumericProperty(0)  # Currently displayed
    track_index = NumericProperty(0)  # Currently displayed
    playing_album = NumericProperty(0)  # Currently playing
    playing_track = NumericProperty(-1)  # Currently playing

    music_lib = ObjectProperty(None)
    album_screen = ObjectProperty(None)
    current_pl_label = None

    def play_track(self, pl_label):
        """ Play the track linked to be the PlaylistLabel. """
        self.playing_album = pl_label.album_index
        self.playing_track = pl_label.track_index
        self.set_selected(pl_label)
        print "play track " + self.music_lib.albums[self.playing_album][
            'tracks'][self.playing_track]

    def move_next(self, advance):
        """
        Play the next track if advance in true, the previous track if False.
        If advance is None, the track info is not changed.
        """
        if advance is None:
            return

        album_index, track_index = self.album_index, self.track_index
        albums = self.music_lib.albums
        if advance:
            if album_index < len(albums):
                album_index = (album_index + 1) % len(albums)
                track_index = 0
        else:
            if 0 < album_index:
                album_index = (len(albums) + album_index - 1) % len(albums)
                track_index = 0
        self.album_index, self.track_index = album_index, track_index

    def get_currrent_album(self):
        """ Build and return a DisplayItem for the current album. """
        return self.music_lib.get_row_item(self.album_index, self)

    def set_selected(self, pl_label):
        """
        Set the PlaylistLabel as the one linked to the currently playing
        track.
        """
        if self.current_pl_label is not None:
            self.current_pl_label.playing = False

        self.current_pl_label = pl_label
        pl_label.playing = True

class AlbumScreen(BoxLayout):
    """"
    The main screen showing a list of albums found.
    """
    music_lib = ObjectProperty(None)
    controller = ObjectProperty(None)

    def __init__(self, **kwargs):
        super(AlbumScreen, self).__init__(**kwargs)
        self.music_lib = MusicLib()
        self.controller = Controller(music_lib=self.music_lib,
                                     album_screen=self)
        self.show_album()

    def show_album(self, advance=None):
        """
        Begins the timed display of albums. If *advance* is True, the next item
        is shown. If False, it moves back. If not specified, the current album
        is shown.
        """
        self.controller.move_next(advance)
        container = self.ids.row_container
        container.clear_widgets()
        if len(self.music_lib.albums) > self.controller.album_index:
            self.ids.header_label.text = "Music Library ({0} / {1})".format(
                self.controller.album_index + 1, len(self.music_lib.albums))

            container.add_widget(self.controller.get_currrent_album())
        else:
            container.add_widget(Label(text="No albums found"))

    def show_next(self):
        """ Show the next album. """
        self.show_album(True)

    def show_previous(self):
        """ Show the next album. """
        self.show_album(False)

class FolderChecker(App):
    def build(self):
        return AlbumScreen()

if __name__ == '__main__':
    FolderChecker().run()
<commit_msg>Audio hooked up<commit_after>"""
ZenCODE's Music Library Checker
===============================

This module checks that the specified albums has properly structured music
albums using the following conventions:

    <Artists>\<Album>\<Track> - <Title>.<ext>
"""
__author__ = 'Richard Larkin a.k.a. ZenCODE'

from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.lang import Builder
from os import sep, listdir, path, walk
from kivy.uix.image import Image
from kivy.uix.label import Label
from kivy.properties import (NumericProperty, ListProperty, ObjectProperty,
                             BooleanProperty)
from kivy.event import EventDispatcher
from audioplayer import SoundLoader

Builder.load_file('lib_checker.kv')

class MusicLib(EventDispatcher):
    """
    This class houses metadata about our music collection.
    """
    source = u'/media/ZenOne/Zen/Music/CD'  # PC. Linux
    # source = u'/media/richard/ZenUno/Zen/Music/MP3'  Laptop, linux
    # source = r"d:\Zen\Music\MP3"  # PC, Windows

    albums = ListProperty([])
    '''
    A list of dictionaries containing data for each album found. Entries are as
    follows:

        'folder': the full folder path

    This is the minimum it will contain. Once processed it contains:

        'artist': the artist
        'album': the album
        'tracks': a list of sorted file names
        'images': a list of images found
        'warning': a list list of warnings found
    '''
    max_albums = 10

    def __init__(self, **kwargs):
        """ The class constructor. """
        super(MusicLib, self).__init__(**kwargs)
        self.albums = MusicLib._get_albums(MusicLib.source,
                                            [],
                                            MusicLib.max_albums)

    @staticmethod
    def _get_albums(folder, albums, max_albums):
        """
        Process the *folder*, appending to the *albums* list adding only
        *max_albums* + 1 albums.
        """
        if len(albums) > max_albums:
            return albums

        def is_not_in_albums(_folder, _albums):
            return all([_folder != album['folder'] for album in _albums])

        for root, sub_albums, files in walk(folder):
            for i in sub_albums:
                MusicLib._get_albums(path.join(folder, i), albums, max_albums)

            if len(sub_albums) == 0 and len(files) > 0:
                if is_not_in_albums(root, albums):
                    albums.append({'folder': root})

                if len(albums) > max_albums:
                    return albums
        return albums

    @staticmethod
    def _populate_album(album):
        """
        Populates the indexed *folders" dictionary with album info and data.
        """
        folder = album['folder']
        parts = folder.split(sep)
        artist, album_name = parts[-2], parts[-1]
        images, tracks, warnings = [], [], []
        files = [file_name for file_name in listdir(folder)]

        for my_file in sorted(files):
            ext = my_file[-4:]
            if ext in [".jpg", ".png", ".gif", "jpeg"]:
                images.append(path.join(folder, my_file))
            elif ext in [".mp3", ".ogg"]:
                tracks.append(my_file)
            else:
                warnings.append("Unrecognized file: {0}".format(my_file))

        album.update({'artist': artist,
                      'album': album_name,
                      'tracks': tracks,
                      'images': images,
                      'warnings': warnings})

    def get_row_item(self, index, controller):
        """
        Build and return a formatted DisplayItem for the folder.
        """
        # Initialize
        album = self.albums[index]

        if 'tracks' not in album.keys():
            self._populate_album(album)
        di = DisplayItem()
        add_label = di.ids.labels.add_widget

        # Add header
        add_label(Label(
            text=u"[b][color=#FFFF00]{0} : {1}[/color][/b]".format(
                 album['artist'], album['album']),
            markup=True))

        # Add images + warnings
        if len(album['images']) == 0:
            di.ids.images.add_widget(Image(source="images/album.png"))
        else:
            [di.ids.images.add_widget(Image(source=image, allow_stretch=True))
             for image in album['images']]
        [add_label(Label(
            text=u"[color=#FF0000]{0}[/color]".format(warn)))
            for warn in album['warnings']]

        # Add tracks
        for k, track in enumerate(album['tracks']):
            playing = bool(
                controller.playing_album == controller.album_index and
                controller.playing_track == k)
            pl_label = PlaylistLabel(
                controller=controller,
                text=track,
                album_index=index,
                track_index=k)
            add_label(pl_label)
            if playing:
                controller.set_selected(pl_label)

        return di

class DisplayItem(BoxLayout):
    """ This class represent an individual album found in the search. """

class PlaylistLabel(Label):
    """
    This class is used to represent each playlist item.
    """
    album_index = NumericProperty()
    track_index = NumericProperty()
    playing = BooleanProperty(False)
    controller = ObjectProperty()

    back_colour = ListProperty([0, 0, 0, 0])

    def on_playing(self, widget, value):
        """ Respond to the change in state. """
        self.back_colour = [0.5, 0.5, 1, 0.3] if value else [0, 0, 0, 0]

    def on_touch_down(self, touch):
        """ Handle the event. """
        if self.collide_point(*touch.pos):
            touch.grab(self)

    def on_touch_up(self, touch):
        """ Handle the event. """
        if touch.grab_current is self:
            touch.ungrab(self)
            if self.collide_point(*touch.pos):
                self.controller.play_track(self)

class Controller(EventDispatcher):
    """
    This class houses the logic and management of the current album and
    track index as well as the management of the audio.
    """
    album_index = NumericProperty(0)  # Currently displayed
    track_index = NumericProperty(0)  # Currently displayed
    playing_album = NumericProperty(0)  # Currently playing
    playing_track = NumericProperty(-1)  # Currently playing

    music_lib = ObjectProperty(None)
    album_screen = ObjectProperty(None)
    current_pl_label = None

    sound = None

    def play_track(self, pl_label):
        """ Play the track linked to be the PlaylistLabel. """
        self.playing_album = pl_label.album_index
        self.playing_track = pl_label.track_index
        self.set_selected(pl_label)
        print "play track " + self.music_lib.albums[self.playing_album][
            'tracks'][self.playing_track]

        if not self.sound is None:
            self.sound.stop()

        album = self.music_lib.albums[self.playing_album]
        full_path = path.join(album['folder'], album['tracks'][
            self.playing_track])
        self.sound = SoundLoader.load(full_path)
        self.sound.play()

    def move_next(self, advance):
        """
        Play the next track if advance in true, the previous track if False.
        If advance is None, the track info is not changed.
        """
        if advance is None:
            return

        album_index, track_index = self.album_index, self.track_index
        albums = self.music_lib.albums
        if advance:
            if album_index < len(albums):
                album_index = (album_index + 1) % len(albums)
                track_index = 0
        else:
            if 0 < album_index:
                album_index = (len(albums) + album_index - 1) % len(albums)
                track_index = 0
        self.album_index, self.track_index = album_index, track_index

    def get_currrent_album(self):
        """ Build and return a DisplayItem for the current album. """
        return self.music_lib.get_row_item(self.album_index, self)

    def set_selected(self, pl_label):
        """
        Set the PlaylistLabel as the one linked to the currently playing
        track.
        """
        if self.current_pl_label is not None:
            self.current_pl_label.playing = False

        self.current_pl_label = pl_label
        pl_label.playing = True

class AlbumScreen(BoxLayout):
    """"
    The main screen showing a list of albums found.
    """
    music_lib = ObjectProperty(None)
    controller = ObjectProperty(None)

    def __init__(self, **kwargs):
        super(AlbumScreen, self).__init__(**kwargs)
        self.music_lib = MusicLib()
        self.controller = Controller(music_lib=self.music_lib,
                                     album_screen=self)
        self.show_album()

    def show_album(self, advance=None):
        """
        Begins the timed display of albums. If *advance* is True, the next item
        is shown. If False, it moves back. If not specified, the current album
        is shown.
        """
        self.controller.move_next(advance)
        container = self.ids.row_container
        container.clear_widgets()
        if len(self.music_lib.albums) > self.controller.album_index:
            self.ids.header_label.text = "Music Library ({0} / {1})".format(
                self.controller.album_index + 1, len(self.music_lib.albums))

            container.add_widget(self.controller.get_currrent_album())
        else:
            container.add_widget(Label(text="No albums found"))

    def show_next(self):
        """ Show the next album. """
        self.show_album(True)

    def show_previous(self):
        """ Show the next album. """
        self.show_album(False)

class FolderChecker(App):
    def build(self):
        return AlbumScreen()

if __name__ == '__main__':
    FolderChecker().run()
<|endoftext|>


<commit_before>#!/usr/bin/python
"""Extracts compile commands from go build output.

Reads either stdin or a specified input file, then extracts out
go compiler invocations, post-processes them to strip build directory
artifacts, and emits a separate shell script containing the invocations.
Useful for doing compiler reruns and re-invocations under the debugger.

"""

import getopt
import hashlib
import os
import re
import shlex
import subprocess
import sys

import script_utils as u

# Extract single cmd vs all cmds
flag_single = True

# Input and output file (if not specified, defaults to stdin/stdout)
flag_infile = None
flag_outfile = None

# Set up for gccgo debugging
flag_gccgo_gdb = False

# Captures first gccgo compilation line
gccgo_invocation = None
gccgo_location = None

# Drivers we encounter while scanning the build output
drivers = {}
driver_count = 0

# Experimental compile-in-parallel feature
flag_parfactor = 0

# Compile cmd line args to skip. Key is arg, val is skip count.
args_to_skip = {"-o": 1}

pfcount = 0
pftempfiles = []

def mktempname(salt, instance):
  """Create /tmp file name for compile output."""
  m = hashlib.md5()
  m.update(salt)
  hd = m.hexdigest()
  return "/tmp/%s.%d.err.txt" % (hd, instance)

def extract_line(outf, driver, driver_var, argstring, curdir):
  """Post-process a line."""
  global pfcount, gccgo_invocation, gccgo_location
  # Now filter the args.
  args = []
  skipcount = 0
  regsrc = re.compile(r"^(\S+)\.go$")
  raw_args = shlex.split(argstring)
  numraw = len(raw_args)
  for idx in range(0, numraw):
    arg = raw_args[idx]
    if skipcount:
      u.verbose(2, "skipping arg: %s" % arg)
      skipcount -= 1
      continue
    if arg in args_to_skip:
      sk = args_to_skip[arg]
      if idx + sk >= numraw:
        u.error("at argument %s (pos %d): unable to skip"
                "ahead %d, not enough args (line: "
                "%s" % (arg, idx, sk, " ".join(raw_args)))
      skipcount = sk
      u.verbose(2, "skipping arg: %s (skipcount set to %d)" % (arg, sk))
      continue
    args.append(arg)

  srcfile = args[-1]
  u.verbose(1, "srcfile is %s" % srcfile)
  if not regsrc.match(srcfile):
    u.warning("suspicious srcfile %s (no regex match)" % srcfile)

  extra = ""
  if flag_parfactor:
    line = driver . argstring
    tempfile = mktempname(line, pfcount)
    pftempfiles.append(tempfile)
    extra = "&> %s &" % tempfile
    pfcount += 1
  outf.write("${%s} %s $* %s\n" % (driver_var, " ".join(args), extra))
  u.verbose(0, "extracted compile cmd for %s" % raw_args[numraw-1])
  if flag_gccgo_gdb:
    gccgo_invocation = []
    gccgo_invocation.append(driver)
    gccgo_invocation.extend(args)
    gccgo_location = curdir
  if flag_single:
    return -1
  return 1

def perform_extract(inf, outf):
  """Read inf and extract compile cmd to outf."""
  global driver_count, pfcount, pftempfiles

  reggcg = re.compile(r"^(\S+/bin/gccgo)\s+(.+)$")
  reggc = re.compile(r"^(\S+/compile)\s+(.+)$")
  regcd = re.compile(r"^cd (\S+)\s*$")
  preamble_emitted = False
  pfcount = 0
  pftempfiles = []
  cdir = "."
  while True:
    line = inf.readline()
    if not line:
      break
    u.verbose(2, "line is %s" % line.strip())
    mcd = regcd.match(line)
    if mcd:
      cdir = mcd.group(1)
      continue
    mgc = reggc.match(line)
    mgcg = reggcg.match(line)
    if not mgc and not mgcg:
      continue
    if mgc:
      driver = mgc.group(1)
      argstring = mgc.group(2)
    else:
      driver = mgcg.group(1)
      argstring = mgcg.group(2)

    if not preamble_emitted:
      preamble_emitted = True
      outf.write("#!/bin/sh\n")
      outf.write("WORK=`pwd`\n")
    u.verbose(1, "matched: %s %s" % (driver, argstring))

    driver_var = "DRIVER%d" % driver_count
    if driver in drivers:
      driver_var = drivers[driver]
    else:
      outf.write("%s=%s\n" % (driver_var, driver))
      drivers[driver] = driver_var
      driver_count += 1
    if extract_line(outf, driver, driver_var, argstring, cdir) < 0:
      break
    if pfcount > flag_parfactor:
      outf.write("wait\n")
      outf.write("cat %s\n" % " ".join(pftempfiles))
      outf.write("rm %s\n" % " ".join(pftempfiles))
      pftempfiles = []
      pfcount = 0

  if pfcount:
    outf.write("wait\ncat")
    for t in pftempfiles:
      outf.write(" %s" % t)
    outf.write("\n")

def setup_gccgo_gdb():
  """Set up for gccgo debugging."""
  outfile = ".gccgo.err.txt"
  here = os.getcwd()
  if here != gccgo_location:
    u.warning("gccgo compile takes place in %s, "
              "not here (%s)" % (gccgo_location, here))
  objfile = "%s/.gccgo.tmp.o" % here
  os.chdir(gccgo_location)
  driver = gccgo_invocation[0]
  args = gccgo_invocation[1:]
  for idx in range(0, len(args)):
    if args[idx] == "$WORK":
      args[idx] = here
  cmd = ("%s -v -o %s %s" % (driver, objfile, " ".join(args)))
  u.verbose(1, "executing gdb setup cmd: %s" % cmd)
  u.docmderrout(cmd, outfile, True)
  os.chdir(here)
  try:
    inf = open(outfile, "rb")
  except IOError as e:
    u.error("internal error: unable to consume tmp "
            "file %s? error %s" % (outfile, e.strerror))
  lines = inf.readlines()
  inf.close()
  reg1 = re.compile(r"^\s*(\S+/go1)\s+(\S.+)$")
  found = False
  for line in lines:
    m = reg1.match(line)
    if m:
      go1exe = m.group(1)
      u.verbose(1, "go1 driver is %s" % go1exe)
      go1args = m.group(2)
      u.verbose(1, "go1 args: %s" % go1args)
      found = True
      # Create symlink
      if not os.path.exists("go1"):
        u.verbose(0, "symlinking to %s" % go1exe)
        os.symlink(go1exe, "go1")
      # Dump args
      u.verbose(0, "writing args to .gdbinit")
      try:
        outf = open(".gdbinit", "wb")
      except IOError as e:
        u.error("unable to open ./.gdbinit: "
                "%s" % (flag_outfile, e.strerror))
      outf.write("# gud-gdb --fullname ./go1\n")
      outf.write("set args %s" % go1args)
      outf.close()
      u.verbose(0, "starting emacs")
      subprocess.Popen(["emacs", ".gdbinit"])

def perform():
  """Main driver routine."""
  inf = sys.stdin
  outf = sys.stdout
  if flag_infile:
    try:
      inf = open(flag_infile, "rb")
    except IOError as e:
      u.error("unable to open input file %s: "
              "%s" % (flag_infile, e.strerror))
  if flag_outfile:
    try:
      outf = open(flag_outfile, "wb")
    except IOError as e:
      u.error("unable to open output file %s: "
              "%s" % (flag_outfile, e.strerror))
  perform_extract(inf, outf)
  if flag_infile:
    inf.close()
  if flag_outfile:
    outf.close()
  if flag_gccgo_gdb:
    setup_gccgo_gdb()

def usage(msgarg):
  """Print usage and exit."""
  if msgarg:
    sys.stderr.write("error: %s\n" % msgarg)
  print """\
    usage:  %s [options] < input > output

    options:
    -d    increase debug msg verbosity level
    -i F  read from input file F (presumably captured 'go build' output)
    -o G  write compile script to file G
    -a    extract all compile cmds (not just first one found)
    -j N  emit code to perform N compilations in parallel
    -G    set up for gccgo debugging; this invokes the compile script
          with -v and then parses the output to collect gccgo/go1 args

    """ % os.path.basename(sys.argv[0])
  sys.exit(1)

def parse_args():
  """Command line argument parsing."""
  global flag_infile, flag_outfile, flag_single, flag_parfactor
  global flag_gccgo_gdb

  try:
    optlist, args = getopt.getopt(sys.argv[1:], "dai:o:j:G")
  except getopt.GetoptError as err:
    # unrecognized option
    usage(str(err))

  if args:
    usage("extra unknown arguments")
  for opt, arg in optlist:
    if opt == "-d":
      u.increment_verbosity()
    elif opt == "-a":
      flag_single = False
    elif opt == "-G":
      flag_gccgo_gdb = True
    elif opt == "-i":
      flag_infile = arg
    elif opt == "-o":
      flag_outfile = arg
    elif opt == "-j":
      flag_parfactor = int(arg)
  if flag_single:
    flag_parfactor = 0

# Setup
u.setdeflanglocale()
parse_args()
perform()
<commit_msg>Revamp for better use with 'go test'.<commit_after>#!/usr/bin/python
"""Extracts compile commands from go build output.

Reads either stdin or a specified input file, then post-processes to
identify go/gccgo compiler invocations. Output is post-processed to
create a script that can be invoked to rerun the compile, or
optionally is examined to select out the gccgo compiler invocation.
Useful for doing compiler reruns and re-invocations under
the debugger.

"""

import getopt
import hashlib
import os
import re
import shlex
import subprocess
import sys

import script_utils as u

# Extract single cmd vs all cmds
flag_single = False

# Input and output file (if not specified, defaults to stdin/stdout)
flag_infile = None
flag_outfile = None

# place to which work dir should be relocated
flag_relocate = None

# Set up for gccgo debugging compilation of specified go src file
flag_gccgo_gdb = None

# Captures first gccgo compilation line
gccgo_invocation = None
gccgo_location = None

# workdir
workdir = None

# Drivers we encounter while scanning the build output
drivers = {}
driver_count = 0

def mktempname(salt, instance):
  """Create /tmp file name for compile output."""
  m = hashlib.md5()
  m.update(salt)
  hd = m.hexdigest()
  return "/tmp/%s.%d.err.txt" % (hd, instance)

def extract_line(outf, driver, driver_var, argstring, curdir):
  """Post-process a line."""
  global gccgo_invocation, gccgo_location
  # Now filter the args.
  args = []
  regparen = re.compile(r"^\-Wl,\-[\(\)]$")
  regsrc = re.compile(r"^(\S+)\.go$")
  raw_args = shlex.split(argstring)
  numraw = len(raw_args)
  gosrcfiles = {}
  srcfiles = []
  for idx in range(0, numraw):
    arg = raw_args[idx]
    mp = regparen.match(arg)
    if mp:
      arg = "'%s'" % arg
    if regsrc.match(arg):
      gosrcfiles[arg] = 1
      srcfiles.append(arg)
    args.append(arg)

  outf.write("${%s} %s $* \n" % (driver_var, " ".join(args)))
  if gosrcfiles:
    u.verbose(0, "extracted compile cmd for: %s" % " ".join(srcfiles))
  if flag_gccgo_gdb and flag_gccgo_gdb in gosrcfiles:
    u.verbose(0, "found gccgo compilation line "
              "including %s" % flag_gccgo_gdb)
    gccgo_invocation = []
    gccgo_invocation.append(driver)
    gccgo_invocation.extend(args)
    gccgo_location = curdir
  if flag_single:
    return -1
  return 1

def perform_extract(inf, outf):
  """Read inf and extract compile cmd to outf."""
  global driver_count, workdir

  regwrk = re.compile(r"^WORK=(\S+)$")
  reggcg = re.compile(r"^(\S+/bin/gccgo)\s+(.+)$")
  reggc = re.compile(r"^(\S+/compile)\s+(.+)$")
  regcd = re.compile(r"^cd (\S+)\s*$")
  regar = re.compile(r"^ar rc .+$")
  regcp = re.compile(r"^cp (\S+) (\S+)$")

  outf.write("#!/bin/sh\n")

  cdir = "."
  while True:
    line = inf.readline()
    if not line:
      break
    u.verbose(2, "line is %s" % line.strip())
    mwrk = regwrk.match(line)
    if mwrk:
      workdir = mwrk.group(1)
      if flag_relocate:
        u.verbose(0, "... relocating work dir %s to "
                  "%s" % (workdir, flag_relocate))
        if os.path.exists(flag_relocate):
          u.docmd("rm -rf %s" % flag_relocate)
        u.docmd("cp -r %s %s" % (workdir, flag_relocate))
        outf.write("WORK=%s\n" % flag_relocate)
      else:
        outf.write(line)
      continue
    mcd = regcd.match(line)
    if mcd:
      cdir = mcd.group(1)
      outf.write("cd %s\n" % cdir)
      continue
    mar = regar.match(line)
    if mar:
      outf.write(line)
      continue
    mcp = regcp.match(line)
    if mcp:
      outf.write(line)
      continue
    mgc = reggc.match(line)
    mgcg = reggcg.match(line)
    if not mgc and not mgcg:
      continue
    if mgc:
      driver = mgc.group(1)
      argstring = mgc.group(2)
    else:
      driver = mgcg.group(1)
      argstring = mgcg.group(2)

    u.verbose(1, "matched: %s %s" % (driver, argstring))

    driver_var = "DRIVER%d" % driver_count
    if driver in drivers:
      driver_var = drivers[driver]
    else:
      outf.write("%s=%s\n" % (driver_var, driver))
      drivers[driver] = driver_var
      driver_count += 1
    if extract_line(outf, driver, driver_var, argstring, cdir) < 0:
      break

def setup_gccgo_gdb():
  """Set up for gccgo debugging."""
  if not gccgo_location:
    u.warning("failed to locate gccgo compilation "
              "of %s" % flag_gccgo_gdb)
    return
  outfile = ".gccgo.err.txt"
  here = os.getcwd()
  gloc = gccgo_location
  if here != gloc:
    u.warning("gccgo compile takes place in %s, "
              "not here (%s)" % (gccgo_location, here))
    regloc = re.compile(r"\$WORK/(\S+)$")
    m = regloc.match(gccgo_location)
    if m:
      if flag_relocate:
        gloc = os.path.join(flag_relocate, m.group(1))
      else:
        gloc = os.path.join(workdir, m.group(1))

      u.verbose(1, "revised gloc dir is %s" % gloc)
  os.chdir(gloc)
  driver = gccgo_invocation[0]
  args = gccgo_invocation[1:]
  for idx in range(0, len(args)):
    if args[idx] == "$WORK":
      if flag_relocate:
        args[idx] = flag_relocate
      else:
        args[idx] = workdir
  cmd = ("%s -v %s" % (driver, " ".join(args)))
  u.verbose(1, "in %s, executing gdb setup cmd: %s" % (os.getcwd(), cmd))
  rc = u.docmderrout(cmd, outfile, True)
  if rc != 0:
    u.warning("cmd failed: %s" % cmd)
    u.error("output is in %s" % outfile)
  try:
    inf = open(outfile, "rb")
  except IOError as e:
    u.error("internal error: unable to consume tmp "
            "file %s? error %s" % (outfile, e.strerror))
  lines = inf.readlines()
  inf.close()
  found = False
  reg1 = re.compile(r"^\s*(\S+/go1)\s+(\S.+)$")
  for line in lines:
    u.verbose(2, "gccgo -v line is %s" % line.strip())
    m = reg1.match(line)
    if m:
      go1exe = m.group(1)
      u.verbose(1, "go1 driver is %s" % go1exe)
      go1args = m.group(2)
      u.verbose(1, "go1 args: %s" % go1args)
      # Create symlink
      if not os.path.exists("go1"):
        u.verbose(0, "symlinking to %s" % go1exe)
        os.symlink(go1exe, "go1")
      # Dump args
      u.verbose(0, "writing args to .gdbinit")
      try:
        outf = open(".gdbinit", "wb")
      except IOError as e:
        u.error("unable to open %s: "
                "%s" % (".gdbinit", e.strerror))
      outf.write("# gud-gdb --fullname ./go1\n")
      outf.write("set args %s" % go1args)
      outf.close()
      u.verbose(0, "starting emacs")
      subprocess.Popen(["emacs", ".gdbinit"])
      return
  if not found:
    u.error("unable to locate go1 invocation in gccgo -v output")
  os.chdir(here)

def perform():
  """Main driver routine."""
  inf = sys.stdin
  outf = sys.stdout
  if flag_infile:
    try:
      inf = open(flag_infile, "rb")
    except IOError as e:
      u.error("unable to open input file %s: "
              "%s" % (flag_infile, e.strerror))
  if flag_outfile:
    try:
      outf = open(flag_outfile, "wb")
    except IOError as e:
      u.error("unable to open output file %s: "
              "%s" % (flag_outfile, e.strerror))
  perform_extract(inf, outf)
  if flag_infile:
    inf.close()
  if flag_outfile:
    outf.close()
  if flag_gccgo_gdb:
    setup_gccgo_gdb()

def usage(msgarg):
  """Print usage and exit."""
  me = os.path.basename(sys.argv[0])
  if msgarg:
    sys.stderr.write("error: %s\n" % msgarg)
  print """\
    usage:  %s [options] < input > output

    options:
    -d    increase debug msg verbosity level
    -i F  read from input file F (presumably captured 'go build' output)
    -o G  write compile script to file G
    -R X  relocate WORK directory to local dir X (overwrites X)
    -G Y  set up for gccgo debugging for source file 'Y'; reruns compile
          with -v to capture args and emits .gdbinit, etc.

    Examples:

    1. Post-process compile transcript 'trans.txt' to produce
       compilation script 'comp.sh', relocating work dir to 'work'

       go test -x -work -c -a -compiler gccgo . &> trans.txt
       %s -i trans.txt -o comp.sh -R ./work

    2. Post-process go build output file 'gobuild.txt' to set up
       for debugging gccgo when compiling blah.go

       %s -i gobuild.txt -G blah.go

    """ % (me, me, me)
  sys.exit(1)

def parse_args():
  """Command line argument parsing."""
  global flag_infile, flag_outfile, flag_single
  global flag_gccgo_gdb, flag_relocate

  try:
    optlist, args = getopt.getopt(sys.argv[1:], "di:o:R:G:S")
  except getopt.GetoptError as err:
    # unrecognized option
    usage(str(err))

  if args:
    usage("extra unknown arguments")
  for opt, arg in optlist:
    if opt == "-d":
      u.increment_verbosity()
    elif opt == "-S":
      flag_single = True
    elif opt == "-G":
      flag_gccgo_gdb = arg
    elif opt == "-R":
      flag_relocate = arg
    elif opt == "-i":
      if flag_infile:
        usage("supply at most one -i option")
      flag_infile = arg
    elif opt == "-o":
      if flag_outfile:
        usage("supply at most one -o option")
      flag_outfile = arg

# Setup
u.setdeflanglocale()
parse_args()
perform()
<|endoftext|>