<commit_before>"""
TESTS is a dict with all you tests.
Keys for this will be categories' names.
Each test is dict with
    "input" -- input data for user function
    "answer" -- your right answer
    "explanation" -- not necessary key, it's using for additional info in animation.
"""

TESTS = {
    "Basics": [
        {
            "input": (6, 4, 2),
            "answer": (6, 4, 2),
        },
        {
            "input": (1, 2, 3, 4, 5),
            "answer": (1, 2, 3, 4, 5),
        },
        {
            "input": (1, 2, 3, 5, 3),
            "answer": (1, 2, 3, 5, 3),
        },

    ],
    "Edge": [
        {
            "input": [1],
            "answer": [1],
        },
        {
            "input": [5],
            "answer": [5],
        },

        {
            "input": [1, 2, 3, 4, 5, 6, 7, 8, 9, 9],
            "answer": [1, 2, 3, 4, 5, 6, 7, 8, 9, 9],
        },
        {
            "input": [1, 2, 3, 4, 5, 6, 7, 8, 9, 1],
            "answer": [1, 2, 3, 4, 5, 6, 7, 8, 9, 1],
        },
        {
            "input": [9, 1, 1, 1, 1, 1, 1, 1, 1, 1],
            "answer": [9, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        },
    ],
    "Extra": [
        {
            "input": [4, 6, 5, 2, 6, 1],
            "answer": [4, 6, 5, 2, 6, 1]
        },
        {
            "input": [3, 2, 7, 2, 3, 7, 9, 6, 9],
            "answer": [3, 2, 7, 2, 3, 7, 9, 6, 9]
        },
        {
            "input": [5, 8, 5, 3, 7, 8, 1, 5, 1, 4],
            "answer": [5, 8, 5, 3, 7, 8, 1, 5, 1, 4]
        },
        {
            "input": [7, 1, 3, 8, 5, 9, 7],
            "answer": [7, 1, 3, 8, 5, 9, 7]
        },
        {
            "input": [1, 8, 7, 9, 6, 4],
            "answer": [1, 8, 7, 9, 6, 4]
        },
        {
            "input": [3, 7, 7, 6, 7, 1, 3, 3, 4, 9],
            "answer": [3, 7, 7, 6, 7, 1, 3, 3, 4, 9]
        },
        {
            "input": [3, 5, 3],
            "answer": [3, 5, 3]
        },
        {
            "input": [1, 7, 2, 4, 7, 9, 2, 9, 7, 3],
            "answer": [1, 7, 2, 4, 7, 9, 2, 9, 7, 3]
        },
    ]

}
<commit_msg>test update<commit_after>"""
TESTS is a dict with all you tests.
Keys for this will be categories' names.
Each test is dict with
    "input" -- input data for user function
    "answer" -- your right answer
    "explanation" -- not necessary key, it's using for additional info in animation.
"""

TESTS = {
    "Basics": [
        {
            "input": (6, 4, 2),
            "answer": (6, 4, 2),
        },
        {
            "input": (1, 2, 3, 4, 5),
            "answer": (1, 2, 3, 4, 5),
        },
        {
            "input": (1, 2, 3, 5, 3),
            "answer": (1, 2, 3, 5, 3),
        },

    ],
    "Edge": [
        {
            "input": [1],
            "answer": [1],
        },
        {
            "input": [6],
            "answer": [6],
        },

        {
            "input": [1, 2, 3, 4, 5, 6, 7, 8, 9, 9],
            "answer": [1, 2, 3, 4, 5, 6, 7, 8, 9, 9],
        },
        {
            "input": [1, 2, 3, 4, 5, 6, 7, 8, 9, 1],
            "answer": [1, 2, 3, 4, 5, 6, 7, 8, 9, 1],
        },
        {
            "input": [9, 1, 1, 1, 1, 1, 1, 1, 1, 1],
            "answer": [9, 1, 1, 1, 1, 1, 1, 1, 1, 1],
        },
    ],
    "Extra": [
        {
            "input": [4, 6, 5, 2, 6, 1],
            "answer": [4, 6, 5, 2, 6, 1]
        },
        {
            "input": [3, 2, 7, 2, 3, 7, 9, 6, 9],
            "answer": [3, 2, 7, 2, 3, 7, 9, 6, 9]
        },
        {
            "input": [5, 8, 5, 3, 7, 8, 1, 5, 1, 4],
            "answer": [5, 8, 5, 3, 7, 8, 1, 5, 1, 4]
        },
        {
            "input": [7, 1, 3, 8, 5, 9, 7],
            "answer": [7, 1, 3, 8, 5, 9, 7]
        },
        {
            "input": [1, 8, 7, 9, 6, 4],
            "answer": [1, 8, 7, 9, 6, 4]
        },
        {
            "input": [3, 7, 7, 6, 7, 1, 3, 3, 4, 9],
            "answer": [3, 7, 7, 6, 7, 1, 3, 3, 4, 9]
        },
        {
            "input": [3, 5, 3],
            "answer": [3, 5, 3]
        },
        {
            "input": [1, 7, 2, 4, 7, 9, 2, 9, 7, 3],
            "answer": [1, 7, 2, 4, 7, 9, 2, 9, 7, 3]
        },
    ]

}
<|endoftext|>


<commit_before>from django.db import models
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from shapely.geometry import JOIN_STYLE
from shapely.ops import cascaded_union

from c3nav.mapdata.models.base import MapItem

class Level(MapItem):
    """
    A map level (-1, 0, 1, 2…)
    """
    name = models.SlugField(_('level name'), unique=True, max_length=50,
                            help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)'))
    altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2)

    class Meta:
        verbose_name = _('Level')
        verbose_name_plural = _('Levels')
        default_related_name = 'levels'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.geometries = LevelGeometries(self)

    def tofilename(self):
        return 'levels/%s.json' % self.name

    @classmethod
    def fromfile(cls, data, file_path):
        kwargs = super().fromfile(data, file_path)

        if 'altitude' not in data:
            raise ValueError('missing altitude.')

        if not isinstance(data['altitude'], (int, float)):
            raise ValueError('altitude has to be int or float.')

        kwargs['altitude'] = data['altitude']

        return kwargs

    def tofile(self):
        result = super().tofile()
        result['altitude'] = float(self.altitude)
        return result

class LevelGeometries():
    def __init__(self, level):
        self.level = level

    @cached_property
    def buildings(self):
        return cascaded_union([building.geometry for building in self.level.buildings.all()])

    @cached_property
    def rooms(self):
        return cascaded_union([room.geometry for room in self.level.rooms.all()]).intersection(self.buildings)

    @cached_property
    def outsides(self):
        return cascaded_union([outside.geometry for outside in self.level.outsides.all()]).difference(self.buildings)

    @cached_property
    def mapped(self):
        return cascaded_union([self.buildings, self.outsides])

    @cached_property
    def obstacles(self):
        return cascaded_union([obstacle.geometry for obstacle in self.level.obstacles.all()])

    @cached_property
    def raw_doors(self):
        return cascaded_union([door.geometry for door in self.level.doors.all()]).intersection(self.mapped)

    @cached_property
    def areas_and_doors(self):
        return cascaded_union([self.rooms, self.raw_doors])

    @cached_property
    def walls(self):
        return self.buildings.difference(self.rooms)

    @cached_property
    def walls_without_doors(self):
        return self.walls.difference(self.areas_and_doors)

    @cached_property
    def walls_shadow(self):
        return self.walls_without_doors.buffer(0.2, join_style=JOIN_STYLE.mitre).intersection(self.mapped)

    @cached_property
    def doors(self):
        return self.raw_doors.difference(self.rooms).difference(self.outsides)
<commit_msg>map renderer: no wall shadows outside<commit_after>from django.db import models
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from shapely.geometry import JOIN_STYLE
from shapely.ops import cascaded_union

from c3nav.mapdata.models.base import MapItem

class Level(MapItem):
    """
    A map level (-1, 0, 1, 2…)
    """
    name = models.SlugField(_('level name'), unique=True, max_length=50,
                            help_text=_('Usually just an integer (e.g. -1, 0, 1, 2)'))
    altitude = models.DecimalField(_('level altitude'), null=True, max_digits=6, decimal_places=2)

    class Meta:
        verbose_name = _('Level')
        verbose_name_plural = _('Levels')
        default_related_name = 'levels'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.geometries = LevelGeometries(self)

    def tofilename(self):
        return 'levels/%s.json' % self.name

    @classmethod
    def fromfile(cls, data, file_path):
        kwargs = super().fromfile(data, file_path)

        if 'altitude' not in data:
            raise ValueError('missing altitude.')

        if not isinstance(data['altitude'], (int, float)):
            raise ValueError('altitude has to be int or float.')

        kwargs['altitude'] = data['altitude']

        return kwargs

    def tofile(self):
        result = super().tofile()
        result['altitude'] = float(self.altitude)
        return result

class LevelGeometries():
    def __init__(self, level):
        self.level = level

    @cached_property
    def buildings(self):
        return cascaded_union([building.geometry for building in self.level.buildings.all()])

    @cached_property
    def rooms(self):
        return cascaded_union([room.geometry for room in self.level.rooms.all()]).intersection(self.buildings)

    @cached_property
    def outsides(self):
        return cascaded_union([outside.geometry for outside in self.level.outsides.all()]).difference(self.buildings)

    @cached_property
    def mapped(self):
        return cascaded_union([self.buildings, self.outsides])

    @cached_property
    def obstacles(self):
        return cascaded_union([obstacle.geometry for obstacle in self.level.obstacles.all()])

    @cached_property
    def raw_doors(self):
        return cascaded_union([door.geometry for door in self.level.doors.all()]).intersection(self.mapped)

    @cached_property
    def areas_and_doors(self):
        return cascaded_union([self.rooms, self.raw_doors])

    @cached_property
    def walls(self):
        return self.buildings.difference(self.rooms)

    @cached_property
    def walls_without_doors(self):
        return self.walls.difference(self.areas_and_doors)

    @cached_property
    def walls_shadow(self):
        return self.walls_without_doors.buffer(0.2, join_style=JOIN_STYLE.mitre).intersection(self.buildings)

    @cached_property
    def doors(self):
        return self.raw_doors.difference(self.rooms).difference(self.outsides)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

'''
Copyright (c) 2009, Patrick Maupin, Austin, Texas

Automated testing for rst2pdf

See LICENSE.txt for licensing terms
'''

import os
import sys
import glob
import shutil
from copy import copy
from optparse import OptionParser
from execmgr import textexec, default_logger as log
from pythonpaths import setpythonpaths

# md5 module deprecated, but hashlib not available in 2.4
try:
    import hashlib
except ImportError:
    import md5 as hashlib

description = '''
autotest.py reads .txt files (and optional associated .style and other files)
from the input directory and generates throw-away results (.pdf and .log) in
the output subdirectory.  It also maintains (with the help of the developers)
a database of unknown, good, and bad MD5 checksums for the .pdf output files
in the md5 subdirectory.

By default, it will process all the files in the input directory, but one or
more individual files can be explicitly specified on the command line.

Use of the -c and -a options can cause usage of an external coverage package
to generate a .coverage file for code coverage.
'''

class PathInfo(object):
    '''  This class is just a namespace to avoid cluttering up the
         module namespace.  It is never instantiated.
    '''
    rootdir = os.path.realpath(os.path.dirname(__file__))
    bindir = os.path.abspath(os.path.join(rootdir, '..', '..', 'bin'))
    runfile = os.path.join(bindir, 'rst2pdf')
    inpdir = os.path.join(rootdir, 'input')
    outdir = os.path.join(rootdir, 'output')
    md5dir = os.path.join(rootdir, 'md5')

    if not os.path.exists(runfile):
        raise SystemExit('Use bootstrap.py and buildout to create executable')

    #setpythonpaths(runfile, rootdir)

    runcmd = [runfile]

    @classmethod
    def add_coverage(cls, keep=False):
        cls.runcmd[0:0] = ['coverage', 'run', '-a']
        fname = os.path.join(cls.rootdir, '.coverage')
        os.environ['COVERAGE_FILE'] = fname
        if not keep:
            if os.path.exists(fname):
                os.remove(fname)

    @classmethod
    def load_subprocess(cls):
        import rst2pdf.createpdf
        return rst2pdf.createpdf.main

class MD5Info(dict):
    ''' The MD5Info class is used to round-trip good, bad, unknown
        information to/from a .json file.
        For formatting reasons, the json module isn't used for writing,
        and since we're not worried about security, we don't bother using
        it for reading, either.
    '''
    categories = 'good bad unknown'.split()
    categories = dict((x, x + '_md5') for x in categories)

    def __str__(self):
        ''' Return the string to output to the MD5 file '''
        result = []
        for name in sorted(self.categories.itervalues()):
            result.append('%s = [' % name)
            result.append(',\n'.join(["        '%s'"%item for item in sorted(getattr(self, name))]))
            result.append(']\n')
        result.append('')
        return '\n'.join(result)

    def __init__(self):
        self.__dict__ = self
        self.changed = False
        for name in self.categories.itervalues():
            setattr(self, name, [])

    def find(self, checksum):
        ''' find() has some serious side-effects.  If the checksum
            is found, the category it was found in is returned.
            If the checksum is not found, then it is automagically
            added to the unknown category.  In all cases, the
            data is prepped to output to the file (if necessary),
            and self.changed is set if the data is modified during
            this process.  Functional programming this isn't...

            A quick word about the 'sentinel'.  This value starts
            with an 's', which happens to sort > highest hexadecimal
            digit of 'f', so it is always a the end of the list.

            The only reason for the sentinel is to make the database
            either to work with.  Both to modify (by moving an MD5
            line from one category to another) and to diff.  This
            is because every hexadecimal line (every line except
            the sentinel) is guaranteed to end with a comma.
        '''
        sets = []
        newinfo = {}
        prev = set()
        sentinel = set(['sentinel'])
        for name, fullname in self.categories.iteritems():
            value = set(getattr(self, fullname)) - sentinel
            # Make sure same checksum didn't make it into two
            # different categories (that would be a committer screwup...)
            assert not value & prev, (name, value, prev)
            prev |= value
            sets.append((name, fullname, value))
            newinfo[fullname] = sorted(value | sentinel)

        result = 'unknown'
        for name, fullname, sumset in sets:
            if checksum in sumset:
                result = name
                break
        else:
            mylist = newinfo['unknown_md5']
            mylist.append(checksum)
            mylist.sort()

        for key, value in newinfo.iteritems():
            if value != self[key]:
                self.update(newinfo)
                print "Updating MD5 file"
                self.changed = True
                break
        return result

def checkmd5(pdfpath, md5path, resultlist):
    ''' checkmd5 validates the checksum of a generated PDF
        against the database, both reporting the results,
        and updating the database to add this MD5 into the
        unknown category if this checksum is not currently
        in the database.

        It updates the resultlist with information to be
        printed and added to the log file, and returns
        a result of 'good', 'bad', 'fail', or 'unknown'
    '''
    if not os.path.exists(pdfpath):
        log(resultlist, 'File %s not generated' % os.path.basename(pdfpath))
        return 'fail'

    # Read the database
    info = MD5Info()
    if os.path.exists(md5path):
        f = open(md5path, 'rb')
        exec f in info
        f.close()

    # Generate the current MD5
    f = open(pdfpath, 'rb')
    data = f.read()
    f.close()
    m = hashlib.md5()
    m.update(data)
    m = m.hexdigest()

    # Check MD5 against database and update if necessary
    resulttype = info.find(m)
    log(resultlist, "Validity of file %s checksum '%s' is %s." % (os.path.basename(pdfpath), m, resulttype))
    if info.changed:
        f = open(md5path, 'wb')
        f.write(str(info))
        f.close()
    return resulttype

def run_single_test(inpfname, incremental=False, fastfork=None):
    use_sphinx = 'sphinx' in inpfname
    if use_sphinx:
        sphinxdir = inpfname
        if sphinxdir.endswith('Makefile'):
            sphinxdir = os.path.dirname(sphinxdir)
        builddir = os.path.join(sphinxdir, '_build')
        basename = os.path.basename(sphinxdir)
    else:
        iprefix = os.path.splitext(inpfname)[0]
        style = iprefix + '.style'
        cli = iprefix + '.cli'
        basename = os.path.basename(iprefix)

    oprefix = os.path.join(PathInfo.outdir, basename)
    mprefix = os.path.join(PathInfo.md5dir, basename)
    outpdf = oprefix + '.pdf'
    outtext = oprefix + '.log'
    md5file = mprefix + '.json'

    if incremental and os.path.exists(outpdf):
        return 'preexisting', 0

    for fname in (outtext, outpdf):
        if os.path.exists(fname):
            os.remove(fname)

    if use_sphinx:
        if os.path.isdir(builddir):
            shutil.rmtree(builddir)
        errcode, result = textexec('make pdf', cwd=sphinxdir)
        pdffiles = glob.glob(os.path.join(builddir, 'pdf', '*.pdf'))
        if len(pdffiles) != 1:
            log(result, 'Output PDF apparently not generated')
            errcode = 1
        else:
            shutil.copyfile(pdffiles[0], outpdf)
    else:
        if cli:
            extraargs=[ x.strip() for x in open(cli).readlines()]
        else:
            extraargs=[]
        print 'EA:',extraargs
        args = PathInfo.runcmd + ['--date-invariant', '-v', os.path.basename(inpfname)]+extraargs
        if os.path.exists(style):
            args.extend(('-s', os.path.basename(style)))
        args.extend(('-o', outpdf))
        errcode, result = textexec(args, cwd=os.path.dirname(inpfname), python_proc=fastfork)

    checkinfo = checkmd5(outpdf, md5file, result)
    log(result, '')
    outf = open(outtext, 'wb')
    outf.write('\n'.join(result))
    outf.close()
    return checkinfo, errcode

def run_testlist(testfiles=None, incremental=False, fastfork=None, sphinx=False):
    if not testfiles:
        if sphinx:
            testfiles = glob.glob(os.path.join(PathInfo.inpdir, 'sphinx*'))
        else:
            testfiles = glob.glob(os.path.join(PathInfo.inpdir, '*.txt'))
            testfiles += glob.glob(os.path.join(PathInfo.inpdir, '*', '*.txt'))
            testfiles = [x for x in testfiles if 'sphinx' not in x]
        testfiles.sort()
    results = {}
    for fname in testfiles:
        key, errcode = run_single_test(fname, incremental, fastfork)
        results[key] = results.get(key, 0) + 1
        if incremental and errcode and 0:
            break
    print
    print 'Final checksum statistics:',
    print ', '.join(sorted('%s=%s' % x for x in results.iteritems()))
    print

def parse_commandline():
    usage = '%prog [options] [<input.txt file> [<input.txt file>]...]'
    parser = OptionParser(usage, description=description)
    parser.add_option('-c', '--coverage', action="store_true",
        dest='coverage', default=False,
        help='Generate new coverage information.')
    parser.add_option('-a', '--add-coverage', action="store_true",
        dest='add_coverage', default=False,
        help='Add coverage information to previous runs.')
    parser.add_option('-i', '--incremental', action="store_true",
        dest='incremental', default=False,
        help='Incremental build -- ignores existing PDFs')
    parser.add_option('-f', '--fast', action="store_true",
        dest='fastfork', default=False,
        help='Fork and reuse process information')
    parser.add_option('-s', '--sphinx', action="store_true",
        dest='sphinx', default=False,
        help='Run sphinx tests')
    return parser

def main(args=None):
    parser = parse_commandline()
    options, args = parser.parse_args(copy(args))
    fastfork = None
    if options.coverage or options.add_coverage:
        assert not options.fastfork, "Cannot fastfork and run coverage simultaneously"
        assert not options.sphinx, "Cannot run sphinx and coverage simultaneously"
        PathInfo.add_coverage(options.add_coverage)
    elif options.fastfork:
        fastfork = PathInfo.load_subprocess()
    run_testlist(args, options.incremental, fastfork, options.sphinx)

if __name__ == '__main__':
    main()
<commit_msg>Fixed bug in extra arguments support<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#$HeadURL$
#$LastChangedDate$
#$LastChangedRevision$

'''
Copyright (c) 2009, Patrick Maupin, Austin, Texas

Automated testing for rst2pdf

See LICENSE.txt for licensing terms
'''

import os
import sys
import glob
import shutil
from copy import copy
from optparse import OptionParser
from execmgr import textexec, default_logger as log
from pythonpaths import setpythonpaths

# md5 module deprecated, but hashlib not available in 2.4
try:
    import hashlib
except ImportError:
    import md5 as hashlib

description = '''
autotest.py reads .txt files (and optional associated .style and other files)
from the input directory and generates throw-away results (.pdf and .log) in
the output subdirectory.  It also maintains (with the help of the developers)
a database of unknown, good, and bad MD5 checksums for the .pdf output files
in the md5 subdirectory.

By default, it will process all the files in the input directory, but one or
more individual files can be explicitly specified on the command line.

Use of the -c and -a options can cause usage of an external coverage package
to generate a .coverage file for code coverage.
'''

class PathInfo(object):
    '''  This class is just a namespace to avoid cluttering up the
         module namespace.  It is never instantiated.
    '''
    rootdir = os.path.realpath(os.path.dirname(__file__))
    bindir = os.path.abspath(os.path.join(rootdir, '..', '..', 'bin'))
    runfile = os.path.join(bindir, 'rst2pdf')
    inpdir = os.path.join(rootdir, 'input')
    outdir = os.path.join(rootdir, 'output')
    md5dir = os.path.join(rootdir, 'md5')

    if not os.path.exists(runfile):
        raise SystemExit('Use bootstrap.py and buildout to create executable')

    #setpythonpaths(runfile, rootdir)

    runcmd = [runfile]

    @classmethod
    def add_coverage(cls, keep=False):
        cls.runcmd[0:0] = ['coverage', 'run', '-a']
        fname = os.path.join(cls.rootdir, '.coverage')
        os.environ['COVERAGE_FILE'] = fname
        if not keep:
            if os.path.exists(fname):
                os.remove(fname)

    @classmethod
    def load_subprocess(cls):
        import rst2pdf.createpdf
        return rst2pdf.createpdf.main

class MD5Info(dict):
    ''' The MD5Info class is used to round-trip good, bad, unknown
        information to/from a .json file.
        For formatting reasons, the json module isn't used for writing,
        and since we're not worried about security, we don't bother using
        it for reading, either.
    '''
    categories = 'good bad unknown'.split()
    categories = dict((x, x + '_md5') for x in categories)

    def __str__(self):
        ''' Return the string to output to the MD5 file '''
        result = []
        for name in sorted(self.categories.itervalues()):
            result.append('%s = [' % name)
            result.append(',\n'.join(["        '%s'"%item for item in sorted(getattr(self, name))]))
            result.append(']\n')
        result.append('')
        return '\n'.join(result)

    def __init__(self):
        self.__dict__ = self
        self.changed = False
        for name in self.categories.itervalues():
            setattr(self, name, [])

    def find(self, checksum):
        ''' find() has some serious side-effects.  If the checksum
            is found, the category it was found in is returned.
            If the checksum is not found, then it is automagically
            added to the unknown category.  In all cases, the
            data is prepped to output to the file (if necessary),
            and self.changed is set if the data is modified during
            this process.  Functional programming this isn't...

            A quick word about the 'sentinel'.  This value starts
            with an 's', which happens to sort > highest hexadecimal
            digit of 'f', so it is always a the end of the list.

            The only reason for the sentinel is to make the database
            either to work with.  Both to modify (by moving an MD5
            line from one category to another) and to diff.  This
            is because every hexadecimal line (every line except
            the sentinel) is guaranteed to end with a comma.
        '''
        sets = []
        newinfo = {}
        prev = set()
        sentinel = set(['sentinel'])
        for name, fullname in self.categories.iteritems():
            value = set(getattr(self, fullname)) - sentinel
            # Make sure same checksum didn't make it into two
            # different categories (that would be a committer screwup...)
            assert not value & prev, (name, value, prev)
            prev |= value
            sets.append((name, fullname, value))
            newinfo[fullname] = sorted(value | sentinel)

        result = 'unknown'
        for name, fullname, sumset in sets:
            if checksum in sumset:
                result = name
                break
        else:
            mylist = newinfo['unknown_md5']
            mylist.append(checksum)
            mylist.sort()

        for key, value in newinfo.iteritems():
            if value != self[key]:
                self.update(newinfo)
                print "Updating MD5 file"
                self.changed = True
                break
        return result

def checkmd5(pdfpath, md5path, resultlist):
    ''' checkmd5 validates the checksum of a generated PDF
        against the database, both reporting the results,
        and updating the database to add this MD5 into the
        unknown category if this checksum is not currently
        in the database.

        It updates the resultlist with information to be
        printed and added to the log file, and returns
        a result of 'good', 'bad', 'fail', or 'unknown'
    '''
    if not os.path.exists(pdfpath):
        log(resultlist, 'File %s not generated' % os.path.basename(pdfpath))
        return 'fail'

    # Read the database
    info = MD5Info()
    if os.path.exists(md5path):
        f = open(md5path, 'rb')
        exec f in info
        f.close()

    # Generate the current MD5
    f = open(pdfpath, 'rb')
    data = f.read()
    f.close()
    m = hashlib.md5()
    m.update(data)
    m = m.hexdigest()

    # Check MD5 against database and update if necessary
    resulttype = info.find(m)
    log(resultlist, "Validity of file %s checksum '%s' is %s." % (os.path.basename(pdfpath), m, resulttype))
    if info.changed:
        f = open(md5path, 'wb')
        f.write(str(info))
        f.close()
    return resulttype

def run_single_test(inpfname, incremental=False, fastfork=None):
    use_sphinx = 'sphinx' in inpfname
    if use_sphinx:
        sphinxdir = inpfname
        if sphinxdir.endswith('Makefile'):
            sphinxdir = os.path.dirname(sphinxdir)
        builddir = os.path.join(sphinxdir, '_build')
        basename = os.path.basename(sphinxdir)
    else:
        iprefix = os.path.splitext(inpfname)[0]
        style = iprefix + '.style'
        cli = iprefix + '.cli'
        basename = os.path.basename(iprefix)

    oprefix = os.path.join(PathInfo.outdir, basename)
    mprefix = os.path.join(PathInfo.md5dir, basename)
    outpdf = oprefix + '.pdf'
    outtext = oprefix + '.log'
    md5file = mprefix + '.json'

    if incremental and os.path.exists(outpdf):
        return 'preexisting', 0

    for fname in (outtext, outpdf):
        if os.path.exists(fname):
            os.remove(fname)

    if use_sphinx:
        if os.path.isdir(builddir):
            shutil.rmtree(builddir)
        errcode, result = textexec('make pdf', cwd=sphinxdir)
        pdffiles = glob.glob(os.path.join(builddir, 'pdf', '*.pdf'))
        if len(pdffiles) != 1:
            log(result, 'Output PDF apparently not generated')
            errcode = 1
        else:
            shutil.copyfile(pdffiles[0], outpdf)
    else:
        if os.path.isfile(cli):
            extraargs=[ x.strip() for x in open(cli).readlines()]
        else:
            extraargs=[]
        print 'EA:',extraargs
        args = PathInfo.runcmd + ['--date-invariant', '-v', os.path.basename(inpfname)]+extraargs
        if os.path.exists(style):
            args.extend(('-s', os.path.basename(style)))
        args.extend(('-o', outpdf))
        errcode, result = textexec(args, cwd=os.path.dirname(inpfname), python_proc=fastfork)

    checkinfo = checkmd5(outpdf, md5file, result)
    log(result, '')
    outf = open(outtext, 'wb')
    outf.write('\n'.join(result))
    outf.close()
    return checkinfo, errcode

def run_testlist(testfiles=None, incremental=False, fastfork=None, sphinx=False):
    if not testfiles:
        if sphinx:
            testfiles = glob.glob(os.path.join(PathInfo.inpdir, 'sphinx*'))
        else:
            testfiles = glob.glob(os.path.join(PathInfo.inpdir, '*.txt'))
            testfiles += glob.glob(os.path.join(PathInfo.inpdir, '*', '*.txt'))
            testfiles = [x for x in testfiles if 'sphinx' not in x]
        testfiles.sort()
    results = {}
    for fname in testfiles:
        key, errcode = run_single_test(fname, incremental, fastfork)
        results[key] = results.get(key, 0) + 1
        if incremental and errcode and 0:
            break
    print
    print 'Final checksum statistics:',
    print ', '.join(sorted('%s=%s' % x for x in results.iteritems()))
    print

def parse_commandline():
    usage = '%prog [options] [<input.txt file> [<input.txt file>]...]'
    parser = OptionParser(usage, description=description)
    parser.add_option('-c', '--coverage', action="store_true",
        dest='coverage', default=False,
        help='Generate new coverage information.')
    parser.add_option('-a', '--add-coverage', action="store_true",
        dest='add_coverage', default=False,
        help='Add coverage information to previous runs.')
    parser.add_option('-i', '--incremental', action="store_true",
        dest='incremental', default=False,
        help='Incremental build -- ignores existing PDFs')
    parser.add_option('-f', '--fast', action="store_true",
        dest='fastfork', default=False,
        help='Fork and reuse process information')
    parser.add_option('-s', '--sphinx', action="store_true",
        dest='sphinx', default=False,
        help='Run sphinx tests')
    return parser

def main(args=None):
    parser = parse_commandline()
    options, args = parser.parse_args(copy(args))
    fastfork = None
    if options.coverage or options.add_coverage:
        assert not options.fastfork, "Cannot fastfork and run coverage simultaneously"
        assert not options.sphinx, "Cannot run sphinx and coverage simultaneously"
        PathInfo.add_coverage(options.add_coverage)
    elif options.fastfork:
        fastfork = PathInfo.load_subprocess()
    run_testlist(args, options.incremental, fastfork, options.sphinx)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import imp
import logging
import math

from . import Veros, settings

class LowercaseAttributeWrapper(object):
    """
    A simple wrapper class that converts attributes to lower case (needed for Fortran interface)
    """

    def __init__(self, wrapped_object):
        object.__setattr__(self, "_w", wrapped_object)

    def __getattr__(self, key):
        if key == "_w":
            return object.__getattribute__(self, "_w")
        return getattr(object.__getattribute__(self, "_w"), key.lower())

    def __setattr__(self, key, value):
        setattr(self._w, key.lower(), value)

class VerosLegacy(Veros):
    """
    An alternative Veros class that supports the pyOM Fortran interface as backend

    .. warning::

       Do not use this class for new setups!

    """

    def __init__(self, fortran=None, *args, **kwargs):
        """
        To use the pyOM2 legacy interface point the fortran argument to the Veros fortran library:

        > simulation = GlobalOneDegree(fortran = "pyOM_code.so")

        """
        if fortran:
            self.legacy_mode = True
            try:
                self.fortran = LowercaseAttributeWrapper(imp.load_dynamic("pyOM_code", fortran))
                self.use_mpi = False
            except ImportError:
                self.fortran = LowercaseAttributeWrapper(imp.load_dynamic("pyOM_code_MPI", fortran))
                self.use_mpi = True
                from mpi4py import MPI
                self.mpi_comm = MPI.COMM_WORLD
            self.main_module = LowercaseAttributeWrapper(self.fortran.main_module)
            self.isoneutral_module = LowercaseAttributeWrapper(self.fortran.isoneutral_module)
            self.idemix_module = LowercaseAttributeWrapper(self.fortran.idemix_module)
            self.tke_module = LowercaseAttributeWrapper(self.fortran.tke_module)
            self.eke_module = LowercaseAttributeWrapper(self.fortran.eke_module)
        else:
            self.legacy_mode = False
            self.use_mpi = False
            self.fortran = self
            self.main_module = self
            self.isoneutral_module = self
            self.idemix_module = self
            self.tke_module = self
            self.eke_module = self
        self.modules = (self.main_module, self.isoneutral_module, self.idemix_module,
                        self.tke_module, self.eke_module)

        if self.use_mpi and self.mpi_comm.Get_rank() != 0:
            kwargs["loglevel"] = "critical"
        super(VerosLegacy, self).__init__(*args, **kwargs)

    def set_legacy_parameter(self, *args, **kwargs):
        m = self.fortran.main_module
        if self.use_mpi:
            proc_combinations = ((ni, m.n_pes / ni) for ni in xrange(1, int(math.sqrt(m.n_pes)+1) if ni * (m.n_pes / ni) == m.n_pes)
            m.n_pes_i, m.n_pes_j = min(proc_combinations, key=lambda x: abs(x[1] - x[0]))
        self.if2py = lambda i: i + m.onx - m.is_pe
        self.jf2py = lambda j: j + m.onx - m.js_pe
        self.ip2fy = lambda i: i + m.is_pe - m.onx
        self.jp2fy = lambda j: j + m.js_pe - m.onx
        self.get_tau = lambda: m.tau - 1 if self.legacy_mode else m.tau

        # force settings that are not supported by Veros
        idm = self.fortran.idemix_module
        m.enable_streamfunction = True
        m.enable_hydrostatic = True
        idm.enable_idemix_m2 = False
        idm.enable_idemix_niw = False

    def _set_commandline_settings(self):
        for key, val in self.command_line_settings:
            for m in self.modules:
                if hasattr(m, key):
                    setattr(m, key, settings.SETTINGS[key].type(val))

    def setup(self, *args, **kwargs):
        with self.timers["setup"]:
            if self.legacy_mode:
                if self.use_mpi:
                    self.fortran.my_mpi_init(self.mpi_comm.py2f())
                else:
                    self.fortran.my_mpi_init(0)
                self.set_parameter()
                self.set_legacy_parameter()
                self._set_commandline_settings()
                self.fortran.pe_decomposition()
                self.fortran.allocate_main_module()
                self.fortran.allocate_isoneutral_module()
                self.fortran.allocate_tke_module()
                self.fortran.allocate_eke_module()
                self.fortran.allocate_idemix_module()
                self.set_grid()
                self.fortran.calc_grid()
                self.set_coriolis()
                self.fortran.calc_beta()
                self.set_topography()
                self.fortran.calc_topo()
                self.fortran.calc_spectral_topo()
                self.set_initial_conditions()
                self.fortran.calc_initial_conditions()
                self.set_forcing()
                self.fortran.streamfunction_init()
                self.set_diagnostics()
                self.fortran.check_isoneutral_slope_crit()
            else:
                # self.set_parameter() is called twice, but that shouldn't matter
                self.set_parameter()
                self.set_legacy_parameter()
                super(VerosLegacy, self).setup(*args, **kwargs)

                diag_legacy_settings = (
                    (self.diagnostics["cfl_monitor"], "output_frequency", "ts_monint"),
                    (self.diagnostics["tracer_monitor"], "output_frequency", "trac_cont_int"),
                    (self.diagnostics["snapshot"], "output_frequency", "snapint"),
                    (self.diagnostics["averages"], "output_frequency", "aveint"),
                    (self.diagnostics["averages"], "sampling_frequency", "avefreq"),
                    (self.diagnostics["overturning"], "output_frequency", "overint"),
                    (self.diagnostics["overturning"], "sampling_frequency", "overfreq"),
                    (self.diagnostics["energy"], "output_frequency", "energint"),
                    (self.diagnostics["energy"], "sampling_frequency", "energfreq"),
                )

                for diag, param, attr in diag_legacy_settings:
                    if hasattr(self, attr):
                        setattr(diag, param, getattr(self, attr))

    def run(self, **kwargs):
        if not self.legacy_mode:
            super(VerosLegacy, self).run(**kwargs)
            return

        f = self.fortran
        m = self.main_module
        idm = self.idemix_module
        ekm = self.eke_module
        tkm = self.tke_module

        m.enditt = m.itt + int(m.runlen / m.dt_tracer)
        logging.info("Starting integration for {:.2e}s".format(float(m.runlen)))
        logging.info(" from time step {} to {}".format(m.itt, m.enditt))

        while m.itt < m.enditt:
            with self.timers["main"]:
                self.set_forcing()

                if idm.enable_idemix:
                    f.set_idemix_parameter()

                f.set_eke_diffusivities()
                f.set_tke_diffusivities()

                with self.timers["momentum"]:
                    f.momentum()

                with self.timers["temperature"]:
                    f.thermodynamics()

                if ekm.enable_eke or tkm.enable_tke or idm.enable_idemix:
                    f.calculate_velocity_on_wgrid()

                with self.timers["eke"]:
                    if ekm.enable_eke:
                        f.integrate_eke()

                with self.timers["idemix"]:
                    if idm.enable_idemix:
                        f.integrate_idemix()

                with self.timers["tke"]:
                    if tkm.enable_tke:
                        f.integrate_tke()

                """
                Main boundary exchange
                for density, temp and salt this is done in integrate_tempsalt.f90
                """
                f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                   m.onx, m.je_pe + m.onx, m.u[:, :, :, m.taup1 - 1], m.nz)
                f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                m.je_pe + m.onx, m.u[:, :, :, m.taup1 - 1], m.nz)
                f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                   m.onx, m.je_pe + m.onx, m.v[:, :, :, m.taup1 - 1], m.nz)
                f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                m.je_pe + m.onx, m.v[:, :, :, m.taup1 - 1], m.nz)

                if tkm.enable_tke:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, tkm.tke[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, tkm.tke[:, :, :, m.taup1 - 1], m.nz)
                if ekm.enable_eke:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, ekm.eke[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, ekm.eke[:, :, :, m.taup1 - 1], m.nz)
                if idm.enable_idemix:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, idm.e_iw[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, idm.e_iw[:, :, :, m.taup1 - 1], m.nz)

                # diagnose vertical velocity at taup1
                f.vertical_velocity()

            # shift time
            m.itt += 1
            self.time += m.dt_tracer

            self.after_timestep()

            otaum1 = m.taum1 * 1
            m.taum1 = m.tau
            m.tau = m.taup1
            m.taup1 = otaum1

            logging.info("Current iteration: {}".format(m.itt))
            logging.debug("Time step took {}s".format(self.timers["main"].getLastTime()))

        logging.debug("Timing summary:")
        logging.debug(" setup time summary       = {}s".format(self.timers["setup"].getTime()))
        logging.debug(" main loop time summary   = {}s".format(self.timers["main"].getTime()))
        logging.debug("     momentum             = {}s".format(self.timers["momentum"].getTime()))
        logging.debug("     thermodynamics       = {}s".format(self.timers["temperature"].getTime()))
        logging.debug("     EKE                  = {}s".format(self.timers["eke"].getTime()))
        logging.debug("     IDEMIX               = {}s".format(self.timers["idemix"].getTime()))
        logging.debug("     TKE                  = {}s".format(self.timers["tke"].getTime()))
<commit_msg>[test] syntax error<commit_after>import imp
import logging
import math

from . import Veros, settings

class LowercaseAttributeWrapper(object):
    """
    A simple wrapper class that converts attributes to lower case (needed for Fortran interface)
    """

    def __init__(self, wrapped_object):
        object.__setattr__(self, "_w", wrapped_object)

    def __getattr__(self, key):
        if key == "_w":
            return object.__getattribute__(self, "_w")
        return getattr(object.__getattribute__(self, "_w"), key.lower())

    def __setattr__(self, key, value):
        setattr(self._w, key.lower(), value)

class VerosLegacy(Veros):
    """
    An alternative Veros class that supports the pyOM Fortran interface as backend

    .. warning::

       Do not use this class for new setups!

    """

    def __init__(self, fortran=None, *args, **kwargs):
        """
        To use the pyOM2 legacy interface point the fortran argument to the Veros fortran library:

        > simulation = GlobalOneDegree(fortran = "pyOM_code.so")

        """
        if fortran:
            self.legacy_mode = True
            try:
                self.fortran = LowercaseAttributeWrapper(imp.load_dynamic("pyOM_code", fortran))
                self.use_mpi = False
            except ImportError:
                self.fortran = LowercaseAttributeWrapper(imp.load_dynamic("pyOM_code_MPI", fortran))
                self.use_mpi = True
                from mpi4py import MPI
                self.mpi_comm = MPI.COMM_WORLD
            self.main_module = LowercaseAttributeWrapper(self.fortran.main_module)
            self.isoneutral_module = LowercaseAttributeWrapper(self.fortran.isoneutral_module)
            self.idemix_module = LowercaseAttributeWrapper(self.fortran.idemix_module)
            self.tke_module = LowercaseAttributeWrapper(self.fortran.tke_module)
            self.eke_module = LowercaseAttributeWrapper(self.fortran.eke_module)
        else:
            self.legacy_mode = False
            self.use_mpi = False
            self.fortran = self
            self.main_module = self
            self.isoneutral_module = self
            self.idemix_module = self
            self.tke_module = self
            self.eke_module = self
        self.modules = (self.main_module, self.isoneutral_module, self.idemix_module,
                        self.tke_module, self.eke_module)

        if self.use_mpi and self.mpi_comm.Get_rank() != 0:
            kwargs["loglevel"] = "critical"
        super(VerosLegacy, self).__init__(*args, **kwargs)

    def set_legacy_parameter(self, *args, **kwargs):
        m = self.fortran.main_module
        if self.use_mpi:
            proc_combinations = [(ni, m.n_pes / ni) for ni in xrange(1, int(math.sqrt(m.n_pes)+1)) if ni * (m.n_pes / ni) == m.n_pes]
            m.n_pes_i, m.n_pes_j = min(proc_combinations, key=lambda x: abs(x[1] - x[0]))
        self.if2py = lambda i: i + m.onx - m.is_pe
        self.jf2py = lambda j: j + m.onx - m.js_pe
        self.ip2fy = lambda i: i + m.is_pe - m.onx
        self.jp2fy = lambda j: j + m.js_pe - m.onx
        self.get_tau = lambda: m.tau - 1 if self.legacy_mode else m.tau

        # force settings that are not supported by Veros
        idm = self.fortran.idemix_module
        m.enable_streamfunction = True
        m.enable_hydrostatic = True
        idm.enable_idemix_m2 = False
        idm.enable_idemix_niw = False

    def _set_commandline_settings(self):
        for key, val in self.command_line_settings:
            for m in self.modules:
                if hasattr(m, key):
                    setattr(m, key, settings.SETTINGS[key].type(val))

    def setup(self, *args, **kwargs):
        with self.timers["setup"]:
            if self.legacy_mode:
                if self.use_mpi:
                    self.fortran.my_mpi_init(self.mpi_comm.py2f())
                else:
                    self.fortran.my_mpi_init(0)
                self.set_parameter()
                self.set_legacy_parameter()
                self._set_commandline_settings()
                self.fortran.pe_decomposition()
                self.fortran.allocate_main_module()
                self.fortran.allocate_isoneutral_module()
                self.fortran.allocate_tke_module()
                self.fortran.allocate_eke_module()
                self.fortran.allocate_idemix_module()
                self.set_grid()
                self.fortran.calc_grid()
                self.set_coriolis()
                self.fortran.calc_beta()
                self.set_topography()
                self.fortran.calc_topo()
                self.fortran.calc_spectral_topo()
                self.set_initial_conditions()
                self.fortran.calc_initial_conditions()
                self.set_forcing()
                self.fortran.streamfunction_init()
                self.set_diagnostics()
                self.fortran.check_isoneutral_slope_crit()
            else:
                # self.set_parameter() is called twice, but that shouldn't matter
                self.set_parameter()
                self.set_legacy_parameter()
                super(VerosLegacy, self).setup(*args, **kwargs)

                diag_legacy_settings = (
                    (self.diagnostics["cfl_monitor"], "output_frequency", "ts_monint"),
                    (self.diagnostics["tracer_monitor"], "output_frequency", "trac_cont_int"),
                    (self.diagnostics["snapshot"], "output_frequency", "snapint"),
                    (self.diagnostics["averages"], "output_frequency", "aveint"),
                    (self.diagnostics["averages"], "sampling_frequency", "avefreq"),
                    (self.diagnostics["overturning"], "output_frequency", "overint"),
                    (self.diagnostics["overturning"], "sampling_frequency", "overfreq"),
                    (self.diagnostics["energy"], "output_frequency", "energint"),
                    (self.diagnostics["energy"], "sampling_frequency", "energfreq"),
                )

                for diag, param, attr in diag_legacy_settings:
                    if hasattr(self, attr):
                        setattr(diag, param, getattr(self, attr))

    def run(self, **kwargs):
        if not self.legacy_mode:
            super(VerosLegacy, self).run(**kwargs)
            return

        f = self.fortran
        m = self.main_module
        idm = self.idemix_module
        ekm = self.eke_module
        tkm = self.tke_module

        m.enditt = m.itt + int(m.runlen / m.dt_tracer)
        logging.info("Starting integration for {:.2e}s".format(float(m.runlen)))
        logging.info(" from time step {} to {}".format(m.itt, m.enditt))

        while m.itt < m.enditt:
            with self.timers["main"]:
                self.set_forcing()

                if idm.enable_idemix:
                    f.set_idemix_parameter()

                f.set_eke_diffusivities()
                f.set_tke_diffusivities()

                with self.timers["momentum"]:
                    f.momentum()

                with self.timers["temperature"]:
                    f.thermodynamics()

                if ekm.enable_eke or tkm.enable_tke or idm.enable_idemix:
                    f.calculate_velocity_on_wgrid()

                with self.timers["eke"]:
                    if ekm.enable_eke:
                        f.integrate_eke()

                with self.timers["idemix"]:
                    if idm.enable_idemix:
                        f.integrate_idemix()

                with self.timers["tke"]:
                    if tkm.enable_tke:
                        f.integrate_tke()

                """
                Main boundary exchange
                for density, temp and salt this is done in integrate_tempsalt.f90
                """
                f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                   m.onx, m.je_pe + m.onx, m.u[:, :, :, m.taup1 - 1], m.nz)
                f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                m.je_pe + m.onx, m.u[:, :, :, m.taup1 - 1], m.nz)
                f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                   m.onx, m.je_pe + m.onx, m.v[:, :, :, m.taup1 - 1], m.nz)
                f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                m.je_pe + m.onx, m.v[:, :, :, m.taup1 - 1], m.nz)

                if tkm.enable_tke:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, tkm.tke[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, tkm.tke[:, :, :, m.taup1 - 1], m.nz)
                if ekm.enable_eke:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, ekm.eke[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, ekm.eke[:, :, :, m.taup1 - 1], m.nz)
                if idm.enable_idemix:
                    f.border_exchg_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe -
                                       m.onx, m.je_pe + m.onx, idm.e_iw[:, :, :, m.taup1 - 1], m.nz)
                    f.setcyclic_xyz(m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx,
                                    m.je_pe + m.onx, idm.e_iw[:, :, :, m.taup1 - 1], m.nz)

                # diagnose vertical velocity at taup1
                f.vertical_velocity()

            # shift time
            m.itt += 1
            self.time += m.dt_tracer

            self.after_timestep()

            otaum1 = m.taum1 * 1
            m.taum1 = m.tau
            m.tau = m.taup1
            m.taup1 = otaum1

            logging.info("Current iteration: {}".format(m.itt))
            logging.debug("Time step took {}s".format(self.timers["main"].getLastTime()))

        logging.debug("Timing summary:")
        logging.debug(" setup time summary       = {}s".format(self.timers["setup"].getTime()))
        logging.debug(" main loop time summary   = {}s".format(self.timers["main"].getTime()))
        logging.debug("     momentum             = {}s".format(self.timers["momentum"].getTime()))
        logging.debug("     thermodynamics       = {}s".format(self.timers["temperature"].getTime()))
        logging.debug("     EKE                  = {}s".format(self.timers["eke"].getTime()))
        logging.debug("     IDEMIX               = {}s".format(self.timers["idemix"].getTime()))
        logging.debug("     TKE                  = {}s".format(self.timers["tke"].getTime()))
<|endoftext|>


<commit_before># This file is part of cclib (http://cclib.sf.net), a library for parsing
# and interpreting the results of computational chemistry packages.
#
# Copyright (C) 2006-2014, the cclib development team
#
# The library is free software, distributed under the terms of
# the GNU Lesser General Public version 2.1 or later. You should have
# received a copy of the license along with cclib. You can also access
# the full license online at http://www.gnu.org/copyleft/lgpl.html.

import io
import collections

try:
    import bz2 # New in Python 2.3.
except ImportError:
    bz2 = None
import fileinput
import gzip
import inspect
import logging
logging.logMultiprocessing =  0 # To avoid a problem with Avogadro
import os
import random
try:
    set # Standard type from Python 2.4+.
except NameError:
    from sets import Set as set
import sys
import zipfile

import numpy

from . import utils
from .data import ccData

class myBZ2File(bz2.BZ2File):
    """Return string instead of bytes"""
    def __next__(self):
        line = super().__next__()
        return line.decode("ascii", "replace")

class myGzipFile(gzip.GzipFile):
    """Return string instead of bytes"""
    def __next__(self):
        line = super().__next__()
        return line.decode("ascii", "replace")

class FileWrapper(object):
    """Wrap a file object so that we can maintain position"""

    def __init__(self, file):
        self.file = file
        self.pos = 0
        self.file.seek(0, 2)
        self.size = self.file.tell()
        self.file.seek(0, 0)

    def next(self):
        line = next(self.file)
        self.pos += len(line)
        return line

    def __next__(self):
        return self.next()

    def __iter__(self):
        return self

    def close(self):
        self.file.close()

    def seek(self, pos, ref):
        self.file.seek(pos, ref)

def openlogfile(filename):
    """Return a file object given a filename.

    Given the filename of a log file or a gzipped, zipped, or bzipped
    log file, this function returns a regular Python file object.

    Given an address starting with http://, this function retrieves the url
    and returns a file object using a temporary file.

    Given a list of filenames, this function returns a FileInput object,
    which can be used for seamless iteration without concatenation.
    """

    # If there is a single string argument given.
    if type(filename) in [str, str]:

        extension = os.path.splitext(filename)[1]

        if extension == ".gz":
            fileobject = myGzipFile(filename, "r")

        elif extension == ".zip":
            zip = zipfile.ZipFile(filename, "r")
            assert len(zip.namelist()) == 1, "ERROR: Zip file contains more than 1 file"
            fileobject = io.StringIO(zip.read(zip.namelist()[0]).decode("ascii", "ignore"))

        elif extension in ['.bz', '.bz2']:
            # Module 'bz2' is not always importable.
            assert bz2 != None, "ERROR: module bz2 cannot be imported"
            fileobject = myBZ2File(filename, "r")

        else:
            fileobject = FileWrapper(io.open(filename, "r", errors='ignore'))

        return fileobject

    elif hasattr(filename, "__iter__"):

        # Compression (gzip and bzip) is supported as of Python 2.5.
        if sys.version_info[0] >= 2 and sys.version_info[1] >= 5:
            fileobject = fileinput.input(filename, openhook=fileinput.hook_compressed)
        else:
            fileobject = fileinput.input(filename)

        return fileobject

class Logfile(object):
    """Abstract class for logfile objects.

    Subclasses defined by cclib:
        ADF, GAMESS, GAMESSUK, Gaussian, Jaguar, Molpro, ORCA

    """

    def __init__(self, source, loglevel=logging.INFO, logname="Log",
                    logstream=sys.stdout, datatype=ccData):
        """Initialise the Logfile object.

        This should be called by a ubclass in its own __init__ method.

        Inputs:
            source - a single logfile, a list of logfiles, or input stream
        """

        # Set the filename to source if it is a string or a list of filenames.
        # In the case of an input stream, set some arbitrary name and the stream.
        # Elsewise, raise an Exception.
        if isinstance(source, str):
            self.filename = source
            self.isstream = False
        elif isinstance(source, list) and all([isinstance(s, str) for s in source]):
            self.filename = source
            self.isstream = False
        elif hasattr(source, "read"):
            self.filename = "stream %s" % str(type(source))
            self.isstream = True
            self.stream = source
        else:
            raise ValueError

        # Set up the logger.
        # Note that calling logging.getLogger() with one name always returns the same instance.
        # Presently in cclib, all parser instances of the same class use the same logger,
        #   which means that care needs to be taken not to duplicate handlers.
        self.loglevel = loglevel
        self.logname  = logname
        self.logger = logging.getLogger('%s %s' % (self.logname, self.filename))
        self.logger.setLevel(self.loglevel)
        if len(self.logger.handlers) == 0:
            handler = logging.StreamHandler(logstream)
            handler.setFormatter(logging.Formatter("[%(name)s %(levelname)s] %(message)s"))
            self.logger.addHandler(handler)

        # Periodic table of elements.
        self.table = utils.PeriodicTable()

        # This is the class that will be used in the data object returned by parse(),
        #   and should normally be ccData or a subclass.
        self.datatype = datatype

    def __setattr__(self, name, value):

        # Send info to logger if the attribute is in the list self._attrlist.
        if name in getattr(self, "_attrlist", {}) and hasattr(self, "logger"):

            # Call logger.info() only if the attribute is new.
            if not hasattr(self, name):
                if type(value) in [numpy.ndarray, list]:
                    self.logger.info("Creating attribute %s[]" %name)
                else:
                    self.logger.info("Creating attribute %s: %s" %(name, str(value)))

        # Set the attribute.
        object.__setattr__(self, name, value)

    def parse(self, progress=None, fupdate=0.05, cupdate=0.002):
        """Parse the logfile, using the assumed extract method of the child."""

        # Check that the sub-class has an extract attribute,
        #  that is callable with the proper number of arguemnts.
        if not hasattr(self, "extract"):
            raise AttributeError("Class %s has no extract() method." %self.__class__.__name__)
        if not callable(self.extract):
            raise AttributeError("Method %s._extract not callable." %self.__class__.__name__)
        if len(inspect.getargspec(self.extract)[0]) != 3:
            raise AttributeError("Method %s._extract takes wrong number of arguments." %self.__class__.__name__)

        # Save the current list of attributes to keep after parsing.
        # The dict of self should be the same after parsing.
        _nodelete = list(set(self.__dict__.keys()))

        # Initiate the FileInput object for the input files.
        # Remember that self.filename can be a list of files.
        if not self.isstream:
            inputfile = openlogfile(self.filename)
        else:
            inputfile = self.stream

        # Intialize self.progress
        if progress and not (isinstance(inputfile, myGzipFile) or
                                isinstance(inputfile, myBZ2File)):
            self.progress = progress
            self.progress.initialize(inputfile.size)
            self.progress.step = 0
        self.fupdate = fupdate
        self.cupdate = cupdate

        # Initialize the ccData object that will be returned.
        # This is normally ccData, but can be changed by passing
        #   the datatype argument to __init__().
        data = self.datatype()

        # Copy the attribute list, so that the parser knows what to expect,
        #   specifically in __setattr__().
        # The class self.datatype (normally ccData) must have this attribute.
        self._attrlist = data._attrlist

        # Maybe the sub-class has something to do before parsing.
        self.before_parsing()

        # Loop over lines in the file object and call extract().
        # This is where the actual parsing is done.
        for line in inputfile:

            self.updateprogress(inputfile, "Unsupported information", cupdate)

            # This call should check if the line begins a section of extracted data.
            # If it does, it parses some lines and sets the relevant attributes (to self).
            # Any attributes can be freely set and used across calls, however only those
            #   in data._attrlist will be moved to final data object that is returned.
            self.extract(inputfile, line)

        # Close input file object.
        if not self.isstream:
            inputfile.close()

        # Maybe the sub-class has something to do after parsing.
        self.after_parsing()

        # If atomcoords were not parsed, but some input coordinates were ("inputcoords").
        # This is originally from the Gaussian parser, a regression fix.
        if not hasattr(self, "atomcoords") and hasattr(self, "inputcoords"):
            self.atomcoords = numpy.array(self.inputcoords, 'd')

        # Set nmo if not set already - to nbasis.
        if not hasattr(self, "nmo") and hasattr(self, "nbasis"):
            self.nmo = self.nbasis

        # Creating deafult coreelectrons array.
        if not hasattr(self, "coreelectrons") and hasattr(self, "natom"):
            self.coreelectrons = numpy.zeros(self.natom, "i")

        # Move all cclib attributes to the ccData object.
        # To be moved, an attribute must be in data._attrlist.
        for attr in data._attrlist:
            if hasattr(self, attr):
                setattr(data, attr, getattr(self, attr))

        # Now make sure that the cclib attributes in the data object
        #   are all the correct type (including arrays and lists of arrays).
        data.arrayify()

        # Delete all temporary attributes (including cclib attributes).
        # All attributes should have been moved to a data object,
        #   which will be returned.
        for attr in list(self.__dict__.keys()):
            if not attr in _nodelete:
                self.__delattr__(attr)

        # Update self.progress as done.
        if hasattr(self, "progress"):
            self.progress.update(inputfile.size, "Done")

        # Return the ccData object that was generated.
        return data

    def before_parsing(self):

        pass

    def after_parsing(self):

        pass

    def updateprogress(self, inputfile, msg, xupdate=0.05):
        """Update progress."""

        if hasattr(self, "progress") and random.random() < xupdate:
            newstep = inputfile.pos
            if newstep != self.progress.step:
                self.progress.update(newstep, msg)
                self.progress.step = newstep

    def normalisesym(self, symlabel):
        """Standardise the symmetry labels between parsers.

        This method should be overwritten by individual parsers, and should
        contain appropriate doctests. If is not overwritten, this is detected
        as an error by unit tests.
        """
        return "ERROR: This should be overwritten by this subclass"

    def float(self, number):
        """Convert a string to a float avoiding the problem with Ds.

        >>> t = Logfile("dummyfile")
        >>> t.float("123.2323E+02")
        12323.23
        >>> t.float("123.2323D+02")
        12323.23
        """
        number = number.replace("D","E")
        return float(number)

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>LogFile: set attributes using ccData.setattributes<commit_after># This file is part of cclib (http://cclib.sf.net), a library for parsing
# and interpreting the results of computational chemistry packages.
#
# Copyright (C) 2006-2014, the cclib development team
#
# The library is free software, distributed under the terms of
# the GNU Lesser General Public version 2.1 or later. You should have
# received a copy of the license along with cclib. You can also access
# the full license online at http://www.gnu.org/copyleft/lgpl.html.

import io
import collections

try:
    import bz2 # New in Python 2.3.
except ImportError:
    bz2 = None
import fileinput
import gzip
import inspect
import logging
logging.logMultiprocessing =  0 # To avoid a problem with Avogadro
import os
import random
try:
    set # Standard type from Python 2.4+.
except NameError:
    from sets import Set as set
import sys
import zipfile

import numpy

from . import utils
from .data import ccData

class myBZ2File(bz2.BZ2File):
    """Return string instead of bytes"""
    def __next__(self):
        line = super().__next__()
        return line.decode("ascii", "replace")

class myGzipFile(gzip.GzipFile):
    """Return string instead of bytes"""
    def __next__(self):
        line = super().__next__()
        return line.decode("ascii", "replace")

class FileWrapper(object):
    """Wrap a file object so that we can maintain position"""

    def __init__(self, file):
        self.file = file
        self.pos = 0
        self.file.seek(0, 2)
        self.size = self.file.tell()
        self.file.seek(0, 0)

    def next(self):
        line = next(self.file)
        self.pos += len(line)
        return line

    def __next__(self):
        return self.next()

    def __iter__(self):
        return self

    def close(self):
        self.file.close()

    def seek(self, pos, ref):
        self.file.seek(pos, ref)

def openlogfile(filename):
    """Return a file object given a filename.

    Given the filename of a log file or a gzipped, zipped, or bzipped
    log file, this function returns a regular Python file object.

    Given an address starting with http://, this function retrieves the url
    and returns a file object using a temporary file.

    Given a list of filenames, this function returns a FileInput object,
    which can be used for seamless iteration without concatenation.
    """

    # If there is a single string argument given.
    if type(filename) in [str, str]:

        extension = os.path.splitext(filename)[1]

        if extension == ".gz":
            fileobject = myGzipFile(filename, "r")

        elif extension == ".zip":
            zip = zipfile.ZipFile(filename, "r")
            assert len(zip.namelist()) == 1, "ERROR: Zip file contains more than 1 file"
            fileobject = io.StringIO(zip.read(zip.namelist()[0]).decode("ascii", "ignore"))

        elif extension in ['.bz', '.bz2']:
            # Module 'bz2' is not always importable.
            assert bz2 != None, "ERROR: module bz2 cannot be imported"
            fileobject = myBZ2File(filename, "r")

        else:
            fileobject = FileWrapper(io.open(filename, "r", errors='ignore'))

        return fileobject

    elif hasattr(filename, "__iter__"):

        # Compression (gzip and bzip) is supported as of Python 2.5.
        if sys.version_info[0] >= 2 and sys.version_info[1] >= 5:
            fileobject = fileinput.input(filename, openhook=fileinput.hook_compressed)
        else:
            fileobject = fileinput.input(filename)

        return fileobject

class Logfile(object):
    """Abstract class for logfile objects.

    Subclasses defined by cclib:
        ADF, GAMESS, GAMESSUK, Gaussian, Jaguar, Molpro, ORCA

    """

    def __init__(self, source, loglevel=logging.INFO, logname="Log",
                    logstream=sys.stdout, datatype=ccData):
        """Initialise the Logfile object.

        This should be called by a ubclass in its own __init__ method.

        Inputs:
            source - a single logfile, a list of logfiles, or input stream
        """

        # Set the filename to source if it is a string or a list of filenames.
        # In the case of an input stream, set some arbitrary name and the stream.
        # Elsewise, raise an Exception.
        if isinstance(source, str):
            self.filename = source
            self.isstream = False
        elif isinstance(source, list) and all([isinstance(s, str) for s in source]):
            self.filename = source
            self.isstream = False
        elif hasattr(source, "read"):
            self.filename = "stream %s" % str(type(source))
            self.isstream = True
            self.stream = source
        else:
            raise ValueError

        # Set up the logger.
        # Note that calling logging.getLogger() with one name always returns the same instance.
        # Presently in cclib, all parser instances of the same class use the same logger,
        #   which means that care needs to be taken not to duplicate handlers.
        self.loglevel = loglevel
        self.logname  = logname
        self.logger = logging.getLogger('%s %s' % (self.logname, self.filename))
        self.logger.setLevel(self.loglevel)
        if len(self.logger.handlers) == 0:
            handler = logging.StreamHandler(logstream)
            handler.setFormatter(logging.Formatter("[%(name)s %(levelname)s] %(message)s"))
            self.logger.addHandler(handler)

        # Periodic table of elements.
        self.table = utils.PeriodicTable()

        # This is the class that will be used in the data object returned by parse(),
        #   and should normally be ccData or a subclass.
        self.datatype = datatype

    def __setattr__(self, name, value):

        # Send info to logger if the attribute is in the list self._attrlist.
        if name in getattr(self, "_attrlist", {}) and hasattr(self, "logger"):

            # Call logger.info() only if the attribute is new.
            if not hasattr(self, name):
                if type(value) in [numpy.ndarray, list]:
                    self.logger.info("Creating attribute %s[]" %name)
                else:
                    self.logger.info("Creating attribute %s: %s" %(name, str(value)))

        # Set the attribute.
        object.__setattr__(self, name, value)

    def parse(self, progress=None, fupdate=0.05, cupdate=0.002):
        """Parse the logfile, using the assumed extract method of the child."""

        # Check that the sub-class has an extract attribute,
        #  that is callable with the proper number of arguemnts.
        if not hasattr(self, "extract"):
            raise AttributeError("Class %s has no extract() method." %self.__class__.__name__)
        if not callable(self.extract):
            raise AttributeError("Method %s._extract not callable." %self.__class__.__name__)
        if len(inspect.getargspec(self.extract)[0]) != 3:
            raise AttributeError("Method %s._extract takes wrong number of arguments." %self.__class__.__name__)

        # Save the current list of attributes to keep after parsing.
        # The dict of self should be the same after parsing.
        _nodelete = list(set(self.__dict__.keys()))

        # Initiate the FileInput object for the input files.
        # Remember that self.filename can be a list of files.
        if not self.isstream:
            inputfile = openlogfile(self.filename)
        else:
            inputfile = self.stream

        # Intialize self.progress
        if progress and not (isinstance(inputfile, myGzipFile) or
                                isinstance(inputfile, myBZ2File)):
            self.progress = progress
            self.progress.initialize(inputfile.size)
            self.progress.step = 0
        self.fupdate = fupdate
        self.cupdate = cupdate

        # Initialize the ccData object that will be returned.
        # This is normally ccData, but can be changed by passing
        #   the datatype argument to __init__().
        data = self.datatype()

        # Copy the attribute list, so that the parser knows what to expect,
        #   specifically in __setattr__().
        # The class self.datatype (normally ccData) must have this attribute.
        self._attrlist = data._attrlist

        # Maybe the sub-class has something to do before parsing.
        self.before_parsing()

        # Loop over lines in the file object and call extract().
        # This is where the actual parsing is done.
        for line in inputfile:

            self.updateprogress(inputfile, "Unsupported information", cupdate)

            # This call should check if the line begins a section of extracted data.
            # If it does, it parses some lines and sets the relevant attributes (to self).
            # Any attributes can be freely set and used across calls, however only those
            #   in data._attrlist will be moved to final data object that is returned.
            self.extract(inputfile, line)

        # Close input file object.
        if not self.isstream:
            inputfile.close()

        # Maybe the sub-class has something to do after parsing.
        self.after_parsing()

        # If atomcoords were not parsed, but some input coordinates were ("inputcoords").
        # This is originally from the Gaussian parser, a regression fix.
        if not hasattr(self, "atomcoords") and hasattr(self, "inputcoords"):
            self.atomcoords = numpy.array(self.inputcoords, 'd')

        # Set nmo if not set already - to nbasis.
        if not hasattr(self, "nmo") and hasattr(self, "nbasis"):
            self.nmo = self.nbasis

        # Creating deafult coreelectrons array.
        if not hasattr(self, "coreelectrons") and hasattr(self, "natom"):
            self.coreelectrons = numpy.zeros(self.natom, "i")

        # Move all cclib attributes to the ccData object, but beware that
        # in order to be moved an attribute must be in data._attrlist.
        data.setattributes(self.__dict__)

        # Now make sure that the cclib attributes in the data object
        #   are all the correct type (including arrays and lists of arrays).
        data.arrayify()

        # Delete all temporary attributes (including cclib attributes).
        # All attributes should have been moved to a data object,
        #   which will be returned.
        for attr in list(self.__dict__.keys()):
            if not attr in _nodelete:
                self.__delattr__(attr)

        # Update self.progress as done.
        if hasattr(self, "progress"):
            self.progress.update(inputfile.size, "Done")

        # Return the ccData object that was generated.
        return data

    def before_parsing(self):

        pass

    def after_parsing(self):

        pass

    def updateprogress(self, inputfile, msg, xupdate=0.05):
        """Update progress."""

        if hasattr(self, "progress") and random.random() < xupdate:
            newstep = inputfile.pos
            if newstep != self.progress.step:
                self.progress.update(newstep, msg)
                self.progress.step = newstep

    def normalisesym(self, symlabel):
        """Standardise the symmetry labels between parsers.

        This method should be overwritten by individual parsers, and should
        contain appropriate doctests. If is not overwritten, this is detected
        as an error by unit tests.
        """
        return "ERROR: This should be overwritten by this subclass"

    def float(self, number):
        """Convert a string to a float avoiding the problem with Ds.

        >>> t = Logfile("dummyfile")
        >>> t.float("123.2323E+02")
        12323.23
        >>> t.float("123.2323D+02")
        12323.23
        """
        number = number.replace("D","E")
        return float(number)

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before># vim: set encoding=utf-8
import re

from lxml import etree

from regparser import content
from regparser.tree.depth import heuristics, rules, markers as mtypes
from regparser.tree.depth.derive import derive_depths
from regparser.tree.struct import Node
from regparser.tree.paragraph import p_level_of
from regparser.tree.xml_parser.appendices import build_non_reg_text
from regparser.tree import reg_text
from regparser.tree.xml_parser import tree_utils

def get_reg_part(reg_doc):
    """
    Depending on source, the CFR part number exists in different places. Fetch
    it, wherever it is.
    """

    potential_parts = []
    potential_parts.extend(
        # FR notice
        node.attrib['PART'] for node in reg_doc.xpath('//REGTEXT'))
    potential_parts.extend(
        # e-CFR XML, under PART/EAR
        node.text.replace('Pt.', '').strip()
        for node in reg_doc.xpath('//PART/EAR')
        if 'Pt.' in node.text)
    potential_parts.extend(
        # e-CFR XML, under FDSYS/HEADING
        node.text.replace('PART', '').strip()
        for node in reg_doc.xpath('//FDSYS/HEADING')
        if 'PART' in node.text)
    potential_parts.extend(
        # e-CFR XML, under FDSYS/GRANULENUM
        node.text.strip() for node in reg_doc.xpath('//FDSYS/GRANULENUM'))

    if potential_parts:
        return potential_parts[0]

def get_title(reg_doc):
    """ Extract the title of the regulation. """
    parent = reg_doc.xpath('//PART/HD')[0]
    title = parent.text
    return title

def preprocess_xml(xml):
    """This transforms the read XML through macros. Each macro consists of
    an xpath and a replacement xml string"""
    for path, replacement in content.Macros():
        replacement = etree.fromstring('<ROOT>' + replacement + '</ROOT>')
        for node in xml.xpath(path):
            parent = node.getparent()
            idx = parent.index(node)
            parent.remove(node)
            for repl in replacement:
                parent.insert(idx, repl)
                idx += 1

def build_tree(reg_xml):
    doc = etree.fromstring(reg_xml)
    preprocess_xml(doc)

    reg_part = get_reg_part(doc)
    title = get_title(doc)

    tree = Node("", [], [reg_part], title)

    part = doc.xpath('//PART')[0]

    subpart_xmls = [c for c in part.getchildren() if c.tag == 'SUBPART']
    if len(subpart_xmls) > 0:
        subparts = [build_subpart(reg_part, s) for s in subpart_xmls]
        tree.children = subparts
    else:
        section_xmls = [c for c in part.getchildren() if c.tag == 'SECTION']
        sections = []
        for section_xml in section_xmls:
            sections.extend(build_from_section(reg_part, section_xml))
        empty_part = reg_text.build_empty_part(reg_part)
        empty_part.children = sections
        tree.children = [empty_part]

    non_reg_sections = build_non_reg_text(doc, reg_part)
    tree.children += non_reg_sections

    return tree

def get_subpart_title(subpart_xml):
    hds = subpart_xml.xpath('./HD')
    return [hd.text for hd in hds][0]

def build_subpart(reg_part, subpart_xml):
    subpart_title = get_subpart_title(subpart_xml)
    subpart = reg_text.build_subpart(subpart_title, reg_part)

    sections = []
    for ch in subpart_xml.getchildren():
        if ch.tag == 'SECTION':
            sections.extend(build_from_section(reg_part, ch))

    subpart.children = sections
    return subpart

def get_markers(text):
    """ Extract all the paragraph markers from text. Do some checks on the
    collapsed markers."""
    markers = tree_utils.get_paragraph_markers(text)
    collapsed_markers = tree_utils.get_collapsed_markers(text)

    #   Check that the collapsed markers make sense (i.e. are at least one
    #   level below the initial marker)
    if markers and collapsed_markers:
        initial_marker_levels = p_level_of(markers[-1])
        final_collapsed_markers = []
        for collapsed_marker in collapsed_markers:
            collapsed_marker_levels = p_level_of(collapsed_marker)
            if any(c > f for f in initial_marker_levels
                    for c in collapsed_marker_levels):
                final_collapsed_markers.append(collapsed_marker)
        collapsed_markers = final_collapsed_markers
    markers_list = [m for m in markers] + [m for m in collapsed_markers]

    return markers_list

def get_markers_and_text(node, markers_list):
    node_text = tree_utils.get_node_text(node, add_spaces=True)
    text_with_tags = tree_utils.get_node_text_tags_preserved(node)

    if len(markers_list) > 1:
        actual_markers = ['(%s)' % m for m in markers_list]
        plain_markers = [m.replace('<E T="03">', '').replace('</E>', '')
                         for m in actual_markers]
        node_texts = tree_utils.split_text(node_text, plain_markers)
        tagged_texts = tree_utils.split_text(text_with_tags, actual_markers)
        node_text_list = zip(node_texts, tagged_texts)
    elif markers_list:
        node_text_list = [(node_text, text_with_tags)]
    return zip(markers_list, node_text_list)

def next_marker(xml_node, remaining_markers):
    """Try to determine the marker following the current xml_node. Remaining
    markers is a list of other marks *within* the xml_node. May return
    None"""
    #   More markers in this xml node
    if remaining_markers:
        return remaining_markers[0][0]

    #   Check the next xml node; skip over stars
    sib = xml_node.getnext()
    while sib is not None and sib.tag in ('STARS', 'PRTPAGE'):
        sib = sib.getnext()
    if sib is not None:
        next_text = tree_utils.get_node_text(sib)
        next_markers = get_markers(next_text)
        if next_markers:
            return next_markers[0]

def build_from_section(reg_part, section_xml):
    section_texts = []
    nodes = []
    # Collect paragraph markers and section text (intro text for the
    # section)
    for ch in filter(lambda ch: ch.tag in ('P', 'STARS'),
                     section_xml.getchildren()):
        text = tree_utils.get_node_text(ch, add_spaces=True)
        tagged_text = tree_utils.get_node_text_tags_preserved(ch)
        markers_list = get_markers(tagged_text.strip())

        if ch.tag == 'STARS':
            nodes.append(Node(label=[mtypes.STARS_TAG]))
        elif not markers_list:
            section_texts.append((text, tagged_text))
        else:
            for m, node_text in get_markers_and_text(ch, markers_list):
                n = Node(node_text[0], [], [m], source_xml=ch)
                n.tagged_text = unicode(node_text[1])
                nodes.append(n)
            if node_text[0].endswith('* * *'):
                nodes.append(Node(label=[mtypes.INLINE_STARS]))

    # Trailing stars don't matter; slightly more efficient to ignore them
    while nodes and nodes[-1].label[0] in mtypes.stars:
        nodes = nodes[:-1]

    # Use constraint programming to figure out possible depth assignments
    depths = derive_depths(
        [node.label[0] for node in nodes],
        [rules.depth_type_order([mtypes.lower, mtypes.ints, mtypes.roman,
                                 mtypes.upper, mtypes.em_ints,
                                 mtypes.em_roman])])
    m_stack = tree_utils.NodeStack()
    if depths:
        # Find the assignment which violates the least of our heuristics
        depths = heuristics.prefer_multiple_children(depths, 0.5)
        depths = sorted(depths, key=lambda d: d.weight, reverse=True)
        depths = depths[0]
        for node, par in zip(nodes, depths):
            if par.typ != mtypes.stars:
                last = m_stack.peek()
                node.label = [l.replace('<E T="03">', '').replace('</E>', '')
                              for l in node.label]
                if len(last) == 0:
                    m_stack.push_last((1 + par.depth, node))
                else:
                    m_stack.add(1 + par.depth, node)

    section_no = section_xml.xpath('SECTNO')[0].text
    subject_xml = section_xml.xpath('SUBJECT')
    if not subject_xml:
        subject_xml = section_xml.xpath('RESERVED')
    subject_text = subject_xml[0].text

    nodes = []
    section_nums = []
    for match in re.finditer(r'%s\.(\d+)' % reg_part, section_no):
        section_nums.append(int(match.group(1)))

    #  Span of section numbers
    if u'§§' == section_no[:2] and '-' in section_no:
        first, last = section_nums
        section_nums = []
        for i in range(first, last + 1):
            section_nums.append(i)

    for section_number in section_nums:
        section_number = str(section_number)
        plain_sect_texts = [s[0] for s in section_texts]
        tagged_sect_texts = [s[1] for s in section_texts]

        section_text = ' '.join([section_xml.text] + plain_sect_texts)
        tagged_section_text = ' '.join([section_xml.text] + tagged_sect_texts)
        section_title = u"§ " + reg_part + "." + section_number
        if subject_text:
            section_title += " " + subject_text

        sect_node = Node(
            section_text, label=[reg_part, section_number],
            title=section_title)
        sect_node.tagged_text = tagged_section_text

        m_stack.add_to_bottom((1, sect_node))

        while m_stack.size() > 1:
            m_stack.unwind()

        nodes.append(m_stack.pop()[0][1])

    return nodes
<commit_msg>Filter out blank potential parts<commit_after># vim: set encoding=utf-8
import re

from lxml import etree

from regparser import content
from regparser.tree.depth import heuristics, rules, markers as mtypes
from regparser.tree.depth.derive import derive_depths
from regparser.tree.struct import Node
from regparser.tree.paragraph import p_level_of
from regparser.tree.xml_parser.appendices import build_non_reg_text
from regparser.tree import reg_text
from regparser.tree.xml_parser import tree_utils

def get_reg_part(reg_doc):
    """
    Depending on source, the CFR part number exists in different places. Fetch
    it, wherever it is.
    """

    potential_parts = []
    potential_parts.extend(
        # FR notice
        node.attrib['PART'] for node in reg_doc.xpath('//REGTEXT'))
    potential_parts.extend(
        # e-CFR XML, under PART/EAR
        node.text.replace('Pt.', '').strip()
        for node in reg_doc.xpath('//PART/EAR')
        if 'Pt.' in node.text)
    potential_parts.extend(
        # e-CFR XML, under FDSYS/HEADING
        node.text.replace('PART', '').strip()
        for node in reg_doc.xpath('//FDSYS/HEADING')
        if 'PART' in node.text)
    potential_parts.extend(
        # e-CFR XML, under FDSYS/GRANULENUM
        node.text.strip() for node in reg_doc.xpath('//FDSYS/GRANULENUM'))
    potential_parts = [p for p in potential_parts if p.strip()]

    if potential_parts:
        return potential_parts[0]

def get_title(reg_doc):
    """ Extract the title of the regulation. """
    parent = reg_doc.xpath('//PART/HD')[0]
    title = parent.text
    return title

def preprocess_xml(xml):
    """This transforms the read XML through macros. Each macro consists of
    an xpath and a replacement xml string"""
    for path, replacement in content.Macros():
        replacement = etree.fromstring('<ROOT>' + replacement + '</ROOT>')
        for node in xml.xpath(path):
            parent = node.getparent()
            idx = parent.index(node)
            parent.remove(node)
            for repl in replacement:
                parent.insert(idx, repl)
                idx += 1

def build_tree(reg_xml):
    doc = etree.fromstring(reg_xml)
    preprocess_xml(doc)

    reg_part = get_reg_part(doc)
    title = get_title(doc)

    tree = Node("", [], [reg_part], title)

    part = doc.xpath('//PART')[0]

    subpart_xmls = [c for c in part.getchildren() if c.tag == 'SUBPART']
    if len(subpart_xmls) > 0:
        subparts = [build_subpart(reg_part, s) for s in subpart_xmls]
        tree.children = subparts
    else:
        section_xmls = [c for c in part.getchildren() if c.tag == 'SECTION']
        sections = []
        for section_xml in section_xmls:
            sections.extend(build_from_section(reg_part, section_xml))
        empty_part = reg_text.build_empty_part(reg_part)
        empty_part.children = sections
        tree.children = [empty_part]

    non_reg_sections = build_non_reg_text(doc, reg_part)
    tree.children += non_reg_sections

    return tree

def get_subpart_title(subpart_xml):
    hds = subpart_xml.xpath('./HD')
    return [hd.text for hd in hds][0]

def build_subpart(reg_part, subpart_xml):
    subpart_title = get_subpart_title(subpart_xml)
    subpart = reg_text.build_subpart(subpart_title, reg_part)

    sections = []
    for ch in subpart_xml.getchildren():
        if ch.tag == 'SECTION':
            sections.extend(build_from_section(reg_part, ch))

    subpart.children = sections
    return subpart

def get_markers(text):
    """ Extract all the paragraph markers from text. Do some checks on the
    collapsed markers."""
    markers = tree_utils.get_paragraph_markers(text)
    collapsed_markers = tree_utils.get_collapsed_markers(text)

    #   Check that the collapsed markers make sense (i.e. are at least one
    #   level below the initial marker)
    if markers and collapsed_markers:
        initial_marker_levels = p_level_of(markers[-1])
        final_collapsed_markers = []
        for collapsed_marker in collapsed_markers:
            collapsed_marker_levels = p_level_of(collapsed_marker)
            if any(c > f for f in initial_marker_levels
                    for c in collapsed_marker_levels):
                final_collapsed_markers.append(collapsed_marker)
        collapsed_markers = final_collapsed_markers
    markers_list = [m for m in markers] + [m for m in collapsed_markers]

    return markers_list

def get_markers_and_text(node, markers_list):
    node_text = tree_utils.get_node_text(node, add_spaces=True)
    text_with_tags = tree_utils.get_node_text_tags_preserved(node)

    if len(markers_list) > 1:
        actual_markers = ['(%s)' % m for m in markers_list]
        plain_markers = [m.replace('<E T="03">', '').replace('</E>', '')
                         for m in actual_markers]
        node_texts = tree_utils.split_text(node_text, plain_markers)
        tagged_texts = tree_utils.split_text(text_with_tags, actual_markers)
        node_text_list = zip(node_texts, tagged_texts)
    elif markers_list:
        node_text_list = [(node_text, text_with_tags)]
    return zip(markers_list, node_text_list)

def next_marker(xml_node, remaining_markers):
    """Try to determine the marker following the current xml_node. Remaining
    markers is a list of other marks *within* the xml_node. May return
    None"""
    #   More markers in this xml node
    if remaining_markers:
        return remaining_markers[0][0]

    #   Check the next xml node; skip over stars
    sib = xml_node.getnext()
    while sib is not None and sib.tag in ('STARS', 'PRTPAGE'):
        sib = sib.getnext()
    if sib is not None:
        next_text = tree_utils.get_node_text(sib)
        next_markers = get_markers(next_text)
        if next_markers:
            return next_markers[0]

def build_from_section(reg_part, section_xml):
    section_texts = []
    nodes = []
    # Collect paragraph markers and section text (intro text for the
    # section)
    for ch in filter(lambda ch: ch.tag in ('P', 'STARS'),
                     section_xml.getchildren()):
        text = tree_utils.get_node_text(ch, add_spaces=True)
        tagged_text = tree_utils.get_node_text_tags_preserved(ch)
        markers_list = get_markers(tagged_text.strip())

        if ch.tag == 'STARS':
            nodes.append(Node(label=[mtypes.STARS_TAG]))
        elif not markers_list:
            section_texts.append((text, tagged_text))
        else:
            for m, node_text in get_markers_and_text(ch, markers_list):
                n = Node(node_text[0], [], [m], source_xml=ch)
                n.tagged_text = unicode(node_text[1])
                nodes.append(n)
            if node_text[0].endswith('* * *'):
                nodes.append(Node(label=[mtypes.INLINE_STARS]))

    # Trailing stars don't matter; slightly more efficient to ignore them
    while nodes and nodes[-1].label[0] in mtypes.stars:
        nodes = nodes[:-1]

    # Use constraint programming to figure out possible depth assignments
    depths = derive_depths(
        [node.label[0] for node in nodes],
        [rules.depth_type_order([mtypes.lower, mtypes.ints, mtypes.roman,
                                 mtypes.upper, mtypes.em_ints,
                                 mtypes.em_roman])])
    m_stack = tree_utils.NodeStack()
    if depths:
        # Find the assignment which violates the least of our heuristics
        depths = heuristics.prefer_multiple_children(depths, 0.5)
        depths = sorted(depths, key=lambda d: d.weight, reverse=True)
        depths = depths[0]
        for node, par in zip(nodes, depths):
            if par.typ != mtypes.stars:
                last = m_stack.peek()
                node.label = [l.replace('<E T="03">', '').replace('</E>', '')
                              for l in node.label]
                if len(last) == 0:
                    m_stack.push_last((1 + par.depth, node))
                else:
                    m_stack.add(1 + par.depth, node)

    section_no = section_xml.xpath('SECTNO')[0].text
    subject_xml = section_xml.xpath('SUBJECT')
    if not subject_xml:
        subject_xml = section_xml.xpath('RESERVED')
    subject_text = subject_xml[0].text

    nodes = []
    section_nums = []
    for match in re.finditer(r'%s\.(\d+)' % reg_part, section_no):
        section_nums.append(int(match.group(1)))

    #  Span of section numbers
    if u'§§' == section_no[:2] and '-' in section_no:
        first, last = section_nums
        section_nums = []
        for i in range(first, last + 1):
            section_nums.append(i)

    for section_number in section_nums:
        section_number = str(section_number)
        plain_sect_texts = [s[0] for s in section_texts]
        tagged_sect_texts = [s[1] for s in section_texts]

        section_text = ' '.join([section_xml.text] + plain_sect_texts)
        tagged_section_text = ' '.join([section_xml.text] + tagged_sect_texts)
        section_title = u"§ " + reg_part + "." + section_number
        if subject_text:
            section_title += " " + subject_text

        sect_node = Node(
            section_text, label=[reg_part, section_number],
            title=section_title)
        sect_node.tagged_text = tagged_section_text

        m_stack.add_to_bottom((1, sect_node))

        while m_stack.size() > 1:
            m_stack.unwind()

        nodes.append(m_stack.pop()[0][1])

    return nodes
<|endoftext|>