<commit_before>import sys
import os
import re
import StringIO
import xml.etree.ElementTree
import zipfile
import tempfile

MAX_SUB_EXAMPLE_LETTERS = 1

# From http://www.daniweb.com/software-development/python/code/216865
def int2roman(number):
    numerals = { 1 : "i", 4 : "iv", 5 : "v", 9 : "ix", 10 : "x", 40 : "xl",
        50 : "l", 90 : "xc", 100 : "c", 400 : "cd", 500 : "d", 900 : "cm", 1000 : "m" }
    result = ""
    for value, numeral in sorted(numerals.items(), reverse=True):
        while number >= value:
            result += numeral
            number -= value
    return result

for x in [("office", "urn:oasis:names:tc:opendocument:xmlns:office:1.0"),
          ("style", "urn:oasis:names:tc:opendocument:xmlns:style:1.0"),
          ("text", "urn:oasis:names:tc:opendocument:xmlns:text:1.0"),
          ("xlink", "http://www.w3.org/1999/xlink"),
          ("ooo", "http://openoffice.org/2004/office"),
          ("dc", "http://purl.org/dc/elements/1.1/"),
          ("meta", "urn:oasis:names:tc:opendocument:xmlns:meta:1.0"),
          ("table", "urn:oasis:names:tc:opendocument:xmlns:table:1.0"),
          ("draw", "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"),
          ("fo", "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"),
          ("number", "urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0"),
          ("chart", "urn:oasis:names:tc:opendocument:xmlns:chart:1.0"),
          ("svg", "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"),
          ("dr3d", "urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0"),
          ("math", "http://www.w3.org/1998/Math/MathML"),
          ("form", "urn:oasis:names:tc:opendocument:xmlns:form:1.0"),
          ("script", "urn:oasis:names:tc:opendocument:xmlns:script:1.0"),
          ("ooow", "http://openoffice.org/2004/writer"),
          ("oooc", "http://openoffice.org/2004/calc"),
          ("dom", "http://www.w3.org/2001/xml-events")]:
    xml.etree.ElementTree.register_namespace(x[0], x[1])

assert sys.argv[1]

_tagre = re.compile(r"(?:\{[^}]*\})?(.*)")
def strip_prefix(tagname):
    return re.match(_tagre, tagname).group(1)

def permissible_label_char(c):
    return c.isupper()

heading_style_to_level = dict() # E.g. P60 -> 3
mapping = dict()
heading_numbers = dict() # E.g. "XX" -> [1,4,2] (= 1.4.2)

STYLEPREF = "{urn:oasis:names:tc:opendocument:xmlns:style:1.0}"
TEXTPREF = "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}"
_hre = re.compile(r"_(\d+)$")
def get_heading_styles(root):
    for elem in root:
        if strip_prefix(elem.tag) == "automatic-styles":
            for c in elem:
                if c.attrib.has_key(STYLEPREF + 'family') and c.attrib[STYLEPREF + 'family'] == 'paragraph':
                    s = c.attrib[STYLEPREF + 'parent-style-name']
                    if s.startswith("Heading_"):
                        m = re.search(_hre, s)
                        if m:
                            heading_style_to_level[c.attrib[STYLEPREF + 'name']] = int(m.group(1))

def search_and_replace_(current, current_number, current_rm_number, current_heading_number):
    for child in current:
        if strip_prefix(child.tag) == "p":
            current_number, current_rm_number = search_and_replace_paragraph(child, current_number, current_rm_number)
        elif strip_prefix(child.tag) == "h" and heading_style_to_level[child.attrib[TEXTPREF + "style-name"]] > 1:
            current_heading_number = search_and_replace_heading(child, current_heading_number)
        else:
            search_and_replace_(child, current_number, current_rm_number, current_heading_number)

def search_and_replace(root, current_number, current_rm_number, current_heading_number):
    get_heading_styles(root)
    search_and_replace_(root, current_number, current_rm_number, current_heading_number)

_labre = re.compile(r"\((!?)(#?[A-Z]+)\)\t")
def search_and_replace_paragraph(elem, start_number, start_rm_number):
    text, links = flatten(elem)
    for match in (re.finditer(_labre, text) or []):
        if match.group(1): # It's escaped; delete the '!' and move on.
            replace_in_linked_string(text, match.start(1), match.end(1), links, "")
        else:
            if mapping.has_key(match.group(2).lstrip('#')):
                sys.stderr.write("WARNING: Label (%s) is multiply defined.\n" % match.group(2).lstrip('#'))

            if match.group(2).startswith('#'):
                rm = int2roman(start_rm_number)
                mapping[match.group(2)[1:]] = rm
                replace_in_linked_string(text, match.start(2), match.end(2), links, rm)
                start_rm_number += 1
            else:
                mapping[match.group(2)] = start_number
                replace_in_linked_string(text, match.start(2), match.end(2), links, str(start_number))
                start_number += 1
    return start_number, start_rm_number

def str_heading_number(l):
    return '.'.join(map(str, l))

_headre = re.compile(r"^([A-Z]+)\.(.*)$")
_unnumbered = re.compile(r"^(\*\s*).*$")
def search_and_replace_heading(elem, start_number):
    text, links = flatten(elem)

    level = heading_style_to_level[elem.attrib[TEXTPREF + "style-name"]]

    if level - 1 == len(start_number): # Non-embedded heading
        start_number[-1] += 1
    elif level - 1 < len(start_number):
        for _ in xrange(level - 1, len(start_number)): start_number.pop()
        start_number[-1] += 1
    elif level - 1 > len(start_number):
        start_number.append(1)

    match = re.match(_headre, text)
    if match:
        if heading_numbers.has_key(match.group(1)):
            sys.stderr.write("WARNING: Heading label '%s' is multiply defined.\n" % match.group(1))
        heading_numbers[match.group(1)] = map(lambda x: x, start_number)
        replace_in_linked_string(text, match.start(1), match.end(1), links, str_heading_number(start_number))
    else:
        match2 = re.match(_unnumbered, text)
        if match2:
            # An unnumbered heading -- just remove the '*' and any whitespace following it.
            replace_in_linked_string(text, 0, match2.end(1), links, "")
        else:
            replace_in_linked_string(text, 0, 1, links, str_heading_number(start_number) + '. ' + (text and text[0] or ''))
    return start_number

def search_and_replace2(current):
    for child in current:
        if strip_prefix(child.tag) == "p":
            search_and_replace_paragraph2(child)
        else:
            search_and_replace2(child)

_labre2 = r"\((!?)([A-Z]+)(?:(?:[a-z]{0,%i}(?:[-/][a-z]{0,%i})?)|(?:[-/]([A-Z]+)))\)[^\t]" % ((MAX_SUB_EXAMPLE_LETTERS,)*2)
_headre2 = re.compile(r"(\$+)([A-Z]+)")
def search_and_replace_paragraph2(elem):
    text, links = flatten(elem)
    for match in (re.finditer(_labre2, text) or []):
        if match.group(1): # It's escaped; delete the '!' and move on.
            replace_in_linked_string(text, match.start(1), match.end(1), links, "")
        else:
            if not mapping.has_key(match.group(2)):
                sys.stderr.write("WARNING: Bad reference to (%s)\n" % match.group(2))
            else:
                replace_in_linked_string(text, match.start(2), match.end(2), links, str(mapping[match.group(2)]))

                if match.group(3):
                    if not mapping.has_key(match.group(3)):
                        sys.stderr.write("WARNING: Bad reference to (%s)\n" % match.group(3))
                    else:
                        replace_in_linked_string(text, match.start(3), match.end(3), links, str(mapping[match.group(3)]))

    text, links = flatten(elem)
    for match in (re.finditer(_headre2, text) or []):
        if len(match.group(1)) > 1: # It's escaped; strip a '$' and move on.
            replace_in_linked_string(text, match.start(1), match.start(1)+1, "")
        else:
            sl = [x for x in links.keys() if x != 'current_i']
            sl.sort()
            if not heading_numbers.has_key(match.group(2)):
                sys.stderr.write("WARNING: Bad reference to $%s\n" % match.group(1))
            else:
                replace_in_linked_string(text, match.start(), match.end(), links, str_heading_number(heading_numbers[match.group(2)]))

def flatten_(elem, text, links):
    if strip_prefix(elem.tag) == "span" and not (len(list(elem)) > 0 and strip_prefix(elem[0].tag) == "note"):
        if elem.text:
            text.write(elem.text)
            links[(links['current_i'], links['current_i']+len(elem.text))] = dict(type="text", elem=elem)
            links['current_i'] += len(elem.text)

        for child in elem:
            pr = strip_prefix(child.tag)
            if child.tail or pr == "tab": # Regex matching is sensitive to tabs so must include these.
                if pr == "tab":
                    text.write('\t')
                text.write(child.tail or "")
                l = len(child.tail or "") + (pr == "tab" and 1 or 0)
                links[(links['current_i'], links['current_i'] + l)] = dict(type="tail", elem=child)
                links['current_i'] += l
    else:
        for c in elem:
            current_i = flatten_(c, text, links)

def flatten(elem):
    text = StringIO.StringIO()
    links = dict(current_i=0)
    flatten_(elem, text, links)
    return (text.getvalue(), links)

def replace_in_linked_string(string, start, end, links, replacement):
    ks = links.keys()
    ks.sort()
    rks = filter(lambda k: k[0] < end and k[1] > start, ks)

    if len(rks) == 0:
        return
#    debug_print_linked_string(string, links, keys=rks)

    into = getattr(links[rks[0]]['elem'], links[rks[0]]['type']) or ""
    new = into[:start-rks[0][0]] + replacement + into[start-rks[0][0]+(end-start):]
    setattr(links[rks[0]]['elem'], links[rks[0]]['type'], new)
#    sys.stderr.write("Replacing 1st with '%s'\n" % new)

    for k in rks[1:-1]:
        setattr(links[k]['elem'], links[k]['type'], "")
#        sys.stderr.write("Setting %i to empty\n" % rks.index(k))

    # This hasn't ever happened, so this code isn't actually tested.
    if len(rks) >= 2:
        into2 = getattr(links[rks[-1]]['elem'], links[rks[-1]]['type']) or ""
        new = into2[end - rks[-1][0]:]
        setattr(links[rks[-1]]['elem'], links[rks[-1]]['type'], new)
#        sys.stderr.write("Setting last to '%s'\n" % new)

def debug_print_linked_string(string, links, keys=None):
    if keys is None: keys = links.keys()
    sys.stderr.write("[[\n")
    lks = links.keys()
    lks.sort()
    for k in lks:
        if k != "current_i" and k in keys:
            sys.stderr.write("%i, %i [%s]: '%s'\n" % (k[0], k[1], links[k]['type'], string[k[0]:k[1]].encode('utf-8')))
    sys.stderr.write("]]\n\n")

with zipfile.ZipFile(sys.argv[1], 'r') as odt:
    with odt.open('content.xml', 'r') as content:
        doc = xml.etree.ElementTree.parse(content)
        root = doc.getroot()
        search_and_replace(root, 1, 1, [0])
        search_and_replace2(root)

        with zipfile.ZipFile(sys.argv[2] or "out.odt", 'w') as newone:
            for n in odt.namelist():
                if n != "content.xml":
                    with odt.open(n, 'r') as f:
                        newone.writestr(n, f.read())
                else:
                    with tempfile.NamedTemporaryFile(delete=False) as t:
                        t.write("""<?xml version="1.0" encoding="utf-8" standalone="yes"?>""")
                        doc.write(t, 'utf-8')
                        t.close()
                        newone.write(t.name, n)
                        os.remove(t.name)
<commit_msg>Attempt to fix google docs numbering of footnotes (not working yet)<commit_after>import sys
import os
import re
import StringIO
import xml.etree.ElementTree
import zipfile
import tempfile

MAX_SUB_EXAMPLE_LETTERS = 1
FIX_FOOTNOTE_NUMBERING = True

# From http://www.daniweb.com/software-development/python/code/216865
def int2roman(number):
    numerals = { 1 : "i", 4 : "iv", 5 : "v", 9 : "ix", 10 : "x", 40 : "xl",
        50 : "l", 90 : "xc", 100 : "c", 400 : "cd", 500 : "d", 900 : "cm", 1000 : "m" }
    result = ""
    for value, numeral in sorted(numerals.items(), reverse=True):
        while number >= value:
            result += numeral
            number -= value
    return result

for x in [("office", "urn:oasis:names:tc:opendocument:xmlns:office:1.0"),
          ("style", "urn:oasis:names:tc:opendocument:xmlns:style:1.0"),
          ("text", "urn:oasis:names:tc:opendocument:xmlns:text:1.0"),
          ("xlink", "http://www.w3.org/1999/xlink"),
          ("ooo", "http://openoffice.org/2004/office"),
          ("dc", "http://purl.org/dc/elements/1.1/"),
          ("meta", "urn:oasis:names:tc:opendocument:xmlns:meta:1.0"),
          ("table", "urn:oasis:names:tc:opendocument:xmlns:table:1.0"),
          ("draw", "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"),
          ("fo", "urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0"),
          ("number", "urn:oasis:names:tc:opendocument:xmlns:datastyle:1.0"),
          ("chart", "urn:oasis:names:tc:opendocument:xmlns:chart:1.0"),
          ("svg", "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"),
          ("dr3d", "urn:oasis:names:tc:opendocument:xmlns:dr3d:1.0"),
          ("math", "http://www.w3.org/1998/Math/MathML"),
          ("form", "urn:oasis:names:tc:opendocument:xmlns:form:1.0"),
          ("script", "urn:oasis:names:tc:opendocument:xmlns:script:1.0"),
          ("ooow", "http://openoffice.org/2004/writer"),
          ("oooc", "http://openoffice.org/2004/calc"),
          ("dom", "http://www.w3.org/2001/xml-events")]:
    xml.etree.ElementTree.register_namespace(x[0], x[1])

assert sys.argv[1]

_tagre = re.compile(r"(?:\{[^}]*\})?(.*)")
def strip_prefix(tagname):
    return re.match(_tagre, tagname).group(1)

def permissible_label_char(c):
    return c.isupper()

heading_style_to_level = dict() # E.g. P60 -> 3
mapping = dict()
heading_numbers = dict() # E.g. "XX" -> [1,4,2] (= 1.4.2)

STYLEPREF = "{urn:oasis:names:tc:opendocument:xmlns:style:1.0}"
TEXTPREF = "{urn:oasis:names:tc:opendocument:xmlns:text:1.0}"
_hre = re.compile(r"_(\d+)$")
def get_heading_styles(root):
    for elem in root:
        if strip_prefix(elem.tag) == "automatic-styles":
            for c in elem:
                if c.attrib.has_key(STYLEPREF + 'family') and c.attrib[STYLEPREF + 'family'] == 'paragraph':
                    s = c.attrib[STYLEPREF + 'parent-style-name']
                    if s.startswith("Heading_"):
                        m = re.search(_hre, s)
                        if m:
                            heading_style_to_level[c.attrib[STYLEPREF + 'name']] = int(m.group(1))

def search_and_replace_(current, current_number, current_rm_number, current_heading_number):
    for child in current:
        if strip_prefix(child.tag) == "p":
            current_number, current_rm_number = search_and_replace_paragraph(child, current_number, current_rm_number)
        elif strip_prefix(child.tag) == "h" and heading_style_to_level[child.attrib[TEXTPREF + "style-name"]] > 1:
            current_heading_number = search_and_replace_heading(child, current_heading_number)
        else:
            search_and_replace_(child, current_number, current_rm_number, current_heading_number)

def search_and_replace(root, current_number, current_rm_number, current_heading_number):
    get_heading_styles(root)
    search_and_replace_(root, current_number, current_rm_number, current_heading_number)

_labre = re.compile(r"\((!?)(#?[A-Z]+)\)\t")
def search_and_replace_paragraph(elem, start_number, start_rm_number):
    text, links = flatten(elem)
    for match in (re.finditer(_labre, text) or []):
        if match.group(1): # It's escaped; delete the '!' and move on.
            replace_in_linked_string(text, match.start(1), match.end(1), links, "")
        else:
            if mapping.has_key(match.group(2).lstrip('#')):
                sys.stderr.write("WARNING: Label (%s) is multiply defined.\n" % match.group(2).lstrip('#'))

            if match.group(2).startswith('#'):
                rm = int2roman(start_rm_number)
                mapping[match.group(2)[1:]] = rm
                replace_in_linked_string(text, match.start(2), match.end(2), links, rm)
                start_rm_number += 1
            else:
                mapping[match.group(2)] = start_number
                replace_in_linked_string(text, match.start(2), match.end(2), links, str(start_number))
                start_number += 1
    return start_number, start_rm_number

def str_heading_number(l):
    return '.'.join(map(str, l))

_headre = re.compile(r"^([A-Z]+)\.(.*)$")
_unnumbered = re.compile(r"^(\*\s*).*$")
def search_and_replace_heading(elem, start_number):
    text, links = flatten(elem)

    level = heading_style_to_level[elem.attrib[TEXTPREF + "style-name"]]

    if level - 1 == len(start_number): # Non-embedded heading
        start_number[-1] += 1
    elif level - 1 < len(start_number):
        for _ in xrange(level - 1, len(start_number)): start_number.pop()
        start_number[-1] += 1
    elif level - 1 > len(start_number):
        start_number.append(1)

    match = re.match(_headre, text)
    if match:
        if heading_numbers.has_key(match.group(1)):
            sys.stderr.write("WARNING: Heading label '%s' is multiply defined.\n" % match.group(1))
        heading_numbers[match.group(1)] = map(lambda x: x, start_number)
        replace_in_linked_string(text, match.start(1), match.end(1), links, str_heading_number(start_number))
    else:
        match2 = re.match(_unnumbered, text)
        if match2:
            # An unnumbered heading -- just remove the '*' and any whitespace following it.
            replace_in_linked_string(text, 0, match2.end(1), links, "")
        else:
            replace_in_linked_string(text, 0, 1, links, str_heading_number(start_number) + '. ' + (text and text[0] or ''))
    return start_number

def search_and_replace2(current):
    for child in current:
        if strip_prefix(child.tag) == "p":
            search_and_replace_paragraph2(child)
        else:
            search_and_replace2(child)

_labre2 = r"\((!?)([A-Z]+)(?:(?:[a-z]{0,%i}(?:[-/][a-z]{0,%i})?)|(?:[-/]([A-Z]+)))\)[^\t]" % ((MAX_SUB_EXAMPLE_LETTERS,)*2)
_headre2 = re.compile(r"(\$+)([A-Z]+)")
def search_and_replace_paragraph2(elem):
    text, links = flatten(elem)
    for match in (re.finditer(_labre2, text) or []):
        if match.group(1): # It's escaped; delete the '!' and move on.
            replace_in_linked_string(text, match.start(1), match.end(1), links, "")
        else:
            if not mapping.has_key(match.group(2)):
                sys.stderr.write("WARNING: Bad reference to (%s)\n" % match.group(2))
            else:
                replace_in_linked_string(text, match.start(2), match.end(2), links, str(mapping[match.group(2)]))

                if match.group(3):
                    if not mapping.has_key(match.group(3)):
                        sys.stderr.write("WARNING: Bad reference to (%s)\n" % match.group(3))
                    else:
                        replace_in_linked_string(text, match.start(3), match.end(3), links, str(mapping[match.group(3)]))

    text, links = flatten(elem)
    for match in (re.finditer(_headre2, text) or []):
        if len(match.group(1)) > 1: # It's escaped; strip a '$' and move on.
            replace_in_linked_string(text, match.start(1), match.start(1)+1, "")
        else:
            sl = [x for x in links.keys() if x != 'current_i']
            sl.sort()
            if not heading_numbers.has_key(match.group(2)):
                sys.stderr.write("WARNING: Bad reference to $%s\n" % match.group(1))
            else:
                replace_in_linked_string(text, match.start(), match.end(), links, str_heading_number(heading_numbers[match.group(2)]))

def number_footnotes(elem, cite_count=1, fn_count=1):
    if strip_prefix(elem.tag) == "note-citation":
        print "REPLACING!"
        elem.text = str(cite_count)
        cite_count += 1
    elif strip_prefix(elem.tag) == "note":
        print "REPLACING! (2)"
        elem.attrib[TEXTPREF + 'id'] = "ftn%i" % fn_count
        fn_count += 1

    for c in elem:
        cite_count, fn_count = number_footnotes(c, cite_count, fn_count)
    return cite_count, fn_count

def flatten_(elem, text, links):
    if strip_prefix(elem.tag) == "span" and not (len(list(elem)) > 0 and strip_prefix(elem[0].tag) == "note"):
        if elem.text:
            text.write(elem.text)
            links[(links['current_i'], links['current_i']+len(elem.text))] = dict(type="text", elem=elem)
            links['current_i'] += len(elem.text)

        for child in elem:
            pr = strip_prefix(child.tag)
            if child.tail or pr == "tab": # Regex matching is sensitive to tabs so must include these.
                if pr == "tab":
                    text.write('\t')
                text.write(child.tail or "")
                l = len(child.tail or "") + (pr == "tab" and 1 or 0)
                links[(links['current_i'], links['current_i'] + l)] = dict(type="tail", elem=child)
                links['current_i'] += l
    else:
        for c in elem:
            current_i = flatten_(c, text, links)

def flatten(elem):
    text = StringIO.StringIO()
    links = dict(current_i=0)
    flatten_(elem, text, links)
    return (text.getvalue(), links)

def replace_in_linked_string(string, start, end, links, replacement):
    ks = links.keys()
    ks.sort()
    rks = filter(lambda k: k[0] < end and k[1] > start, ks)

    if len(rks) == 0:
        return
#    debug_print_linked_string(string, links, keys=rks)

    into = getattr(links[rks[0]]['elem'], links[rks[0]]['type']) or ""
    new = into[:start-rks[0][0]] + replacement + into[start-rks[0][0]+(end-start):]
    setattr(links[rks[0]]['elem'], links[rks[0]]['type'], new)
#    sys.stderr.write("Replacing 1st with '%s'\n" % new)

    for k in rks[1:-1]:
        setattr(links[k]['elem'], links[k]['type'], "")
#        sys.stderr.write("Setting %i to empty\n" % rks.index(k))

    # This hasn't ever happened, so this code isn't actually tested.
    if len(rks) >= 2:
        into2 = getattr(links[rks[-1]]['elem'], links[rks[-1]]['type']) or ""
        new = into2[end - rks[-1][0]:]
        setattr(links[rks[-1]]['elem'], links[rks[-1]]['type'], new)
#        sys.stderr.write("Setting last to '%s'\n" % new)

def debug_print_linked_string(string, links, keys=None):
    if keys is None: keys = links.keys()
    sys.stderr.write("[[\n")
    lks = links.keys()
    lks.sort()
    for k in lks:
        if k != "current_i" and k in keys:
            sys.stderr.write("%i, %i [%s]: '%s'\n" % (k[0], k[1], links[k]['type'], string[k[0]:k[1]].encode('utf-8')))
    sys.stderr.write("]]\n\n")

with zipfile.ZipFile(sys.argv[1], 'r') as odt:
    with odt.open('content.xml', 'r') as content:
        doc = xml.etree.ElementTree.parse(content)
        root = doc.getroot()
        search_and_replace(root, 1, 1, [0])
        search_and_replace2(root)
        if FIX_FOOTNOTE_NUMBERING:
            number_footnotes(root, 1)

        with zipfile.ZipFile(sys.argv[2] or "out.odt", 'w') as newone:
            for n in odt.namelist():
                if n != "content.xml":
                    with odt.open(n, 'r') as f:
                        newone.writestr(n, f.read())
                else:
                    with tempfile.NamedTemporaryFile(delete=False) as t:
                        t.write("""<?xml version="1.0" encoding="utf-8" standalone="yes"?>""")
                        doc.write(t, 'utf-8')
                        t.close()
                        newone.write(t.name, n)
                        os.remove(t.name)
<|endoftext|>


<commit_before>from metro.models import IndicatorQuarterResult
from django import template

register = template.Library()

@register.filter(name="percentage")
def percentage(value, arg):
    """
    convert quarterly value into percentage
    """
    if not value:
        return 0
    if not arg:
        return 0
    if "%" in value:
        return float(value.replace("%", ""))
    else:
        result = (
            IndicatorQuarterResult.clean_value(value)
            / IndicatorQuarterResult.clean_value(arg)
        ) * 100
        return int(result)
<commit_msg>percentage must not be more the 100, so that they can fit on chart<commit_after>from metro.models import IndicatorQuarterResult
from django import template

register = template.Library()

@register.filter(name="percentage")
def percentage(value, arg):
    """
    convert quarterly value into percentage
    """
    if not value:
        return 0
    if not arg:
        return 0
    if "%" in value:
        return float(value.replace("%", ""))
    else:
        result = (
            IndicatorQuarterResult.clean_value(value)
            / IndicatorQuarterResult.clean_value(arg)
        ) * 100
        if result > 100:
            return 99
        return int(result)
<|endoftext|>


<commit_before>"""
Abstraction layer for persistence operations.

As much as `SQLAlchemy` provides a great deal of power, overuse of its features
creates dangerous coupling within applications. The two worst violations are:

 a. Using models directly to perform persistence operations causes persistence code
    to permeate all layers of the application, making it hard to change relationships
    and to write generic service logic in terms of a uniform persistence interface.

 b. Using explicit relationships between models makes it harder to migrate responsiblity
    for one side of the relationship to different services.

Instead, persistence operations should pass through a `Store` layer and should obey
CRUD conventions as much as possible.

"""
from contextlib import contextmanager

from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound

from microcosm_postgres.context import SessionContext
from microcosm_postgres.diff import Version
from microcosm_postgres.errors import (
    DuplicateModelError,
    MissingDependencyError,
    ModelIntegrityError,
    ModelNotFoundError,
    ReferencedModelError
)
from microcosm_postgres.identifiers import new_object_id

class Store(object):

    def __init__(self, graph, model_class):
        self.graph = graph
        self.model_class = model_class
        # Give the model class a backref to allow model-oriented CRUD
        # short cuts while still having an abstraction layer we can replace.
        self.model_class.store = self

    @property
    def session(self):
        return SessionContext.session

    def new_object_id(self):
        """
        Injectable id generation to facilitate mocking.

        """
        return new_object_id()

    @contextmanager
    def flushing(self):
        """
        Flush the current session, handling common errors.

        """
        try:
            yield
            self.session.flush()
        except IntegrityError as error:
            error_message = str(error)
            # There ought to be a cleaner way to capture this condition
            if "duplicate" in error_message or "already exists" in error_message:
                raise DuplicateModelError(error)
            elif "still referenced" in error_message:
                raise ReferencedModelError(error)
            elif "is not present" in error_message:
                raise MissingDependencyError(error)
            else:
                raise ModelIntegrityError(error)

    def create(self, instance):
        """
        Create a new model instance.

        """
        with self.flushing():
            if instance.id is None:
                instance.id = self.new_object_id()
            self.session.add(instance)
        return instance

    def retrieve(self, identifier, *criterion):
        """
        Retrieve a model by primary key and zero or more other criteria.

        :raises `NotFound` if there is no existing model

        """
        return self._retrieve(
            self.model_class.id == identifier,
            *criterion
        )

    def update(self, identifier, new_instance):
        """
        Update an existing model with a new one.

        :raises `ModelNotFoundError` if there is no existing model

        """
        with self.flushing():
            instance = self.retrieve(identifier)
            self.merge(instance, new_instance)
            instance.updated_at = instance.new_timestamp()
        return instance

    def update_with_diff(self, identifier, new_instance):
        """
        Update an existing model with a new one.

        :raises `ModelNotFoundError` if there is no existing model

        """
        with self.flushing():
            instance = self.retrieve(identifier)
            before = Version(instance)
            self.merge(instance, new_instance)
            instance.updated_at = instance.new_timestamp()
            after = Version(instance)
        return instance, before - after

    def replace(self, identifier, new_instance):
        """
        Create or update a model.

        """
        try:
            # Note that `self.update()` ultimately calls merge, which will not enforce
            # a strict replacement; absent fields will default to the current values.
            return self.update(identifier, new_instance)
        except ModelNotFoundError:
            return self.create(new_instance)

    def delete(self, identifier):
        """
        Delete a model by primary key.

        :raises `ModelNotFoundError` if the row cannot be deleted.

        """
        return self._delete(self.model_class.id == identifier)

    def count(self, *criterion, **kwargs):
        """
        Count the number of models matching some criterion.

        """
        query = self._query(*criterion)
        query = self._filter(query, **kwargs)
        return query.count()

    def search(self, *criterion, **kwargs):
        """
        Return the list of models matching some criterion.

        :param offset: pagination offset, if any
        :param limit: pagination limit, if any

        """
        query = self._query(*criterion)
        query = self._order_by(query, **kwargs)
        query = self._filter(query, **kwargs)
        return query.all()

    def search_exact(self, *criterion, **kwargs):
        """
        Returns the first exact match based on criteria or None.

        :param offset: pagination offset, if any
        :param limit: pagination limit, if any

        """
        query = self._query(*criterion)
        query = self._order_by(query, **kwargs)
        query = self._filter(query, **kwargs)
        return query.first()

    def expunge(self, instance):
        return self.session.expunge(instance)

    def merge(self, instance, new_instance):
        self.session.merge(new_instance)

    def _order_by(self, query, **kwargs):
        """
        Add an order by clause to a (search) query.

        By default, is a noop.

        """
        return query

    def _filter(self, query, **kwargs):
        """
        Filter a query with user-supplied arguments.

        :param offset: pagination offset, if any
        :param limit: pagination limit, if any

        """
        offset, limit = kwargs.get("offset"), kwargs.get("limit")
        if offset is not None:
            query = query.offset(offset)
        if limit is not None:
            query = query.limit(limit)
        return query

    def _retrieve(self, *criterion):
        """
        Retrieve a model by some criteria.

        :raises `ModelNotFoundError` if the row cannot be deleted.

        """
        try:
            return self._query(*criterion).one()
        except NoResultFound as error:
            raise ModelNotFoundError(
                "{} not found".format(
                    self.model_class.__name__,
                ),
                error,
            )

    def _delete(self, *criterion):
        """
        Delete a model by some criterion.

        Avoids race-condition check-then-delete logic by checking the count of affected rows.

        :raises `ResourceNotFound` if the row cannot be deleted.

        """
        with self.flushing():
            count = self._query(*criterion).delete()
        if count == 0:
            raise ModelNotFoundError
        return True

    def _query(self, *criterion):
        """
        Construct a query for the model.

        """
        return self.session.query(
            self.model_class
        ).filter(
            *criterion
        )
<commit_msg>Removing comments<commit_after>"""
Abstraction layer for persistence operations.

As much as `SQLAlchemy` provides a great deal of power, overuse of its features
creates dangerous coupling within applications. The two worst violations are:

 a. Using models directly to perform persistence operations causes persistence code
    to permeate all layers of the application, making it hard to change relationships
    and to write generic service logic in terms of a uniform persistence interface.

 b. Using explicit relationships between models makes it harder to migrate responsiblity
    for one side of the relationship to different services.

Instead, persistence operations should pass through a `Store` layer and should obey
CRUD conventions as much as possible.

"""
from contextlib import contextmanager

from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound

from microcosm_postgres.context import SessionContext
from microcosm_postgres.diff import Version
from microcosm_postgres.errors import (
    DuplicateModelError,
    MissingDependencyError,
    ModelIntegrityError,
    ModelNotFoundError,
    ReferencedModelError
)
from microcosm_postgres.identifiers import new_object_id

class Store(object):

    def __init__(self, graph, model_class):
        self.graph = graph
        self.model_class = model_class
        # Give the model class a backref to allow model-oriented CRUD
        # short cuts while still having an abstraction layer we can replace.
        self.model_class.store = self

    @property
    def session(self):
        return SessionContext.session

    def new_object_id(self):
        """
        Injectable id generation to facilitate mocking.

        """
        return new_object_id()

    @contextmanager
    def flushing(self):
        """
        Flush the current session, handling common errors.

        """
        try:
            yield
            self.session.flush()
        except IntegrityError as error:
            error_message = str(error)
            # There ought to be a cleaner way to capture this condition
            if "duplicate" in error_message or "already exists" in error_message:
                raise DuplicateModelError(error)
            elif "still referenced" in error_message:
                raise ReferencedModelError(error)
            elif "is not present" in error_message:
                raise MissingDependencyError(error)
            else:
                raise ModelIntegrityError(error)

    def create(self, instance):
        """
        Create a new model instance.

        """
        with self.flushing():
            if instance.id is None:
                instance.id = self.new_object_id()
            self.session.add(instance)
        return instance

    def retrieve(self, identifier, *criterion):
        """
        Retrieve a model by primary key and zero or more other criteria.

        :raises `NotFound` if there is no existing model

        """
        return self._retrieve(
            self.model_class.id == identifier,
            *criterion
        )

    def update(self, identifier, new_instance):
        """
        Update an existing model with a new one.

        :raises `ModelNotFoundError` if there is no existing model

        """
        with self.flushing():
            instance = self.retrieve(identifier)
            self.merge(instance, new_instance)
            instance.updated_at = instance.new_timestamp()
        return instance

    def update_with_diff(self, identifier, new_instance):
        """
        Update an existing model with a new one.

        :raises `ModelNotFoundError` if there is no existing model

        """
        with self.flushing():
            instance = self.retrieve(identifier)
            before = Version(instance)
            self.merge(instance, new_instance)
            instance.updated_at = instance.new_timestamp()
            after = Version(instance)
        return instance, before - after

    def replace(self, identifier, new_instance):
        """
        Create or update a model.

        """
        try:
            # Note that `self.update()` ultimately calls merge, which will not enforce
            # a strict replacement; absent fields will default to the current values.
            return self.update(identifier, new_instance)
        except ModelNotFoundError:
            return self.create(new_instance)

    def delete(self, identifier):
        """
        Delete a model by primary key.

        :raises `ModelNotFoundError` if the row cannot be deleted.

        """
        return self._delete(self.model_class.id == identifier)

    def count(self, *criterion, **kwargs):
        """
        Count the number of models matching some criterion.

        """
        query = self._query(*criterion)
        query = self._filter(query, **kwargs)
        return query.count()

    def search(self, *criterion, **kwargs):
        """
        Return the list of models matching some criterion.

        :param offset: pagination offset, if any
        :param limit: pagination limit, if any

        """
        query = self._query(*criterion)
        query = self._order_by(query, **kwargs)
        query = self._filter(query, **kwargs)
        return query.all()

    def search_exact(self, *criterion, **kwargs):
        """
        Returns the first exact match based on criteria or None.

        """
        query = self._query(*criterion)
        query = self._order_by(query, **kwargs)
        query = self._filter(query, **kwargs)
        return query.first()

    def expunge(self, instance):
        return self.session.expunge(instance)

    def merge(self, instance, new_instance):
        self.session.merge(new_instance)

    def _order_by(self, query, **kwargs):
        """
        Add an order by clause to a (search) query.

        By default, is a noop.

        """
        return query

    def _filter(self, query, **kwargs):
        """
        Filter a query with user-supplied arguments.

        :param offset: pagination offset, if any
        :param limit: pagination limit, if any

        """
        offset, limit = kwargs.get("offset"), kwargs.get("limit")
        if offset is not None:
            query = query.offset(offset)
        if limit is not None:
            query = query.limit(limit)
        return query

    def _retrieve(self, *criterion):
        """
        Retrieve a model by some criteria.

        :raises `ModelNotFoundError` if the row cannot be deleted.

        """
        try:
            return self._query(*criterion).one()
        except NoResultFound as error:
            raise ModelNotFoundError(
                "{} not found".format(
                    self.model_class.__name__,
                ),
                error,
            )

    def _delete(self, *criterion):
        """
        Delete a model by some criterion.

        Avoids race-condition check-then-delete logic by checking the count of affected rows.

        :raises `ResourceNotFound` if the row cannot be deleted.

        """
        with self.flushing():
            count = self._query(*criterion).delete()
        if count == 0:
            raise ModelNotFoundError
        return True

    def _query(self, *criterion):
        """
        Construct a query for the model.

        """
        return self.session.query(
            self.model_class
        ).filter(
            *criterion
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
@file
@brief Quelques problèmes récurrents avec `pandas <http://pandas.pydata.org/>`_.
"""

def read_csv(filepath_or_buffer, encoding="utf8", sep="\t", **args):
    """
    Calls function `read_csv <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html?highlight=read_csv#pandas.read_csv>`_
    with different defaults values. If the encoding is utf8 and the data is a file name, the function
    checks there is no BOM at the beginning. Otherwise, it uses the encoding ``utf-8-sig``.

    @param      encoding                encoding
    @param      filepath_or_buffer      filepath_or_buffer
    @param      sep                     column separator
    @return                             DataFrame

    @FAQ(pandas___Caractères bizarres en utf8 et sous Windows (BOM) ?)

    .. index:: encoding, BOM, UTF8

    Sous Windows, certains logiciels comme `Notepad <http://fr.wikipedia.org/wiki/Bloc-notes_%28Windows%29>`_
    permettent d'enregister un fichier sous différents `encodings <http://en.wikipedia.org/wiki/Character_encoding>`_.
    Avec l'encoding `UTF8 <http://fr.wikipedia.org/wiki/UTF-8>`_, on a parfois un problème avec le premier caractère
    ``\\ufeff`` car Notepad ajoute ce qu'on appelle un `BOM <http://fr.wikipedia.org/wiki/Indicateur_d%27ordre_des_octets>`_.
    Par exemple ::

        import pandas
        df = pandas.read_csv("dataframe.txt",sep="\\t", encoding="utf8")
        print(df)

    Provoque une erreur des plus énervantes ::

        UnicodeEncodeError: 'charmap' codec can't encode character '\\ufeff' in position 0: character maps to <undefined>

    Pour contrecarrer ceci, il suffit de modifier l'encoding par `utf-8-sig <https://docs.python.org/3.4/library/codecs.html#encodings-and-unicode>`_ ::

        import pandas
        df = pandas.read_csv("dataframe.txt",sep="\\t", encoding="utf-8-sig")
        print(df)

    @endFAQ
    """
    import pandas
    if isinstance(filepath_or_buffer, str):
        if encoding in ["utf8", "utf-8"]:
            try:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding=encoding,
                    sep=sep,
                    **args)
                if df.columns[0].startswith("\ufeff"):
                    raise UnicodeError(
                        "'charmap' codec can't encode characters in position 0-1325: character maps to <undefined>")
                return df
            except UnicodeError:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding="utf-8-sig",
                    sep=sep,
                    **args)
                return df
            except UnicodeDecodeError:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding="utf-8-sig",
                    sep=sep,
                    **args)
                return df
        else:
            return pandas.read_csv(
                filepath_or_buffer, encoding=encoding, sep=sep, **args)
    else:
        return pandas.read_csv(
            filepath_or_buffer, encoding=encoding, sep=sep, **args)

def df_to_clipboard(df, **args):
    """
    Copy a dataframe as csv text into the clipboard

    @param      df      dataframe
    @param      sep     by default the separator is ``\\t`` for this function until it is defined otherwise

    It relies on method
    `to_clipboard <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_clipboard.html>`_.

    @FAQ(pandas___Copier un dataframe dans le presse-papier - clipboard)

    Pour récupérer un dataframe dans Excel, on peut utiliser la méthode
    `to_excel <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_excel.html>`_
    puis ouvrir le fichier dans Excel ou le copier dans le presse-papier et le coller
    dans une feuille ouverte dans Excel. C'est l'objet de la méthode
    `to_clipboard <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_clipboard.html>`_ ::

        df = pandas.DataFrame ( ... )
        df.to_clipboard(sep="\\t")

    @endFAQ
    """
    if "sep" in args:
        df.to_clipboard(**args)
    else:
        df.to_clipboard(sep="\t", **args)

def df_equal(df1, df2):
    """
    compares two dataframe and tells if they are equal

    @param          df1     first dataframe
    @param          df2     second dataframe
    @return                 boolean

    The function compare column one by one.
    It does not check the order of the columns is the same.
    It reorders the columns before doing the comparison.

    If you need more complex comparison,
    you can look into function `assert_frame_equal <https://github.com/pydata/pandas/blob/master/pandas/util/testing.py>`_.

    The function does not handle well NaN values because ``numpy.nan != numpy.nan`` is true.
    It also compares types:

    @FAQ(pandas___Comment comparer deux dataframe?)

    Ecrire ``df1 == df2`` ne compare pas deux dataframes entre deux
    car le sens n'est pas forcément le même pour tout le monde.
    Même si les valeurs sont les mêmes, est-ce l'ordre des colonnes
    est important ?
    Il faut donc le faire soi-même. Le code ci-dessus
    compare d'abord les dimensions, ensuite compare l'ordre
    des colonnes puis enfin les valeurs ::

        if df1.shape != df2.shape:
            return False
        l1 = list(df1.columns)
        l2 = list(df2.columns)
        l1.sort()
        l2.sort()
        if l1 != l2:
            return False
        df1 = df1[l1]
        df2 = df2[l2]
        t = (df1 == df2).all()
        s = set(t)
        return False not in s

    @endFAQ
    """
    if df1.shape != df2.shape:
        return False
    l1 = list(df1.columns)
    l2 = list(df2.columns)
    l1.sort()
    l2.sort()
    if l1 != l2:
        return False
    df1 = df1[l1]
    df2 = df2[l2]
    s = set((df1.dtypes == df2.dtypes))
    if False in s:
        return False
    s = set((df1 == df2).all())
    return False not in s

def groupby_topn(df, by_keys, sort_keys, ascending=True, n=1, as_index=True):
    """
    takes the top n rows per group

    @param      df          dataframe
    @param      by_keys     rows will be grouped by these columns
    @param      sort_keys   rows will be sorted by these columns
    @param      ascending   parameter associated to sord function
    @param      n           n in top *n*
    @param      as_index    if False, remove the index after the group by
    @return                 result

    @FAQ(pandas___top n lignes avec pandas)

    Grouper puis garder les premières observations de ce groupe est un problème
    classique. Il n'existe pas de meilleure façon de le faire,
    cela dépend du nombre d'obervations par groupe. Le moyen le plus simple
    de le faire avec pandas est :

    * grouper les lignes
    * trier les lignes dans chaque groupe
    * garder les premières lignes dans chaque groupe

    Ceci donne ::

        df.groupby(by_keys)
          .apply(lambda x: x.sort(sort_keys, ascending=ascending).head(head))
          .reset_index(drop=True)

    La dernière instruction supprimer l'index ce qui donne au dataframe final
    la même structure que le dataframe initial.

    .. runpython::
        :showcode:

        import pandas
        l = [ dict(k1="a", k2="b", v=4, i=1),
              dict(k1="a", k2="b", v=5, i=1),
              dict(k1="a", k2="b", v=4, i=2),
              dict(k1="b", k2="b", v=1, i=2),
              dict(k1="b", k2="b", v=1, i=3)]
        df = pandas.DataFrame(l)
        df.groupby(["k1", "k2"]).apply(lambda x: x.sort(["v", "i"], ascending=True).head(1))

    @endFAQ
    """
    res = df.groupby(by_keys).apply(lambda x: x.sort(
        sort_keys, ascending=ascending).head(n))
    if not as_index:
        res = res.reset_index(drop=True)
    return res
<commit_msg>documentation<commit_after># -*- coding: utf-8 -*-
"""
@file
@brief Quelques problèmes récurrents avec `pandas <http://pandas.pydata.org/>`_.
"""

def read_csv(filepath_or_buffer, encoding="utf8", sep="\t", **args):
    """
    Calls function `read_csv <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html?highlight=read_csv#pandas.read_csv>`_
    with different defaults values. If the encoding is utf8 and the data is a file name, the function
    checks there is no BOM at the beginning. Otherwise, it uses the encoding ``utf-8-sig``.

    @param      encoding                encoding
    @param      filepath_or_buffer      filepath_or_buffer
    @param      sep                     column separator
    @return                             DataFrame

    @FAQ(pandas___Caractères bizarres en utf8 et sous Windows (BOM) ?)

    .. index:: encoding, BOM, UTF8

    Sous Windows, certains logiciels comme `Notepad <http://fr.wikipedia.org/wiki/Bloc-notes_%28Windows%29>`_
    permettent d'enregister un fichier sous différents `encodings <http://en.wikipedia.org/wiki/Character_encoding>`_.
    Avec l'encoding `UTF8 <http://fr.wikipedia.org/wiki/UTF-8>`_, on a parfois un problème avec le premier caractère
    ``\\ufeff`` car Notepad ajoute ce qu'on appelle un `BOM <http://fr.wikipedia.org/wiki/Indicateur_d%27ordre_des_octets>`_.
    Par exemple ::

        import pandas
        df = pandas.read_csv("dataframe.txt",sep="\\t", encoding="utf8")
        print(df)

    Provoque une erreur des plus énervantes ::

        UnicodeEncodeError: 'charmap' codec can't encode character '\\ufeff' in position 0: character maps to <undefined>

    Pour contrecarrer ceci, il suffit de modifier l'encoding par `utf-8-sig <https://docs.python.org/3.4/library/codecs.html#encodings-and-unicode>`_ ::

        import pandas
        df = pandas.read_csv("dataframe.txt",sep="\\t", encoding="utf-8-sig")
        print(df)

    @endFAQ
    """
    import pandas
    if isinstance(filepath_or_buffer, str):
        if encoding in ["utf8", "utf-8"]:
            try:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding=encoding,
                    sep=sep,
                    **args)
                if df.columns[0].startswith("\ufeff"):
                    raise UnicodeError(
                        "'charmap' codec can't encode characters in position 0-1325: character maps to <undefined>")
                return df
            except UnicodeError:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding="utf-8-sig",
                    sep=sep,
                    **args)
                return df
            except UnicodeDecodeError:
                df = pandas.read_csv(
                    filepath_or_buffer,
                    encoding="utf-8-sig",
                    sep=sep,
                    **args)
                return df
        else:
            return pandas.read_csv(
                filepath_or_buffer, encoding=encoding, sep=sep, **args)
    else:
        return pandas.read_csv(
            filepath_or_buffer, encoding=encoding, sep=sep, **args)

def df_to_clipboard(df, **args):
    """
    Copy a dataframe as csv text into the clipboard

    @param      df      dataframe
    @param      sep     by default the separator is ``\\t`` for this function until it is defined otherwise

    It relies on method
    `to_clipboard <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_clipboard.html>`_.

    @FAQ(pandas___Copier un dataframe dans le presse-papier - clipboard)

    Pour récupérer un dataframe dans Excel, on peut utiliser la méthode
    `to_excel <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_excel.html>`_
    puis ouvrir le fichier dans Excel ou le copier dans le presse-papier et le coller
    dans une feuille ouverte dans Excel. C'est l'objet de la méthode
    `to_clipboard <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_clipboard.html>`_ ::

        df = pandas.DataFrame ( ... )
        df.to_clipboard(sep="\\t")

    @endFAQ
    """
    if "sep" in args:
        df.to_clipboard(**args)
    else:
        df.to_clipboard(sep="\t", **args)

def df_equal(df1, df2):
    """
    compares two dataframe and tells if they are equal

    @param          df1     first dataframe
    @param          df2     second dataframe
    @return                 boolean

    The function compare column one by one.
    It does not check the order of the columns is the same.
    It reorders the columns before doing the comparison.

    If you need more complex comparison,
    you can look into function `assert_frame_equal <https://github.com/pydata/pandas/blob/master/pandas/util/testing.py>`_.

    The function does not handle well NaN values because ``numpy.nan != numpy.nan`` is true.
    It also compares types:

    @FAQ(pandas___Comment comparer deux dataframe?)

    Ecrire ``df1 == df2`` ne compare pas deux dataframes entre deux
    car le sens n'est pas forcément le même pour tout le monde.
    Même si les valeurs sont les mêmes, est-ce l'ordre des colonnes
    est important ?
    Il faut donc le faire soi-même. Le code ci-dessus
    compare d'abord les dimensions, ensuite compare l'ordre
    des colonnes puis enfin les valeurs ::

        if df1.shape != df2.shape:
            return False
        l1 = list(df1.columns)
        l2 = list(df2.columns)
        l1.sort()
        l2.sort()
        if l1 != l2:
            return False
        df1 = df1[l1]
        df2 = df2[l2]
        t = (df1 == df2).all()
        s = set(t)
        return False not in s

    @endFAQ
    """
    if df1.shape != df2.shape:
        return False
    l1 = list(df1.columns)
    l2 = list(df2.columns)
    l1.sort()
    l2.sort()
    if l1 != l2:
        return False
    df1 = df1[l1]
    df2 = df2[l2]
    s = set((df1.dtypes == df2.dtypes))
    if False in s:
        return False
    s = set((df1 == df2).all())
    return False not in s

def groupby_topn(df, by_keys, sort_keys, ascending=True, n=1, as_index=True):
    """
    takes the top n rows per group

    @param      df          dataframe
    @param      by_keys     rows will be grouped by these columns
    @param      sort_keys   rows will be sorted by these columns
    @param      ascending   parameter associated to sord function
    @param      n           n in top *n*
    @param      as_index    if False, remove the index after the group by
    @return                 result

    @FAQ(pandas___top n lignes avec pandas)

    Grouper puis garder les premières observations de ce groupe est un problème
    classique. Il n'existe pas de meilleure façon de le faire,
    cela dépend du nombre d'obervations par groupe. Le moyen le plus simple
    de le faire avec pandas est :

    * grouper les lignes
    * trier les lignes dans chaque groupe
    * garder les premières lignes dans chaque groupe

    Ceci donne ::

        df.groupby(by_keys)
          .apply(lambda x: x.sort(sort_keys, ascending=ascending).head(head))
          .reset_index(drop=True)

    La dernière instruction supprimer l'index ce qui donne au dataframe final
    la même structure que le dataframe initial.

    .. runpython::
        :showcode:

        import pandas
        l = [ dict(k1="a", k2="b", v=4, i=1),
              dict(k1="a", k2="b", v=5, i=1),
              dict(k1="a", k2="b", v=4, i=2),
              dict(k1="b", k2="b", v=1, i=2),
              dict(k1="b", k2="b", v=1, i=3)]
        df = pandas.DataFrame(l)
        df.groupby(["k1", "k2"]).apply(lambda x: x.sort(["v", "i"], ascending=True).head(1))
        print(df)

    @endFAQ
    """
    res = df.groupby(by_keys).apply(lambda x: x.sort(
        sort_keys, ascending=ascending).head(n))
    if not as_index:
        res = res.reset_index(drop=True)
    return res
<|endoftext|>


<commit_before>"""
MIDI file reading and playback.

References:

http://home.roadrunner.com/~jgglatt/
http://home.roadrunner.com/~jgglatt/tech/miditech.htm
http://home.roadrunner.com/~jgglatt/tech/midifile.htm

http://www.sonicspot.com/guide/midifiles.html
http://www.ccarh.org/courses/253/assignment/midifile/
https://code.google.com/p/binasc/wiki/mainpage
http://stackoverflow.com/questions/2984608/midi-delta-time
http://www.recordingblogs.com/sa/tabid/82/EntryId/44/MIDI-Part-XIII-Delta-time-a
http://www.sonicspot.com/guide/midifiles.html
"""

from __future__ import print_function, division
import io
import sys
import time
import string
import struct
from ..messages import build_message, Message, get_spec
from .meta import MetaMessage, build_meta_message, meta_charset
from .meta import MetaSpec, add_meta_spec, encode_variable_int
from .tracks import MidiTrack, merge_tracks, fix_end_of_track

PY2 = (sys.version_info.major == 2)

# The default tempo is 120 BPM.
# (500000 microseconds per beat (quarter note).)
DEFAULT_TEMPO = 500000
DEFAULT_TICKS_PER_BEAT = 480

def print_byte(byte, pos=0):
    char = chr(byte)
    if char.isspace() or not char in string.printable:
        char = '.'

    print('  {:06x}: {:02x}  {}'.format(pos, byte, char))

class DebugFileWrapper(object):
    def __init__(self, file):
        self.file = file

    def read(self, size):
        data = self.file.read(size)

        for byte in data:
            print_byte(byte, self.file.tell())

        return data

    def tell(self):
        return self.file.tell()

def read_byte(self):
    byte = self.read(1)
    if byte == b'':
        raise EOFError
    else:
        return ord(byte)

def read_bytes(infile, size):
    return [read_byte(infile) for _ in range(size)]

def _dbg(text=''):
    print(text)

# We can't use the chunk module for two reasons:
#
# 1. we may have mixed big and little endian chunk sizes. (RIFF is
# little endian while MTrk is big endian.)
#
# 2. the chunk module assumes that chunks are padded to the neares
# multiple of 2. This is not true of MIDI files.

def read_chunk_header(infile):
    header = infile.read(8)
    if len(header) < 8:
        raise EOFError

    # Todo: check for b'RIFF' and switch endian?

    return struct.unpack('>4sL', header)

def read_file_header(infile):
    name, size = read_chunk_header(infile)

    if name != b'MThd':
        raise IOError('MThd not found. Probably not a MIDI file')
    else:
        data = infile.read(size)

        if len(data) < 6:
            raise EOFError

        return struct.unpack('>hhh', data[:6])

def read_message(infile, status_byte, peek_data, delta):
    try:
        spec = get_spec(status_byte)
    except LookupError:
        raise IOError('undefined status byte 0x{:02x}'.format(status_byte))

    # Subtrac 1 for status byte.
    size = spec.length - 1 - len(peek_data)
    data_bytes = peek_data + read_bytes(infile, size)

    for byte in data_bytes:
        if byte > 127:
            raise IOError('data byte must be in range 0..127')
    return build_message(spec, [status_byte] + data_bytes, time=delta)

def read_sysex(infile, delta):
    length = read_variable_int(infile)
    data = read_bytes(infile, length)

    # Strip start and end bytes.
    # Todo: is this necessary?
    if data and data[0] == 0xf0:
        data = data[1:]
    if data and data[-1] == 0xf7:
        data = data[:-1]

    return Message('sysex', data=data, time=delta)

def read_variable_int(infile):
    delta = 0

    while True:
        byte = read_byte(infile)
        delta = (delta << 7) | (byte & 0x7f)
        if byte < 0x80:
            return delta

def read_meta_message(infile, delta):
    type = read_byte(infile)
    length = read_variable_int(infile)
    data = read_bytes(infile, length)
    return build_meta_message(type, data, delta)

def read_track(infile, debug=False):
    track = MidiTrack()

    name, size = read_chunk_header(infile)

    if name != b'MTrk':
        raise IOError('no MTrk header at start of track')

    if debug:
        _dbg('-> size={}'.format(size))
        _dbg()

    start = infile.tell()
    last_status = None

    while True:
        # End of track reached.
        if infile.tell() - start == size:
            break

        if debug:
            _dbg('Message:')

        delta = read_variable_int(infile)

        if debug:
            _dbg('-> delta={}'.format(delta))

        # Todo: not all messages have running status
        status_byte = read_byte(infile)

        if status_byte < 0x80:
            if last_status is None:
                raise IOError('running status without last_status')
            peek_data = [status_byte]
            status_byte = last_status
        else:
            if status_byte != 0xff:
                # Meta messages don't set running status.
                last_status = status_byte
            peek_data = []

        if status_byte == 0xff:
            msg = read_meta_message(infile, delta)
        elif status_byte in [0xf0, 0xf7]:
            # Todo: I'm not quite clear on the difference between
            # f0 and f7 events.
            msg = read_sysex(infile, delta)
        else:
            msg = read_message(infile, status_byte, peek_data, delta)

        track.append(msg)

        if debug:
            _dbg('-> {!r}'.format(msg))
            _dbg()

    return track

def write_chunk(outfile, name, data):
    """Write an IFF chunk to the file.

    `name` must be a bytestring."""
    outfile.write(name)
    outfile.write(struct.pack('>L', len(data)))
    outfile.write(data)

def write_track(outfile, track):
    data = bytearray()

    running_status_byte = None
    for msg in fix_end_of_track(track):
        if not isinstance(msg, MetaMessage):
            if msg._spec.status_byte >= 0xf8:
                raise ValueError(
                    ("realtime message '{}' is not "
                     "allowed in a MIDI file".format(
                            message.type)))

        data.extend(encode_variable_int(msg.time))
        if msg.type == 'sysex':
            running_status_byte = None
            data.append(0xf0)
            # length (+ 1 for end byte (0xf7))
            data.extend(encode_variable_int(len(msg.data) + 1))
            data.extend(msg.data)
            data.append(0xf7)
        else:
            raw = msg.bytes()
            if (isinstance(msg, Message)
                and raw[0] < 0xf0
                and raw[0] == running_status_byte):
                data.extend(raw[1:])
            else:
                data.extend(raw)
            running_status_byte = raw[0]

    write_chunk(outfile, b'MTrk', data)

class MidiFile:
    def __init__(self, filename=None, file=None,
                 type=1, ticks_per_beat=DEFAULT_TICKS_PER_BEAT,
                 charset='latin1',
                 debug=False):

        self.filename = filename
        self.type = type
        self.ticks_per_beat = ticks_per_beat
        self.charset = charset
        self.debug = debug

        self.tracks = []

        if type not in range(3):
            raise ValueError(
                'invalid format {} (must be 0, 1 or 2)'.format(format))

        if file is not None:
            self._load(file)
        elif self.filename is not None:
            with io.open(filename, 'rb') as file:
                self._load(file)

    def add_track(self, name=None):
        """Add a new track to the file.

        This will create a new MidiTrack object and append it to the
        track list.
        """
        track = MidiTrack()
        if name is not None:
            track.name = name
        self.tracks.append(track)
        return track

    def _load(self, infile):
        if self.debug:
            infile = DebugFileWrapper(infile)

        with meta_charset(self.charset):
            if self.debug:
                _dbg('Header:')

            (self.type,
             num_tracks,
             self.ticks_per_beat) = read_file_header(infile)

            if self.debug:
                _dbg('-> type={}, tracks={}, ticks_per_beat={}'.format(
                    self.type, num_tracks, self.ticks_per_beat))
                _dbg()

            for i in range(num_tracks):
                if self.debug:
                    _dbg('Track {}:'.format(i))

                self.tracks.append(read_track(infile, debug=self.debug))
                # Todo: used to ignore EOFError. I hope things still work.

    @property
    def length(self):
        """Playback time in seconds.

        This will be computed by going through every message in every
        track and adding up delta times.
        """
        if self.type == 2:
            raise ValueError('impossible to compute length'
                             ' for type 2 (asynchronous) file')

        return sum(msg.time for msg in self)

    def _get_seconds_per_tick(self, tempo):
        # Tempo is given in microseconds per beat (default 500000).
        # At this tempo there are (500000 / 1000000) == 0.5 seconds
        # per beat. At the default resolution of 480 ticks per beat
        # this is:
        #
        #    (500000 / 1000000) / 480 == 0.5 / 480 == 0.0010417
        #
        return (tempo / 1000000.0) / self.ticks_per_beat

    def __iter__(self):
        # The tracks of type 2 files are not in sync, so they can
        # not be played back like this.
        if self.type == 2:
            raise TypeError("can't merge tracks in type 2 (asynchronous) file")

        seconds_per_tick = self._get_seconds_per_tick(DEFAULT_TEMPO)

        for msg in merge_tracks(self.tracks):
            # Convert message time from absolute time
            # in ticks to relative time in seconds.
            if msg.time > 0:
                delta = msg.time * seconds_per_tick
            else:
                delta = 0

            yield msg.copy(time=delta)

            if msg.type == 'set_tempo':
                seconds_per_tick = self._get_seconds_per_tick(msg.tempo)

    def play(self, meta_messages=False):
        """Play back all tracks.

        The generator will sleep between each message by
        default. Messages are yielded with correct timing. The time
        attribute is set to the number of seconds slept since the
        previous message.

        By default you will only get normal MIDI messages. Pass
        meta_messages=True if you also want meta messages.

        You will receive copies of the original messages, so you can
        safely modify them without ruining the tracks.

        """
        sleep = time.sleep

        for msg in self:
            sleep(msg.time)

            if isinstance(msg, MetaMessage) and not meta_messages:
                continue
            else:
                yield msg

    def save(self, filename=None, file=None):
        """Save to a file.

        If file is passed the data will be saved to that file. This is typically
        an in-memory file or and already open file like sys.stdout.

        If filename is passed the data will be saved to that file.

        Raises ValueError if both file and filename are None,
        or if a type 0 file has != one track.
        """
        if self.type == 0 and len(self.tracks) != 1:
            raise ValueError('type 0 file must have exactly 1 track')

        if file is not None:
            self._save(file)
        elif filename is not None:
            with io.open(filename, 'wb') as file:
                self._save(file)
        else:
            raise ValueError('requires filename or file')

    def _save(self, outfile):
        with meta_charset(self.charset):
            header = struct.pack('>hhh', self.type,
                                 len(self.tracks),
                                 self.ticks_per_beat)

            write_chunk(outfile, b'MThd', header)

            for track in self.tracks:
                write_track(outfile, track)

    def print_tracks(self, meta_only=False):
        """Prints out all messages in a .midi file.

        May take argument meta_only to show only meta messages.

        Use:
        print_tracks() -> will print all messages
        print_tracks(meta_only=True) -> will print only MetaMessages
        """
        for i, track in enumerate(self.tracks):
            print('=== Track {}'.format(i))
            for msg in track:
                if not isinstance(msg, MetaMessage) and meta_only:
                    pass
                else:
                    print('{!r}'.format(msg))

    def __repr__(self):
        return '<midi file {!r} type {}, {} tracks, {} messages>'.format(
            self.filename, self.type, len(self.tracks),
            sum([len(track) for track in self.tracks]))

    # The context manager has no purpose but is kept around since it was
    # used in examples in the past.
    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False
<commit_msg>Use tick2second<commit_after>"""
MIDI file reading and playback.

References:

http://home.roadrunner.com/~jgglatt/
http://home.roadrunner.com/~jgglatt/tech/miditech.htm
http://home.roadrunner.com/~jgglatt/tech/midifile.htm

http://www.sonicspot.com/guide/midifiles.html
http://www.ccarh.org/courses/253/assignment/midifile/
https://code.google.com/p/binasc/wiki/mainpage
http://stackoverflow.com/questions/2984608/midi-delta-time
http://www.recordingblogs.com/sa/tabid/82/EntryId/44/MIDI-Part-XIII-Delta-time-a
http://www.sonicspot.com/guide/midifiles.html
"""

from __future__ import print_function, division
import io
import sys
import time
import string
import struct
from ..messages import build_message, Message, get_spec
from .meta import (MetaMessage, build_meta_message, meta_charset, MetaSpec,
                   add_meta_spec, encode_variable_int, tick2second)
from .tracks import MidiTrack, merge_tracks, fix_end_of_track

PY2 = (sys.version_info.major == 2)

# The default tempo is 120 BPM.
# (500000 microseconds per beat (quarter note).)
DEFAULT_TEMPO = 500000
DEFAULT_TICKS_PER_BEAT = 480

def print_byte(byte, pos=0):
    char = chr(byte)
    if char.isspace() or not char in string.printable:
        char = '.'

    print('  {:06x}: {:02x}  {}'.format(pos, byte, char))

class DebugFileWrapper(object):
    def __init__(self, file):
        self.file = file

    def read(self, size):
        data = self.file.read(size)

        for byte in data:
            print_byte(byte, self.file.tell())

        return data

    def tell(self):
        return self.file.tell()

def read_byte(self):
    byte = self.read(1)
    if byte == b'':
        raise EOFError
    else:
        return ord(byte)

def read_bytes(infile, size):
    return [read_byte(infile) for _ in range(size)]

def _dbg(text=''):
    print(text)

# We can't use the chunk module for two reasons:
#
# 1. we may have mixed big and little endian chunk sizes. (RIFF is
# little endian while MTrk is big endian.)
#
# 2. the chunk module assumes that chunks are padded to the neares
# multiple of 2. This is not true of MIDI files.

def read_chunk_header(infile):
    header = infile.read(8)
    if len(header) < 8:
        raise EOFError

    # Todo: check for b'RIFF' and switch endian?

    return struct.unpack('>4sL', header)

def read_file_header(infile):
    name, size = read_chunk_header(infile)

    if name != b'MThd':
        raise IOError('MThd not found. Probably not a MIDI file')
    else:
        data = infile.read(size)

        if len(data) < 6:
            raise EOFError

        return struct.unpack('>hhh', data[:6])

def read_message(infile, status_byte, peek_data, delta):
    try:
        spec = get_spec(status_byte)
    except LookupError:
        raise IOError('undefined status byte 0x{:02x}'.format(status_byte))

    # Subtrac 1 for status byte.
    size = spec.length - 1 - len(peek_data)
    data_bytes = peek_data + read_bytes(infile, size)

    for byte in data_bytes:
        if byte > 127:
            raise IOError('data byte must be in range 0..127')
    return build_message(spec, [status_byte] + data_bytes, time=delta)

def read_sysex(infile, delta):
    length = read_variable_int(infile)
    data = read_bytes(infile, length)

    # Strip start and end bytes.
    # Todo: is this necessary?
    if data and data[0] == 0xf0:
        data = data[1:]
    if data and data[-1] == 0xf7:
        data = data[:-1]

    return Message('sysex', data=data, time=delta)

def read_variable_int(infile):
    delta = 0

    while True:
        byte = read_byte(infile)
        delta = (delta << 7) | (byte & 0x7f)
        if byte < 0x80:
            return delta

def read_meta_message(infile, delta):
    type = read_byte(infile)
    length = read_variable_int(infile)
    data = read_bytes(infile, length)
    return build_meta_message(type, data, delta)

def read_track(infile, debug=False):
    track = MidiTrack()

    name, size = read_chunk_header(infile)

    if name != b'MTrk':
        raise IOError('no MTrk header at start of track')

    if debug:
        _dbg('-> size={}'.format(size))
        _dbg()

    start = infile.tell()
    last_status = None

    while True:
        # End of track reached.
        if infile.tell() - start == size:
            break

        if debug:
            _dbg('Message:')

        delta = read_variable_int(infile)

        if debug:
            _dbg('-> delta={}'.format(delta))

        # Todo: not all messages have running status
        status_byte = read_byte(infile)

        if status_byte < 0x80:
            if last_status is None:
                raise IOError('running status without last_status')
            peek_data = [status_byte]
            status_byte = last_status
        else:
            if status_byte != 0xff:
                # Meta messages don't set running status.
                last_status = status_byte
            peek_data = []

        if status_byte == 0xff:
            msg = read_meta_message(infile, delta)
        elif status_byte in [0xf0, 0xf7]:
            # Todo: I'm not quite clear on the difference between
            # f0 and f7 events.
            msg = read_sysex(infile, delta)
        else:
            msg = read_message(infile, status_byte, peek_data, delta)

        track.append(msg)

        if debug:
            _dbg('-> {!r}'.format(msg))
            _dbg()

    return track

def write_chunk(outfile, name, data):
    """Write an IFF chunk to the file.

    `name` must be a bytestring."""
    outfile.write(name)
    outfile.write(struct.pack('>L', len(data)))
    outfile.write(data)

def write_track(outfile, track):
    data = bytearray()

    running_status_byte = None
    for msg in fix_end_of_track(track):
        if not isinstance(msg, MetaMessage):
            if msg._spec.status_byte >= 0xf8:
                raise ValueError(
                    ("realtime message '{}' is not "
                     "allowed in a MIDI file".format(
                            message.type)))

        data.extend(encode_variable_int(msg.time))
        if msg.type == 'sysex':
            running_status_byte = None
            data.append(0xf0)
            # length (+ 1 for end byte (0xf7))
            data.extend(encode_variable_int(len(msg.data) + 1))
            data.extend(msg.data)
            data.append(0xf7)
        else:
            raw = msg.bytes()
            if (isinstance(msg, Message)
                and raw[0] < 0xf0
                and raw[0] == running_status_byte):
                data.extend(raw[1:])
            else:
                data.extend(raw)
            running_status_byte = raw[0]

    write_chunk(outfile, b'MTrk', data)

class MidiFile:
    def __init__(self, filename=None, file=None,
                 type=1, ticks_per_beat=DEFAULT_TICKS_PER_BEAT,
                 charset='latin1',
                 debug=False):

        self.filename = filename
        self.type = type
        self.ticks_per_beat = ticks_per_beat
        self.charset = charset
        self.debug = debug

        self.tracks = []

        if type not in range(3):
            raise ValueError(
                'invalid format {} (must be 0, 1 or 2)'.format(format))

        if file is not None:
            self._load(file)
        elif self.filename is not None:
            with io.open(filename, 'rb') as file:
                self._load(file)

    def add_track(self, name=None):
        """Add a new track to the file.

        This will create a new MidiTrack object and append it to the
        track list.
        """
        track = MidiTrack()
        if name is not None:
            track.name = name
        self.tracks.append(track)
        return track

    def _load(self, infile):
        if self.debug:
            infile = DebugFileWrapper(infile)

        with meta_charset(self.charset):
            if self.debug:
                _dbg('Header:')

            (self.type,
             num_tracks,
             self.ticks_per_beat) = read_file_header(infile)

            if self.debug:
                _dbg('-> type={}, tracks={}, ticks_per_beat={}'.format(
                    self.type, num_tracks, self.ticks_per_beat))
                _dbg()

            for i in range(num_tracks):
                if self.debug:
                    _dbg('Track {}:'.format(i))

                self.tracks.append(read_track(infile, debug=self.debug))
                # Todo: used to ignore EOFError. I hope things still work.

    @property
    def length(self):
        """Playback time in seconds.

        This will be computed by going through every message in every
        track and adding up delta times.
        """
        if self.type == 2:
            raise ValueError('impossible to compute length'
                             ' for type 2 (asynchronous) file')

        return sum(msg.time for msg in self)

    def __iter__(self):
        # The tracks of type 2 files are not in sync, so they can
        # not be played back like this.
        if self.type == 2:
            raise TypeError("can't merge tracks in type 2 (asynchronous) file")

        tempo = DEFAULT_TEMPO
        for msg in merge_tracks(self.tracks):
            # Convert message time from absolute time
            # in ticks to relative time in seconds.
            if msg.time > 0:
                delta = tick2second(msg.time, self.ticks_per_beat, tempo)
            else:
                delta = 0

            yield msg.copy(time=delta)

            if msg.type == 'set_tempo':
                tempo = msg.tempo

    def play(self, meta_messages=False):
        """Play back all tracks.

        The generator will sleep between each message by
        default. Messages are yielded with correct timing. The time
        attribute is set to the number of seconds slept since the
        previous message.

        By default you will only get normal MIDI messages. Pass
        meta_messages=True if you also want meta messages.

        You will receive copies of the original messages, so you can
        safely modify them without ruining the tracks.

        """
        sleep = time.sleep

        for msg in self:
            sleep(msg.time)

            if isinstance(msg, MetaMessage) and not meta_messages:
                continue
            else:
                yield msg

    def save(self, filename=None, file=None):
        """Save to a file.

        If file is passed the data will be saved to that file. This is typically
        an in-memory file or and already open file like sys.stdout.

        If filename is passed the data will be saved to that file.

        Raises ValueError if both file and filename are None,
        or if a type 0 file has != one track.
        """
        if self.type == 0 and len(self.tracks) != 1:
            raise ValueError('type 0 file must have exactly 1 track')

        if file is not None:
            self._save(file)
        elif filename is not None:
            with io.open(filename, 'wb') as file:
                self._save(file)
        else:
            raise ValueError('requires filename or file')

    def _save(self, outfile):
        with meta_charset(self.charset):
            header = struct.pack('>hhh', self.type,
                                 len(self.tracks),
                                 self.ticks_per_beat)

            write_chunk(outfile, b'MThd', header)

            for track in self.tracks:
                write_track(outfile, track)

    def print_tracks(self, meta_only=False):
        """Prints out all messages in a .midi file.

        May take argument meta_only to show only meta messages.

        Use:
        print_tracks() -> will print all messages
        print_tracks(meta_only=True) -> will print only MetaMessages
        """
        for i, track in enumerate(self.tracks):
            print('=== Track {}'.format(i))
            for msg in track:
                if not isinstance(msg, MetaMessage) and meta_only:
                    pass
                else:
                    print('{!r}'.format(msg))

    def __repr__(self):
        return '<midi file {!r} type {}, {} tracks, {} messages>'.format(
            self.filename, self.type, len(self.tracks),
            sum([len(track) for track in self.tracks]))

    # The context manager has no purpose but is kept around since it was
    # used in examples in the past.
    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        return False
<|endoftext|>