<commit_before>#!/usr/bin/env python

"""
A class object for generating and storing Toytree plotting coordinates.
"""

import numpy as np
from .utils import ToytreeError

"""
TODO:

- it may be useful to decompose update so plotting coords change but not idxs
-
"""

class Coords:
    """
    Generates and stores plotting coordinates for nodes and edges of a tree.
    Uses the toytree _style information (e.g., layout, use_edge_lengths).

    Attributes:
    -----------
    edges: ndarray
        2-d array of pairs showing (parent, child) relationships.
        Rows are ordered in postorder traversal order for plotting.
    verts: ndarray
        2-d array with relative cartesian coordinate positions of nodes (n)
        Rows are ordered by the numbered 'idx' feature of nodes.
        ...
    """
    def __init__(self, ttree):

        # the toytree
        self.ttree = ttree

        # the vertices and edge tuples for normal layouts ('n')
        self.edges = None
        self.verts = None

    def update(self, layout=None):
        """
        Updates cartesian coordinates for drawing tree graph
        """
        # new idx_dict (can't just overwrite existing b/c nnodes may changed)
        self.ttree.idx_dict = {}

        # store tree dimensions on update
        self.ttree.nnodes = 0
        self.ttree.ntips = 0
        for node in self.ttree.treenode.traverse():
            self.ttree.nnodes += 1
            if node.is_leaf():
                self.ttree.ntips += 1

        # updates idxs and fixed_idx for any tree manipulations
        self.update_idxs()
        self.circ = Circle(self.ttree)

        # get new edges shape and fill idx_dict
        self.edges = self.get_edges()

        # get edges and verts (node locations)
        layout = self.ttree.style.layout
        if layout == 'c':
            self.verts = self.get_radial_coords()
        else:
            self.verts = self.get_linear_coords(layout=layout)

    def update_idxs(self):
        """
        set root idx highest, then all internal nodes are numbered down
        in levelorder traversal, but tips are ordered numerically from
        bottom to top in right facing ladderized tree plot order.
        """

        # internal nodes: root is highest idx
        idx = self.ttree.nnodes - 1

        for node in self.ttree.treenode.traverse("levelorder"):
            if not node.is_leaf():
                node.add_feature("idx", idx)
                self.ttree.idx_dict[idx] = node
                if not node.name:
                    node.name = str(idx)
                idx -= 1

        # external nodes: lowest numbers are for tips (0-N)
        for node in self.ttree.treenode.get_leaves():
            node.add_feature("idx", idx)
            self.ttree.idx_dict[idx] = node
            if not node.name:
                node.name = str(idx)
            idx -= 1

    # def update_fixed_order(self):
    #     "after pruning fixed order needs update to match new nnodes/ntips."
    #     # set tips order if fixing for multi-tree plotting (default None)
    #     fixed_order = self.ttree._fixed_order
    #     self.ttree_fixed_order = None
    #     self.ttree_fixed_idx = list(range(self.ttree.ntips))

    #     # check if fixed_order changed:
    #     if fixed_order:
    #         fixed_order = [
    #             i for i in fixed_order if i in self.ttree.get_tip_labels()
    #         ]
    #         self.ttree._set_fixed_order(fixed_order)
    #     # else:
    #         # self.ttree._fixed_idx = list(range(self.ttree.ntips))

    def get_edges(self):
        """
        This could change if a tree dropped nodes.
        """
        # use cache to fill edges array
        edges = np.zeros((self.ttree.nnodes - 1, 2), dtype=int)
        for idx in range(self.ttree.nnodes - 1):
            parent = self.ttree.idx_dict[idx].up
            if parent:
                edges[idx, :] = (parent.idx, idx)
        return edges

    def get_radial_coords(self, use_edge_lengths=True):
        """
        Assign .edges and .verts for node positions in a fan tree.
        The farthest tip aligns at the circumference.
        """

        verts = np.zeros((self.ttree.nnodes, 2), dtype=float)

        # shortname
        if not use_edge_lengths:
            nbits = self.ttree.treenode.get_farthest_leaf(True)[1]

        # use cache to fill edges array
        for idx in range(self.ttree.nnodes):
            node = self.ttree.idx_dict[idx]

            # leaves: x positions are evenly spaced around circumference
            if node.is_leaf() and (not node.is_root()):

                # store radians (how far around from zero to 2pi)
                node.radians = self.circ.tip_radians[idx]

                # get positions of tips using radians and radius
                if use_edge_lengths:
                    node.radius = self.circ.radius - node.height
                    node.x, node.y = self.circ.get_node_coords(node)
                else:
                    node.radius = nbits
                    node.x, node.y = self.circ.get_node_coords(node)

            # internal nodes comes after tips. Inherit position from children.
            else:

                # height is either distance or nodes from root
                if use_edge_lengths:
                    node.radius = self.circ.radius - node.height
                else:
                    node.radius = sum(1 for i in node.iter_ancestors())
                    # max([i.radius for i in node.children]) - 1

                # x position is halfway between children x-pos
                node.radians = np.mean([i.radians for i in node.children])
                node.x, node.y = self.circ.get_node_coords(node)

            # store the x,y vertex positions
            verts[node.idx] = [node.x, node.y]
        return verts

        # scale so that node idx 0 (or fixed_order x) is at (0, 0)
        # adjust = self.ttree._coords.verts[:, 1].min()
        # self.ttree._coords.verts[:, 1] -= adjust

    def get_linear_coords(
        self,
        layout=None,
        use_edge_lengths=True,
        fixed_order=None,
        fixed_position=None,
        ):
        """
        Sets .edges, .verts for node positions.
        X and Y positions here refer to base assumption that tree is right
        facing, reorient_coordinates() will handle re-translating this.
        """
        verts = np.zeros((self.ttree.nnodes, 2), dtype=float)

        # store verts array with x,y positions of nodes (lengths of branches)
        # we want tips to align at the right face (larger axis number)
        fartip = self.ttree.treenode.get_farthest_leaf()[0]
        hgt = sum([
            fartip.dist,
            sum(i.dist for i in fartip.iter_ancestors() if not i.is_root())
        ])

        # finding tip distance from root is a little slow, it would be faster
        # just progress forward from root, but this wouldn't easily allow for
        # spacing fixed order tips. So instead we start from tips.
        for idx in range(self.ttree.nnodes):
            node = self.ttree.idx_dict[idx]

            # position the x-labels
            if node.is_leaf() and (not node.is_root()):
                if use_edge_lengths:
                    toroot = sum(
                        i.dist for i in node.iter_ancestors()
                        if not i.is_root()
                    )
                    node.y = hgt - (node.dist + toroot)
                else:
                    node.y = 0.

                # order of xlabels
                if fixed_order is not None:

                    # the position is exxplicit
                    if fixed_position is not None:
                        node.x = fixed_position[fixed_order.index(node.name)]

                    # simply use the index as position
                    else:
                        node.x = fixed_order.index(node.name)
                else:

                    # the position is exxplicit
                    if fixed_position is not None:
                        node.x = fixed_position[node.idx]

                    # simply use the index as position
                    else:
                        node.x = node.idx

            else:
                nch = node.children
                # empty trees only
                if not nch:
                    node.x = node.y = 0

                # all other trees
                else:
                    if use_edge_lengths:
                        node.y = nch[0].y + nch[0].dist
                    else:
                        node.y = max((i.y for i in nch)) + 1

                    # internal node at midpoint of children
                    node.x = sum(i.x for i in nch) / float(len(nch))

            # store the vertex
            verts[idx] = [node.x, node.y]

        # scale so that node idx 0 (or fixed_order x) is at (0, 0)
        if use_edge_lengths:
            adjust = verts[:, 1].min()
            verts[:, 1] -= adjust

        # default: Down-facing tips align at y=0, first ladderized tip at x=0
        if layout == 'd':
            return verts

        # right-facing tips align at x=0, last ladderized tip at y=0
        elif layout == 'r':

            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 0]
            tmp[:, 0] = verts[:, 1] * -1
            return tmp

        elif layout == 'l':

            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 0]
            tmp[:, 0] = verts[:, 1]
            return tmp

        elif layout == 'u':
            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 1] * -1
            tmp[:, 0] = verts[:, 0] * -1
            return tmp

        else:
            raise ToytreeError("layout not recognized")

    def get_radii(self, layout):
        """
        radius of every node relative to 0,0 origin.
        """
        # custom request layout
        if layout is None:
            layout = self.ttree.layout

        # get circular layout
        if layout == 'c':
            self.radii = np.array(
                [self.ttree.idx_dict[i].radius for i in range(self.nnodes)])
        else:
            self.radii = np.repeat(0, self.ntips)

class Circle:
    """
    When init from a toytree it can return coordinates based on
    distributing tips equally in radians from 0-2pi (TODO: allow
    setting start/end).

    The radius for farthest tip-ends is set to the tree height,
    and origin is at 0.0.
    """
    def __init__(self, tre):

        # set radius
        self.tre = tre
        self.radius = self.tre.treenode.height
        # get_distance(self.tre.treenode.get_farthest_leaf()[0])

        # origin
        self.o = (0, 0)
        # self.tre.style.xbaseline, self.tre.style.ybaseline)  # -self.radius, 0)

        # tips (bottom to top) are evenly spread from 0 to -2pi (counter clock)
        self.tip_radians = np.linspace(0, -np.pi * 2, self.tre.ntips + 1)[:-1]

    def get_node_coords(self, node):
        """
        get node coord. Node has radius assigned on the fly.
        """
        x = self.o[0] + node.radius * np.cos(node.radians)
        y = self.o[1] - node.radius * np.sin(node.radians)
        return x, y

    # def get_node_lines(self, node):
    #     x = self.o[0] + node.up.radius * np.cos(node.radians)
    #     y = self.o[1] - node.up.radius * np.sin(node.radians)
    #     return x, y

    def get_tip_end_angles(self):
        """
        node radians for printing tip labels should be from the root (0,0)
        """
        return np.array([np.rad2deg(abs(i)) for i in self.tip_radians])

    def get_tip_end_coords(self):
        """
        node tip coords must calculate new radian angle relative to parent
        node and then add offset amount to radius when calculating (x, y).
        """
        xs = [self.o[0] + self.radius * np.cos(i) for i in self.tip_radians]
        ys = [self.o[1] - self.radius * np.sin(i) for i in self.tip_radians]
        return np.stack([xs, ys], axis=1)

    # def get_parent_coords(self, node):
    #     """
    #     get x,y coordinates of parent based on radians of children
    #     """
    #     # calculate x, y coords
    #     x = self.o[0] + node.radius * np.cos(midradian)
    #     y = self.o[1] + node.radius * np.sin(midradian)
    #     return x, y

<commit_msg>linting style<commit_after>#!/usr/bin/env python

"""
A class object for generating and storing Toytree plotting coordinates.
"""

import numpy as np
from .utils import ToytreeError

"""
TODO:

- it may be useful to decompose update so plotting coords change but not idxs
-
"""

class Coords:
    """
    Generates and stores plotting coordinates for nodes and edges of a tree.
    Uses the toytree _style information (e.g., layout, use_edge_lengths).

    Attributes:
    -----------
    edges: ndarray
        2-d array of pairs showing (parent, child) relationships.
        Rows are ordered in postorder traversal order for plotting.
    verts: ndarray
        2-d array with relative cartesian coordinate positions of nodes (n)
        Rows are ordered by the numbered 'idx' feature of nodes.
        ...
    """
    def __init__(self, ttree):

        # the toytree
        self.ttree = ttree

        # the vertices and edge tuples for normal layouts ('n')
        self.edges = None
        self.verts = None

    def update(self, layout=None):
        """
        Updates cartesian coordinates for drawing tree graph
        """
        # new idx_dict (can't just overwrite existing b/c nnodes may changed)
        self.ttree.idx_dict = {}

        # store tree dimensions on update
        self.ttree.nnodes = 0
        self.ttree.ntips = 0
        for node in self.ttree.treenode.traverse():
            self.ttree.nnodes += 1
            if node.is_leaf():
                self.ttree.ntips += 1

        # updates idxs and fixed_idx for any tree manipulations
        self.update_idxs()
        self.circ = Circle(self.ttree)

        # get new edges shape and fill idx_dict
        self.edges = self.get_edges()

        # get edges and verts (node locations)
        layout = self.ttree.style.layout
        if layout == 'c':
            self.verts = self.get_radial_coords()
        else:
            self.verts = self.get_linear_coords(layout=layout)

    def update_idxs(self):
        """
        set root idx highest, then all internal nodes are numbered down
        in levelorder traversal, but tips are ordered numerically from
        bottom to top in right facing ladderized tree plot order.
        """

        # internal nodes: root is highest idx
        idx = self.ttree.nnodes - 1

        for node in self.ttree.treenode.traverse("levelorder"):
            if not node.is_leaf():
                node.add_feature("idx", idx)
                self.ttree.idx_dict[idx] = node
                if not node.name:
                    node.name = str(idx)
                idx -= 1

        # external nodes: lowest numbers are for tips (0-N)
        for node in self.ttree.treenode.get_leaves():
            node.add_feature("idx", idx)
            self.ttree.idx_dict[idx] = node
            if not node.name:
                node.name = str(idx)
            idx -= 1

    # def update_fixed_order(self):
    #     "after pruning fixed order needs update to match new nnodes/ntips."
    #     # set tips order if fixing for multi-tree plotting (default None)
    #     fixed_order = self.ttree._fixed_order
    #     self.ttree_fixed_order = None
    #     self.ttree_fixed_idx = list(range(self.ttree.ntips))

    #     # check if fixed_order changed:
    #     if fixed_order:
    #         fixed_order = [
    #             i for i in fixed_order if i in self.ttree.get_tip_labels()
    #         ]
    #         self.ttree._set_fixed_order(fixed_order)
    #     # else:
    #         # self.ttree._fixed_idx = list(range(self.ttree.ntips))

    def get_edges(self):
        """
        This could change if a tree dropped nodes.
        """
        # use cache to fill edges array
        edges = np.zeros((self.ttree.nnodes - 1, 2), dtype=int)
        for idx in range(self.ttree.nnodes - 1):
            parent = self.ttree.idx_dict[idx].up
            if parent:
                edges[idx, :] = (parent.idx, idx)
        return edges

    def get_radial_coords(self, use_edge_lengths=True):
        """
        Assign .edges and .verts for node positions in a fan tree.
        The farthest tip aligns at the circumference.
        """

        verts = np.zeros((self.ttree.nnodes, 2), dtype=float)

        # shortname
        if not use_edge_lengths:
            nbits = self.ttree.treenode.get_farthest_leaf(True)[1]

        # use cache to fill edges array
        for idx in range(self.ttree.nnodes):
            node = self.ttree.idx_dict[idx]

            # leaves: x positions are evenly spaced around circumference
            if node.is_leaf() and (not node.is_root()):

                # store radians (how far around from zero to 2pi)
                node.radians = self.circ.tip_radians[idx]

                # get positions of tips using radians and radius
                if use_edge_lengths:
                    node.radius = self.circ.radius - node.height
                    node.x, node.y = self.circ.get_node_coords(node)
                else:
                    node.radius = nbits
                    node.x, node.y = self.circ.get_node_coords(node)

            # internal nodes comes after tips. Inherit position from children.
            else:

                # height is either distance or nodes from root
                if use_edge_lengths:
                    node.radius = self.circ.radius - node.height
                else:
                    node.radius = sum(1 for i in node.iter_ancestors())
                    # max([i.radius for i in node.children]) - 1

                # x position is halfway between children x-pos
                node.radians = np.mean([i.radians for i in node.children])
                node.x, node.y = self.circ.get_node_coords(node)

            # store the x,y vertex positions
            verts[node.idx] = [node.x, node.y]
        return verts

        # scale so that node idx 0 (or fixed_order x) is at (0, 0)
        # adjust = self.ttree._coords.verts[:, 1].min()
        # self.ttree._coords.verts[:, 1] -= adjust

    def get_linear_coords(
        self,
        layout=None,
        use_edge_lengths=True,
        fixed_order=None,
        fixed_position=None,
        ):
        """
        Sets .edges, .verts for node positions.
        X and Y positions here refer to base assumption that tree is right
        facing, reorient_coordinates() will handle re-translating this.
        """
        verts = np.zeros((self.ttree.nnodes, 2), dtype=float)

        # store verts array with x,y positions of nodes (lengths of branches)
        # we want tips to align at the right face (larger axis number)
        fartip = self.ttree.treenode.get_farthest_leaf()[0]
        hgt = sum([
            fartip.dist,
            sum(i.dist for i in fartip.iter_ancestors() if not i.is_root())
        ])

        # finding tip distance from root is a little slow, it would be faster
        # just progress forward from root, but this wouldn't easily allow for
        # spacing fixed order tips. So instead we start from tips.
        for idx in range(self.ttree.nnodes):
            node = self.ttree.idx_dict[idx]

            # position the x-labels
            if node.is_leaf() and (not node.is_root()):
                if use_edge_lengths:
                    toroot = sum(
                        i.dist for i in node.iter_ancestors()
                        if not i.is_root()
                    )
                    node.y = hgt - (node.dist + toroot)
                else:
                    node.y = 0.

                # order of xlabels
                if fixed_order is not None:

                    # the position is explicit
                    if fixed_position is not None:
                        node.x = fixed_position[fixed_order.index(node.name)]

                    # simply use the index as position
                    else:
                        node.x = fixed_order.index(node.name)
                else:

                    # the position is exxplicit
                    if fixed_position is not None:
                        node.x = fixed_position[node.idx]

                    # simply use the index as position
                    else:
                        node.x = node.idx

            else:
                nch = node.children
                # empty trees only
                if not nch:
                    node.x = node.y = 0

                # all other trees
                else:
                    if use_edge_lengths:
                        node.y = nch[0].y + nch[0].dist
                    else:
                        node.y = max((i.y for i in nch)) + 1

                    # internal node at midpoint of children
                    node.x = sum(i.x for i in nch) / float(len(nch))

            # store the vertex
            verts[idx] = [node.x, node.y]

        # scale so that node idx 0 (or fixed_order x) is at (0, 0)
        if use_edge_lengths:
            adjust = verts[:, 1].min()
            verts[:, 1] -= adjust

        # check layout is in directionals
        if layout not in "drlu":
            raise ToytreeError("layout not recognized")

        # default: Down-facing tips align at y=0, first ladderized tip at x=0
        if layout == 'd':
            tmp = verts

        # right-facing tips align at x=0, last ladderized tip at y=0
        if layout == 'r':
            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 0]
            tmp[:, 0] = verts[:, 1] * -1

        elif layout == 'l':
            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 0]
            tmp[:, 0] = verts[:, 1]

        # elif layout == 'u':
        else:
            # verts swap x and ys and make xs 0 to negative
            tmp = np.zeros(verts.shape)
            tmp[:, 1] = verts[:, 1] * -1
            tmp[:, 0] = verts[:, 0] * -1

        return tmp

    def get_radii(self, layout):
        """
        radius of every node relative to 0,0 origin.
        """
        # custom request layout
        if layout is None:
            layout = self.ttree.layout

        # get circular layout
        if layout == 'c':
            self.radii = np.array(
                [self.ttree.idx_dict[i].radius for i in range(self.nnodes)])
        else:
            self.radii = np.repeat(0, self.ntips)

    def get_unrooted_coords(self, layout):
        """
        Project nodes onto an unrooted layout of either type A or B.
        """

        # get projection
        return self

class Circle:
    """
    When init from a toytree it can return coordinates based on
    distributing tips equally in radians from 0-2pi (TODO: allow
    setting start/end).

    The radius for farthest tip-ends is set to the tree height,
    and origin is at 0.0.
    """
    def __init__(self, tre):

        # set radius
        self.tre = tre
        self.radius = self.tre.treenode.height
        # get_distance(self.tre.treenode.get_farthest_leaf()[0])

        # origin
        self.o = (0, 0)
        # self.tre.style.xbaseline, self.tre.style.ybaseline)  # -self.radius, 0)

        # tips (bottom to top) are evenly spread from 0 to -2pi (counter clock)
        self.tip_radians = np.linspace(0, -np.pi * 2, self.tre.ntips + 1)[:-1]

    def get_node_coords(self, node):
        """
        get node coord. Node has radius assigned on the fly.
        """
        x = self.o[0] + node.radius * np.cos(node.radians)
        y = self.o[1] - node.radius * np.sin(node.radians)
        return x, y

    # def get_node_lines(self, node):
    #     x = self.o[0] + node.up.radius * np.cos(node.radians)
    #     y = self.o[1] - node.up.radius * np.sin(node.radians)
    #     return x, y

    def get_tip_end_angles(self):
        """
        node radians for printing tip labels should be from the root (0,0)
        """
        return np.array([np.rad2deg(abs(i)) for i in self.tip_radians])

    def get_tip_end_coords(self):
        """
        node tip coords must calculate new radian angle relative to parent
        node and then add offset amount to radius when calculating (x, y).
        """
        xs = [self.o[0] + self.radius * np.cos(i) for i in self.tip_radians]
        ys = [self.o[1] - self.radius * np.sin(i) for i in self.tip_radians]
        return np.stack([xs, ys], axis=1)

    # def get_parent_coords(self, node):
    #     """
    #     get x,y coordinates of parent based on radians of children
    #     """
    #     # calculate x, y coords
    #     x = self.o[0] + node.radius * np.cos(midradian)
    #     y = self.o[1] + node.radius * np.sin(midradian)
    #     return x, y

<|endoftext|>


<commit_before>"""loads Markdown files, converts each one to HTML and parses the HTML into an ElementTree structure.
The collection of ElementTrees are further parsed to generate a table of contents structure, and are
 manipulated to replace various markdown-generated HTML with specific Mako tags before being written
 to Mako templates, which then re-access the table of contents structure at runtime.

Much thanks to Alexey Shamrin, who came up with the original idea and did all the heavy Markdown/Elementtree
lifting for this module."""
import sys, re, os
from toc import TOCElement

try:
    import elementtree.ElementTree as et
except:
    raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)"

import markdown

def dump_tree(elem, stream):
    if elem.tag.startswith('MAKO:'):
        dump_mako_tag(elem, stream)
    else:
        if elem.tag != 'html':
            if len(elem.attrib):
                stream.write("<%s %s>" % (elem.tag, " ".join(["%s=%s" % (key, repr(val)) for key, val in elem.attrib.iteritems()])))
            else:
                stream.write("<%s>" % elem.tag)
        if elem.text:
            stream.write(elem.text)
        for child in elem:
            dump_tree(child, stream)
            if child.tail:
                stream.write(child.tail)
        stream.write("</%s>" % elem.tag)

def dump_mako_tag(elem, stream):
    tag = elem.tag[5:]
    params = ','.join(['%s=%s' % i for i in elem.items()])
    stream.write('<%%call expr="%s(%s)">' % (tag, params))
    if elem.text:
        stream.write(elem.text)
    for n in elem:
        dump_tree(n, stream)
        if n.tail:
            stream.write(n.tail)
    stream.write("</%call>")

def create_toc(filename, tree, tocroot):
    title = [None]
    current = [tocroot]
    level = [0]
    def process(tree):
        while True:
            i = find_header_index(tree)
            if i is None:
                return
            node = tree[i]
            taglevel = int(node.tag[1])
            start, end = i, end_of_header(tree, taglevel, i+1)
            content = tree[start+1:end]
            description = node.text.strip()
            if title[0] is None:
                title[0] = description
            name = node.get('name')
            if name is None:
                name = description.split()[0].lower()

            taglevel = node.tag[1]
            if taglevel > level[0]:
                current[0] = TOCElement(filename, name, description, current[0])
            elif taglevel == level[0]:
                current[0] = TOCElement(filename, name, description, current[0].parent)
            else:
                current[0] = TOCElement(filename, name, description, current[0].parent.parent)

            level[0] = taglevel

            tag = et.Element("MAKO:formatting.section", path=repr(current[0].path), paged='paged', extension='extension', toc='toc')
            tag.text = (node.tail or "") + '\n'
            tag.tail = '\n'
            tag[:] = content
            tree[start:end] = [tag]

            process(tag)

    process(tree)
    return (title[0], tocroot.get_by_file(filename))

def index(parent, item):
    for n, i in enumerate(parent):
        if i is item:
            return n

def find_header_index(tree):
    for i, node in enumerate(tree):
        if is_header(node):
            return i

def is_header(node):
    t = node.tag
    return (isinstance(t, str) and len(t) == 2 and t[0] == 'h'
            and t[1] in '123456789')

def end_of_header(tree, level, start):
    for i, node in enumerate(tree[start:]):
        if is_header(node) and int(node.tag[1]) <= level:
            return start + i
    return len(tree)

def process_rel_href(tree):
    parent = get_parent_map(tree)
    for a in tree.findall('.//a'):
        m = re.match(r'(bold)?rel\:(.+)', a.get('href'))
        if m:
            (bold, path) = m.group(1,2)
            text = a.text
            if text == path:
                tag = et.Element("MAKO:nav.toclink", path=repr(path), extension='extension', paged='paged', toc='toc')
            else:
                tag = et.Element("MAKO:nav.toclink", path=repr(path), description=repr(text), extension='extension', paged='paged', toc='toc')
            a_parent = parent[a]
            if bold:
                bold = et.Element('strong')
                bold.tail = a.tail
                bold.append(tag)
                a_parent[index(a_parent, a)] = bold
            else:
                tag.tail = a.tail
                a_parent[index(a_parent, a)] = tag

def replace_pre_with_mako(tree):
    parents = get_parent_map(tree)

    for precode in tree.findall('.//pre/code'):
        reg = re.compile(r'\{(python|mako|html|ini)(?: title="(.*?)"){0,1}\}(.*)', re.S)
        m = reg.match(precode[0].text.lstrip())
        if m:
            code = m.group(1)
            title = m.group(2)
            precode[0].text = m.group(3)
        else:
            code = title = None

        tag = et.Element("MAKO:formatting.code")
        if code:
            tag.attrib["syntaxtype"] = repr(code)
        if title:
            tag.attrib["title"] = repr(title)
        tag.text = precode.text
        [tag.append(x) for x in precode]
        pre = parents[precode]
        tag.tail = pre.tail

        pre_parent = parents[pre]
        pre_parent[reverse_parent(pre_parent, pre)] = tag

def safety_code(tree):
    parents = get_parent_map(tree)
    for code in tree.findall('.//code'):
        tag = et.Element('%text')
        if parents[code].tag != 'pre':
            tag.attrib["filter"] = "h"
        tag.text = code.text
        code.append(tag)
        code.text = ""

def reverse_parent(parent, item):
    for n, i in enumerate(parent):
        if i is item:
            return n

def get_parent_map(tree):
    return dict([(c, p) for p in tree.getiterator() for c in p])

def header(toc, title, filename):
    return """# -*- coding: utf-8 -*-
<%%inherit file="content_layout.html"/>
<%%page args="toc, extension, paged"/>
<%%namespace  name="formatting" file="formatting.html"/>
<%%namespace  name="nav" file="nav.html"/>
<%%def name="title()">%s - %s</%%def>
<%%!
    filename = '%s'
%%>
## This file is generated.  Edit the .txt files instead of this one.
""" % (toc.root.doctitle, title, filename)

class utf8stream(object):
    def __init__(self, stream):
        self.stream = stream
    def write(self, str):
        self.stream.write(str.encode('utf8'))

def parse_markdown_files(toc, files):
    for inname in files:
        infile = 'content/%s.txt' % inname
        if not os.access(infile, os.F_OK):
            continue
        html = markdown.markdown(file(infile).read())
        tree = et.fromstring("" + html + "")
        (title, toc_element) = create_toc(inname, tree, toc)
        safety_code(tree)
        replace_pre_with_mako(tree)
        process_rel_href(tree)
        outname = 'output/%s.html' % inname
        print infile, '->', outname
        outfile = utf8stream(file(outname, 'w'))
        outfile.write(header(toc, title, inname))
        dump_tree(tree, outfile)

<commit_msg>use 2.5 elementtree if available<commit_after>"""loads Markdown files, converts each one to HTML and parses the HTML into an ElementTree structure.
The collection of ElementTrees are further parsed to generate a table of contents structure, and are
 manipulated to replace various markdown-generated HTML with specific Mako tags before being written
 to Mako templates, which then re-access the table of contents structure at runtime.

Much thanks to Alexey Shamrin, who came up with the original idea and did all the heavy Markdown/Elementtree
lifting for this module."""
import sys, re, os
from toc import TOCElement

try:
    import xml.etree.ElementTree as et
except:
    try:
        import elementtree.ElementTree as et
    except:
        raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)"

import markdown

def dump_tree(elem, stream):
    if elem.tag.startswith('MAKO:'):
        dump_mako_tag(elem, stream)
    else:
        if elem.tag != 'html':
            if len(elem.attrib):
                stream.write("<%s %s>" % (elem.tag, " ".join(["%s=%s" % (key, repr(val)) for key, val in elem.attrib.iteritems()])))
            else:
                stream.write("<%s>" % elem.tag)
        if elem.text:
            stream.write(elem.text)
        for child in elem:
            dump_tree(child, stream)
            if child.tail:
                stream.write(child.tail)
        stream.write("</%s>" % elem.tag)

def dump_mako_tag(elem, stream):
    tag = elem.tag[5:]
    params = ','.join(['%s=%s' % i for i in elem.items()])
    stream.write('<%%call expr="%s(%s)">' % (tag, params))
    if elem.text:
        stream.write(elem.text)
    for n in elem:
        dump_tree(n, stream)
        if n.tail:
            stream.write(n.tail)
    stream.write("</%call>")

def create_toc(filename, tree, tocroot):
    title = [None]
    current = [tocroot]
    level = [0]
    def process(tree):
        while True:
            i = find_header_index(tree)
            if i is None:
                return
            node = tree[i]
            taglevel = int(node.tag[1])
            start, end = i, end_of_header(tree, taglevel, i+1)
            content = tree[start+1:end]
            description = node.text.strip()
            if title[0] is None:
                title[0] = description
            name = node.get('name')
            if name is None:
                name = description.split()[0].lower()

            taglevel = node.tag[1]
            if taglevel > level[0]:
                current[0] = TOCElement(filename, name, description, current[0])
            elif taglevel == level[0]:
                current[0] = TOCElement(filename, name, description, current[0].parent)
            else:
                current[0] = TOCElement(filename, name, description, current[0].parent.parent)

            level[0] = taglevel

            tag = et.Element("MAKO:formatting.section", path=repr(current[0].path), paged='paged', extension='extension', toc='toc')
            tag.text = (node.tail or "") + '\n'
            tag.tail = '\n'
            tag[:] = content
            tree[start:end] = [tag]

            process(tag)

    process(tree)
    return (title[0], tocroot.get_by_file(filename))

def index(parent, item):
    for n, i in enumerate(parent):
        if i is item:
            return n

def find_header_index(tree):
    for i, node in enumerate(tree):
        if is_header(node):
            return i

def is_header(node):
    t = node.tag
    return (isinstance(t, str) and len(t) == 2 and t[0] == 'h'
            and t[1] in '123456789')

def end_of_header(tree, level, start):
    for i, node in enumerate(tree[start:]):
        if is_header(node) and int(node.tag[1]) <= level:
            return start + i
    return len(tree)

def process_rel_href(tree):
    parent = get_parent_map(tree)
    for a in tree.findall('.//a'):
        m = re.match(r'(bold)?rel\:(.+)', a.get('href'))
        if m:
            (bold, path) = m.group(1,2)
            text = a.text
            if text == path:
                tag = et.Element("MAKO:nav.toclink", path=repr(path), extension='extension', paged='paged', toc='toc')
            else:
                tag = et.Element("MAKO:nav.toclink", path=repr(path), description=repr(text), extension='extension', paged='paged', toc='toc')
            a_parent = parent[a]
            if bold:
                bold = et.Element('strong')
                bold.tail = a.tail
                bold.append(tag)
                a_parent[index(a_parent, a)] = bold
            else:
                tag.tail = a.tail
                a_parent[index(a_parent, a)] = tag

def replace_pre_with_mako(tree):
    parents = get_parent_map(tree)

    for precode in tree.findall('.//pre/code'):
        reg = re.compile(r'\{(python|mako|html|ini)(?: title="(.*?)"){0,1}\}(.*)', re.S)
        m = reg.match(precode[0].text.lstrip())
        if m:
            code = m.group(1)
            title = m.group(2)
            precode[0].text = m.group(3)
        else:
            code = title = None

        tag = et.Element("MAKO:formatting.code")
        if code:
            tag.attrib["syntaxtype"] = repr(code)
        if title:
            tag.attrib["title"] = repr(title)
        tag.text = precode.text
        [tag.append(x) for x in precode]
        pre = parents[precode]
        tag.tail = pre.tail

        pre_parent = parents[pre]
        pre_parent[reverse_parent(pre_parent, pre)] = tag

def safety_code(tree):
    parents = get_parent_map(tree)
    for code in tree.findall('.//code'):
        tag = et.Element('%text')
        if parents[code].tag != 'pre':
            tag.attrib["filter"] = "h"
        tag.text = code.text
        code.append(tag)
        code.text = ""

def reverse_parent(parent, item):
    for n, i in enumerate(parent):
        if i is item:
            return n

def get_parent_map(tree):
    return dict([(c, p) for p in tree.getiterator() for c in p])

def header(toc, title, filename):
    return """# -*- coding: utf-8 -*-
<%%inherit file="content_layout.html"/>
<%%page args="toc, extension, paged"/>
<%%namespace  name="formatting" file="formatting.html"/>
<%%namespace  name="nav" file="nav.html"/>
<%%def name="title()">%s - %s</%%def>
<%%!
    filename = '%s'
%%>
## This file is generated.  Edit the .txt files instead of this one.
""" % (toc.root.doctitle, title, filename)

class utf8stream(object):
    def __init__(self, stream):
        self.stream = stream
    def write(self, str):
        self.stream.write(str.encode('utf8'))

def parse_markdown_files(toc, files):
    for inname in files:
        infile = 'content/%s.txt' % inname
        if not os.access(infile, os.F_OK):
            continue
        html = markdown.markdown(file(infile).read())
        tree = et.fromstring("" + html + "")
        (title, toc_element) = create_toc(inname, tree, toc)
        safety_code(tree)
        replace_pre_with_mako(tree)
        process_rel_href(tree)
        outname = 'output/%s.html' % inname
        print infile, '->', outname
        outfile = utf8stream(file(outname, 'w'))
        outfile.write(header(toc, title, inname))
        dump_tree(tree, outfile)

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""The main entry point. Invoke as `http' or `python -m httpie'.

"""
import sys
from .core import main

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Stricter KeyboardInterrupt silencing<commit_after>#!/usr/bin/env python
"""The main entry point. Invoke as `http' or `python -m httpie'.

"""
import sys

if __name__ == '__main__':
    try:
        from .core import main
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(1)
<|endoftext|>


<commit_before># coding: utf-8
# Copyright (c) 2008 Joost Cassee
# Licensed under the terms of the MIT License (see LICENSE.txt)

from __future__ import absolute_import
from django.db import models
from django.contrib.admin.widgets import AdminTextareaWidget
from tinymce.widgets import TinyMCE, AdminTinyMCE

class HTMLField(models.TextField):
    """
    A text area field for HTML content.

    It uses the TinyMCE 4 widget in forms.
    """
    def __init__(self, *args, **kwargs):
        self.tinymce_profile = kwargs.pop('profile', None)
        super(HTMLField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        defaults = {
            'widget': TinyMCE(profile=self.tinymce_profile)
        }
        defaults.update(kwargs)
        # As an ugly hack, we override the admin widget
        if defaults['widget'] == AdminTextareaWidget:
            defaults['widget'] = AdminTinyMCE(profile=self.tinymce_profile)
        return super(HTMLField, self).formfield(**defaults)
<commit_msg>Adds __all__ to models<commit_after># coding: utf-8
# Copyright (c) 2008 Joost Cassee
# Licensed under the terms of the MIT License (see LICENSE.txt)

from __future__ import absolute_import
from django.db import models
from django.contrib.admin.widgets import AdminTextareaWidget
from tinymce.widgets import TinyMCE, AdminTinyMCE

__all__ = ['HTMLField']

class HTMLField(models.TextField):
    """
    A text area field for HTML content.

    It uses the TinyMCE 4 widget in forms.
    """
    def __init__(self, *args, **kwargs):
        self.tinymce_profile = kwargs.pop('profile', None)
        super(HTMLField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        defaults = {
            'widget': TinyMCE(profile=self.tinymce_profile)
        }
        defaults.update(kwargs)
        # As an ugly hack, we override the admin widget
        if defaults['widget'] == AdminTextareaWidget:
            defaults['widget'] = AdminTinyMCE(profile=self.tinymce_profile)
        return super(HTMLField, self).formfield(**defaults)
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# md2epub.py
#
# by Ben Crowder
# http://bencrowder.net/
#
# Based off Matt Turner's GetBook.py
# http://staff.washington.edu/mdturner/personal.htm
#

import urllib, re, os, zipfile, glob, shutil, sys, markdown2, string, datetime

class Chapter:
	title = ''
	filename = ''
	htmlfile = ''
	id = ''
	children = []

	def __init__(self):
		self.children = []

class EPub:
	title = ''
	author = ''
	lang = 'en-US'
	basename = ''			# base name of ePub (before the extension)
	path = ''				# directory ePub will be stored in
	bookid = ''				# unique identifier
	url = ''				# URL for this book
	css = ''				# CSS file
	cover = ''				# book cover art
	images = []				# list of images to be included
	children = []			# list of Chapters

	navpointcount = 1		# used for navpoint counts
	chapterids = []

	def cleanup(self):
		os.rmdir(self.path)

	# takes a list of chapters and writes the <item> tags for them and their children
	def write_items(self, children, f, pre):
		for chapter in children:
			# For child chapters, prepend the parent id to make this one unique
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id

			# Make sure we don't put duplicates in
			if id not in self.chapterids:
				self.chapterids.append(id)
			else:
				print 'Duplicate ID: %s' % (id)
				sys.exit(-1)

			# Write it out
			f.write('''
		<item id="''' + id + '''" href="''' + chapter.htmlfile + '''" media-type="application/xhtml+xml" />''')
			if chapter.children:
				self.write_items(chapter.children, f, id)

	# takes a list of chapters and writes the <itemref> tags for them and their children
	def write_itemrefs(self, children, f, pre):
		for chapter in children:
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id
			f.write('''
		<itemref idref="''' + id + '''"/>''')
			if chapter.children:
				self.write_itemrefs(chapter.children, f, id)

	# takes a list of chapters and writes them and their children to a navmap
	def write_chapter_navpoints(self, children, f, pre):
		for chapter in children:
			# For child chapters, prepend the parent id to make this one unique
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id

			f.write('''
		<navPoint id="''' + id + '''" playOrder="''' + str(self.navpointcount) + '''">
			<navLabel><text>''' + chapter.title + '''</text></navLabel>
			<content src="''' + chapter.htmlfile + '''" />''')
			self.navpointcount += 1
			if chapter.children:
				self.write_chapter_navpoints(chapter.children, f, id)
			f.write('''
		</navPoint>''')

	# takes a list of chapters and converts them and their children to Markdown
	def convert_chapters_to_markdown(self, children):
		for chapter in children:
			try:
				input = open('../../' + chapter.filename, 'r')
				f = open(chapter.htmlfile, 'w')
			except:
				print 'Error reading file from table of contents.'
				sys.exit(-1)
			sourcetext = input.read()
			input.close()

			# write HTML header
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">

<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>''' + self.title + '''</title>''')
			if self.css:
				f.write('\n\t<link rel="stylesheet" type="text/css" href="' + os.path.basename(self.css) + '" />')
			f.write('''
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
</head>
<body>''')

			# write the Markdowned text
			f.write(markdown2.markdown(sourcetext).encode('utf-8'))

			# write HTML footer
			f.write('''
</body>
</html>''')

			f.close()

			if chapter.children:
				self.convert_chapters_to_markdown(chapter.children)

	# the main worker
	def save(self):
		# get current working directory
		cwd = os.getcwd()

		# create directory for the ePub
		os.mkdir(self.path)
		os.chdir(self.path)

		try:
			# make mimetype
			f = open('mimetype','w')
			f.write('application/epub+zip')
			f.close()

			# make the META-INF/container.xml
			os.mkdir('META-INF')
			os.chdir('META-INF')
			f = open('container.xml','w')
			f.write('''<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
	<rootfiles>
		<rootfile full-path="OEBPS/metadata.opf" media-type="application/oebps-package+xml" />
	</rootfiles>
</container>''')
			f.close()
			os.chdir('..')

			# make OEBPS/metadata.opf
			os.mkdir('OEBPS')
			os.chdir('OEBPS')
			f = open('metadata.opf','w')
			f.write('''<?xml version="1.0"?>
<package version="2.0" xmlns="http://www.idpf.org/2007/opf"
		 unique-identifier="bookid">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
		<dc:title>''' + self.title + '''</dc:title>
		<dc:creator opf:role="aut">''' + self.author + '''</dc:creator>
		<dc:language>''' + self.lang + '''</dc:language>
		<dc:identifier id="bookid">''' + self.url + '''</dc:identifier>''')
			if self.cover:
				f.write('\t\t<meta name="cover" content="book-cover" />')

			f.write('''
	</metadata>
	<manifest>
		<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
''')

			if self.css:
				f.write('\t\t<item id="style" href="' + os.path.basename(self.css) + '" media-type="application/css" />')

			if self.cover:
				imagefile = os.path.basename(self.cover)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('\t\t<item id="book-cover" href="' + imagefile + '" media-type="image/' + ext + '" />')

			# write the <item> tags
			self.write_items(self.children, f, '')

			for image in self.images:
				imagefile = os.path.basename(image)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('''
		<item id="''' + imagefile + '''" href="''' + imagefile + '''" media-type="image/''' + ext + '''" />''')

			f.write('''
	</manifest>
	<spine toc="ncx">''')

			# write the <itemref> tags
			self.write_itemrefs(self.children, f, '')

			f.write('''
	</spine>
</package>''')

			f.close()

			# make toc.ncx
			f = open('toc.ncx','w')
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
	<head>
		<meta name="dtb:uid" content="''' + self.url + '''" />
		<meta name="dtb:depth" content="1" />
		<meta name="dtb:totalPageCount" content="0" />
		<meta name="dtb:maxPageNumber" content="0" />
	</head>
	<docTitle>
		<text>''' + self.title + '''</text>
	</docTitle>
	<navMap>''')

			self.write_chapter_navpoints(self.children, f, '')

			f.write('''
	</navMap>
</ncx>''')
			f.close()

			# convert the texts to Markdown and save in the directory
			self.convert_chapters_to_markdown(self.children)

			# if there's a CSS file, copy it in
			if self.css:
				css = open('../../' + self.css, 'r')
				csstext = css.read()
				css.close()
				f = open(os.path.basename(self.css), 'w')
				f.write(csstext)
				f.close()

			# copy cover art into the directory
			if self.cover:
				dest = os.path.basename(self.cover)
				shutil.copyfile('../../' + self.cover, dest)

			# copy images into the directory
			for image in self.images:
				dest = os.path.basename(image)
				shutil.copyfile('../../' + image, dest)

			# now zip the ePub up
			os.chdir('../..')
			file = zipfile.ZipFile(self.basename + '.epub', "w")
			os.chdir(self.path)

			file.write('mimetype','mimetype', zipfile.ZIP_STORED)
			file.write('META-INF/container.xml', 'META-INF/container.xml', zipfile.ZIP_DEFLATED)
			for filename in glob.glob("OEBPS/*"):
				file.write(filename, filename, zipfile.ZIP_DEFLATED)

			# and remove the directory
			os.chdir('..')
			shutil.rmtree(self.path)
		except:
			# if something went wrong, remove the temp directory
			os.chdir(cwd)
			shutil.rmtree(self.path)

def add_chapter(chapter, children, depth):
	if depth > 0:
		add_chapter(chapter, children[-1].children, depth - 1)
	else:
		children.append(chapter)

def process_book(filename):
	epub = EPub()

	now = datetime.datetime.now()
	epub.basename = filename.split('.')[0]
	epub.path = '%s_%s-%s-%s_%s-%s-%s' % (epub.basename, now.year, now.month, now.day, now.hour, now.minute, now.second)
	epub.url = 'http://localhost/' + epub.path		# dummy URL, replaced in control file

	fh = open(filename, 'r')
	for line in fh.readlines():
		if ':' in line:					# keywords
			values = line.split(':', 1)
			keyword = values[0].strip()
			value = values[1].strip()
			if keyword == 'Title':
				epub.title = value
			elif keyword == 'Author':
				epub.author = value
			elif keyword == 'Language':
				epub.lang = value
			elif keyword == 'URL':
				epub.url = value
			elif keyword == 'Image' or keyword == 'Images':
				images = value.split(',')
				# split list
				for image in images:
					epub.images.append(image.strip())
			elif keyword == 'CSS':
				epub.css = value
			elif keyword == 'Cover':
				epub.cover = value
		elif '|' in line:				# contents
			values = line.split('|')
			title = values[0].strip()
			filename = values[1].strip()

			chapter = Chapter()
			chapter.title = values[0].strip()
			chapter.filename = values[1].strip()

			# replace extension with .html
			basename = os.path.basename(chapter.filename)
			chapter.htmlfile = os.path.splitext(basename)[0] + '.html'

			# for the ID, lowercase it all, strip punctuation, and replace spaces with underscores
			chapter.id = re.sub(r'[^a-zA-Z0-9]', r'', chapter.title.lower()).replace(' ', '_')

			# if there's no ID left (because the chapter title is all Unicode, for example),
			# use the basename of the file instead
			if not chapter.id:
				chapter.id = os.path.splitext(basename)[0]

			depth = line.rfind("\t") + 1
			add_chapter(chapter, epub.children, depth)
		else:
			if line[0] == '#' or not line.strip():
				# ignore comments and blank lines
				pass
			else:
				print "Error on the following line:\n"
				print line
				sys.exit(-1)

	fh.close()

	# create a (hopefully unique) book ID
	epub.bookid = '[' + epub.title + '|' + epub.author + ']'

	return epub

##### Main

def main():
	if (len(sys.argv) > 1):
		epub = process_book(sys.argv[1])
		epub.save()

if __name__ == "__main__":
	main()
<commit_msg>Now updates depth in toc.ncx<commit_after>#!/usr/bin/python
#
# md2epub.py
#
# by Ben Crowder
# http://bencrowder.net/
#
# Based off Matt Turner's GetBook.py
# http://staff.washington.edu/mdturner/personal.htm
#

import urllib, re, os, zipfile, glob, shutil, sys, markdown2, string, datetime

class Chapter:
	title = ''
	filename = ''
	htmlfile = ''
	id = ''
	children = []

	def __init__(self):
		self.children = []

class EPub:
	title = ''
	author = ''
	lang = 'en-US'
	basename = ''			# base name of ePub (before the extension)
	path = ''				# directory ePub will be stored in
	bookid = ''				# unique identifier
	url = ''				# URL for this book
	css = ''				# CSS file
	cover = ''				# book cover art
	images = []				# list of images to be included
	children = []			# list of Chapters

	navpointcount = 1		# used for navpoint counts
	chapterids = []
	maxdepth = 1

	def cleanup(self):
		os.rmdir(self.path)

	# takes a list of chapters and writes the <item> tags for them and their children
	def write_items(self, children, f, pre):
		for chapter in children:
			# For child chapters, prepend the parent id to make this one unique
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id

			# Make sure we don't put duplicates in
			if id not in self.chapterids:
				self.chapterids.append(id)
			else:
				print 'Duplicate ID: %s' % (id)
				sys.exit(-1)

			# Write it out
			f.write('''
		<item id="''' + id + '''" href="''' + chapter.htmlfile + '''" media-type="application/xhtml+xml" />''')
			if chapter.children:
				self.write_items(chapter.children, f, id)

	# takes a list of chapters and writes the <itemref> tags for them and their children
	def write_itemrefs(self, children, f, pre):
		for chapter in children:
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id
			f.write('''
		<itemref idref="''' + id + '''"/>''')
			if chapter.children:
				self.write_itemrefs(chapter.children, f, id)

	# takes a list of chapters and writes them and their children to a navmap
	def write_chapter_navpoints(self, children, f, pre):
		for chapter in children:
			# For child chapters, prepend the parent id to make this one unique
			if pre:
				id = pre + '_' + chapter.id
			else:
				id = chapter.id

			f.write('''
		<navPoint id="''' + id + '''" playOrder="''' + str(self.navpointcount) + '''">
			<navLabel><text>''' + chapter.title + '''</text></navLabel>
			<content src="''' + chapter.htmlfile + '''" />''')
			self.navpointcount += 1
			if chapter.children:
				self.write_chapter_navpoints(chapter.children, f, id)
			f.write('''
		</navPoint>''')

	# takes a list of chapters and converts them and their children to Markdown
	def convert_chapters_to_markdown(self, children):
		for chapter in children:
			try:
				input = open('../../' + chapter.filename, 'r')
				f = open(chapter.htmlfile, 'w')
			except:
				print 'Error reading file from table of contents.'
				sys.exit(-1)
			sourcetext = input.read()
			input.close()

			# write HTML header
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">

<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>''' + self.title + '''</title>''')
			if self.css:
				f.write('\n\t<link rel="stylesheet" type="text/css" href="' + os.path.basename(self.css) + '" />')
			f.write('''
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
</head>
<body>''')

			# write the Markdowned text
			f.write(markdown2.markdown(sourcetext).encode('utf-8'))

			# write HTML footer
			f.write('''
</body>
</html>''')

			f.close()

			if chapter.children:
				self.convert_chapters_to_markdown(chapter.children)

	# the main worker
	def save(self):
		# get current working directory
		cwd = os.getcwd()

		# create directory for the ePub
		os.mkdir(self.path)
		os.chdir(self.path)

		try:
			# make mimetype
			f = open('mimetype','w')
			f.write('application/epub+zip')
			f.close()

			# make the META-INF/container.xml
			os.mkdir('META-INF')
			os.chdir('META-INF')
			f = open('container.xml','w')
			f.write('''<?xml version="1.0"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
	<rootfiles>
		<rootfile full-path="OEBPS/metadata.opf" media-type="application/oebps-package+xml" />
	</rootfiles>
</container>''')
			f.close()
			os.chdir('..')

			# make OEBPS/metadata.opf
			os.mkdir('OEBPS')
			os.chdir('OEBPS')
			f = open('metadata.opf','w')
			f.write('''<?xml version="1.0"?>
<package version="2.0" xmlns="http://www.idpf.org/2007/opf"
		 unique-identifier="bookid">
	<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
		<dc:title>''' + self.title + '''</dc:title>
		<dc:creator opf:role="aut">''' + self.author + '''</dc:creator>
		<dc:language>''' + self.lang + '''</dc:language>
		<dc:identifier id="bookid">''' + self.url + '''</dc:identifier>''')
			if self.cover:
				f.write('\t\t<meta name="cover" content="book-cover" />')

			f.write('''
	</metadata>
	<manifest>
		<item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
''')

			if self.css:
				f.write('\t\t<item id="style" href="' + os.path.basename(self.css) + '" media-type="application/css" />')

			if self.cover:
				imagefile = os.path.basename(self.cover)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('\t\t<item id="book-cover" href="' + imagefile + '" media-type="image/' + ext + '" />')

			# write the <item> tags
			self.write_items(self.children, f, '')

			for image in self.images:
				imagefile = os.path.basename(image)
				ext = os.path.splitext(imagefile)[1][1:]	# get the extension
				if ext == 'jpg':
					ext = 'jpeg'
				f.write('''
		<item id="''' + imagefile + '''" href="''' + imagefile + '''" media-type="image/''' + ext + '''" />''')

			f.write('''
	</manifest>
	<spine toc="ncx">''')

			# write the <itemref> tags
			self.write_itemrefs(self.children, f, '')

			f.write('''
	</spine>
</package>''')

			f.close()

			# make toc.ncx
			f = open('toc.ncx','w')
			f.write('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1">
	<head>
		<meta name="dtb:uid" content="''' + self.url + '''" />
		<meta name="dtb:depth" content="''' + str(self.maxdepth) + '''" />
		<meta name="dtb:totalPageCount" content="0" />
		<meta name="dtb:maxPageNumber" content="0" />
	</head>
	<docTitle>
		<text>''' + self.title + '''</text>
	</docTitle>
	<navMap>''')

			self.write_chapter_navpoints(self.children, f, '')

			f.write('''
	</navMap>
</ncx>''')
			f.close()

			# convert the texts to Markdown and save in the directory
			self.convert_chapters_to_markdown(self.children)

			# if there's a CSS file, copy it in
			if self.css:
				css = open('../../' + self.css, 'r')
				csstext = css.read()
				css.close()
				f = open(os.path.basename(self.css), 'w')
				f.write(csstext)
				f.close()

			# copy cover art into the directory
			if self.cover:
				dest = os.path.basename(self.cover)
				shutil.copyfile('../../' + self.cover, dest)

			# copy images into the directory
			for image in self.images:
				dest = os.path.basename(image)
				shutil.copyfile('../../' + image, dest)

			# now zip the ePub up
			os.chdir('../..')
			file = zipfile.ZipFile(self.basename + '.epub', "w")
			os.chdir(self.path)

			file.write('mimetype','mimetype', zipfile.ZIP_STORED)
			file.write('META-INF/container.xml', 'META-INF/container.xml', zipfile.ZIP_DEFLATED)
			for filename in glob.glob("OEBPS/*"):
				file.write(filename, filename, zipfile.ZIP_DEFLATED)

			# and remove the directory
			os.chdir('..')
			shutil.rmtree(self.path)
		except:
			# if something went wrong, remove the temp directory
			os.chdir(cwd)
			shutil.rmtree(self.path)

def add_chapter(chapter, children, depth):
	# Go recursive
	if depth > 0:
		add_chapter(chapter, children[-1].children, depth - 1)
	else:
		children.append(chapter)

def process_book(filename):
	epub = EPub()

	now = datetime.datetime.now()
	epub.basename = filename.split('.')[0]
	epub.path = '%s_%s-%s-%s_%s-%s-%s' % (epub.basename, now.year, now.month, now.day, now.hour, now.minute, now.second)
	epub.url = 'http://localhost/' + epub.path		# dummy URL, replaced in control file
	epub.maxdepth = 1

	fh = open(filename, 'r')
	for line in fh.readlines():
		if ':' in line:					# keywords
			values = line.split(':', 1)
			keyword = values[0].strip()
			value = values[1].strip()
			if keyword == 'Title':
				epub.title = value
			elif keyword == 'Author':
				epub.author = value
			elif keyword == 'Language':
				epub.lang = value
			elif keyword == 'URL':
				epub.url = value
			elif keyword == 'Image' or keyword == 'Images':
				images = value.split(',')
				# split list
				for image in images:
					epub.images.append(image.strip())
			elif keyword == 'CSS':
				epub.css = value
			elif keyword == 'Cover':
				epub.cover = value
		elif '|' in line:				# contents
			values = line.split('|')
			title = values[0].strip()
			filename = values[1].strip()

			chapter = Chapter()
			chapter.title = values[0].strip()
			chapter.filename = values[1].strip()

			# replace extension with .html
			basename = os.path.basename(chapter.filename)
			chapter.htmlfile = os.path.splitext(basename)[0] + '.html'

			# for the ID, lowercase it all, strip punctuation, and replace spaces with underscores
			chapter.id = re.sub(r'[^a-zA-Z0-9]', r'', chapter.title.lower()).replace(' ', '_')

			# if there's no ID left (because the chapter title is all Unicode, for example),
			# use the basename of the file instead
			if not chapter.id:
				chapter.id = os.path.splitext(basename)[0]

			# see how deep in the tree we are
			depth = line.rfind("\t") + 1

			# keep track of maximum depth
			if depth > (epub.maxdepth - 1):
				epub.maxdepth = depth + 1

			# add the current chapter
			add_chapter(chapter, epub.children, depth)
		else:
			if line[0] == '#' or not line.strip():
				# ignore comments and blank lines
				pass
			else:
				print "Error on the following line:\n"
				print line
				sys.exit(-1)

	fh.close()

	# create a (hopefully unique) book ID
	epub.bookid = '[' + epub.title + '|' + epub.author + ']'

	return epub

##### Main

def main():
	if (len(sys.argv) > 1):
		epub = process_book(sys.argv[1])
		epub.save()

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># Django settings for COG project.
from django.conf.global_settings import TEMPLATE_CONTEXT_PROCESSORS
import os

rel = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x)

DEBUG = True
TEMPLATE_DEBUG = DEBUG

# IMPORTANT: this setting must be set to True if using COG behind a proxy server,
# otherwise redirects won't work properly
USE_X_FORWARDED_HOST = True

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
         #'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
         #'NAME': '',                      # Or path to database file if using sqlite3.
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': rel('./database/django.data'),
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Denver'
# use the system time zone, wherever the application is installed
#TIME_ZONE = None

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = rel('site_media/')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
#MEDIA_URL = 'http://localhost:8000/site_media/'
MEDIA_URL = '/site_media/'

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a trailing slash.
# Examples: "http://foo.com/media/", "/media/".
STATIC_URL = '/static/'
STATIC_ROOT = rel('static/')
#ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'yb@$-bub$i_mrxqe5it)v%p=^(f-h&x3%uy040x))19g^iha&#'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pagination.middleware.PaginationMiddleware',
    #'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
)

ROOT_URLCONF = 'COG.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    #os.path.join(os.path.basename(__file__), 'templates'),
    rel('templates/'),
    rel('static/'),
    #'/usr/COG/filebrowser/templates',
    #'/Users/cinquini/Documents/workspace-python/django-projects/COG/templates',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'grappelli',
    'filebrowser',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.webdesign',
    'django.contrib.staticfiles',
    'pagination',
    'south',
    'layouts',
    'cog',
    'django_cim_forms', 'django_cim_forms.cim_1_5', 'dycore', 'cascade',
)

# login page URL (default: '/accounts/login')
LOGIN_URL = '/login'
# page to redirect after successfull authentication, if 'next' parameter is not provided
#LOGIN_REDIRECT_URL='/cog/' # COG projects index
LOGIN_REDIRECT_URL='/' # welcome page
# Custom user profile
#AUTH_PROFILE_MODULE = "cog.UserProfile"

# makes 'request' object available in templates
TEMPLATE_CONTEXT_PROCESSORS += (
     'django.core.context_processors.request',
)

#CONTACT_EMAIL = 'Luca.Cinquini@jpl.nasa.gov'

# site-specific settings for grid directories
SRCGRID_DIR = '/export/shared/grids'
DSTGRID_DIR = '/export/shared/grids'
WEIGHTS_DIR = '/tmp'

# default search configuration
DEFAULT_SEARCH_URL = 'http://hydra.fsl.noaa.gov/esg-search/search/'
#DEFAULT_SEARCH_URL = 'http://esg-datanode.jpl.nasa.gov/esg-search/search/'
DEFAULT_SEARCH_FACETS = { 'project':'Project',
                          'realm':'Realm',
                          'variable':'Variable',
                          'cf_variable':'CF Variable'  }

COG_MAILING_LIST = "cog_info@list.woc.noaa.gov"

# CSS styles
#COLOR_DARK_TEAL = "#358C92"
#COLOR_LIGHT_TEAL = "#B9E0E3"

#COLOR_DARK_YELLOW = "#FAC2A4";
#COLOR_LIGHT_YELLOW = "#FCE79F";

#COLOR_DARK_GRAY = "#666666";

# ATOM feed information
ATOM_FEED_DIR = '/esg/content/atom_feed_home'<commit_msg>Inserting FILEBROWSER_DIRECTORY in settings.py<commit_after># Django settings for COG project.
from django.conf.global_settings import TEMPLATE_CONTEXT_PROCESSORS
import os

rel = lambda *x: os.path.join(os.path.abspath(os.path.dirname(__file__)), *x)

DEBUG = True
TEMPLATE_DEBUG = DEBUG

# IMPORTANT: this setting must be set to True if using COG behind a proxy server,
# otherwise redirects won't work properly
USE_X_FORWARDED_HOST = True

ADMINS = (
    # ('Your Name', 'your_email@domain.com'),
)

MANAGERS = ADMINS

DATABASES = {
    'default': {
         #'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
         #'NAME': '',                      # Or path to database file if using sqlite3.
        'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
        'NAME': rel('./database/django.data'),
        'USER': '',                      # Not used with sqlite3.
        'PASSWORD': '',                  # Not used with sqlite3.
        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.
        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.
    }
}

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Denver'
# use the system time zone, wherever the application is installed
#TIME_ZONE = None

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale
USE_L10N = True

# Absolute path to the directory that holds media.
# Example: "/home/media/media.lawrence.com/"
MEDIA_ROOT = rel('site_media/')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash if there is a path component (optional in other cases).
# Examples: "http://media.lawrence.com", "http://example.com/media/"
#MEDIA_URL = 'http://localhost:8000/site_media/'
MEDIA_URL = '/site_media/'

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a trailing slash.
# Examples: "http://foo.com/media/", "/media/".
STATIC_URL = '/static/'
STATIC_ROOT = rel('static/')
#ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.
SECRET_KEY = 'yb@$-bub$i_mrxqe5it)v%p=^(f-h&x3%uy040x))19g^iha&#'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'pagination.middleware.PaginationMiddleware',
    #'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware',
)

ROOT_URLCONF = 'COG.urls'

TEMPLATE_DIRS = (
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    #os.path.join(os.path.basename(__file__), 'templates'),
    rel('templates/'),
    rel('static/'),
    #'/usr/COG/filebrowser/templates',
    #'/Users/cinquini/Documents/workspace-python/django-projects/COG/templates',
)

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'grappelli',
    'filebrowser',
    'django.contrib.admin',
    'django.contrib.comments',
    'django.contrib.webdesign',
    'django.contrib.staticfiles',
    'pagination',
    'south',
    'layouts',
    'cog',
    'django_cim_forms', 'django_cim_forms.cim_1_5', 'dycore', 'cascade',
)

# login page URL (default: '/accounts/login')
LOGIN_URL = '/login'
# page to redirect after successfull authentication, if 'next' parameter is not provided
#LOGIN_REDIRECT_URL='/cog/' # COG projects index
LOGIN_REDIRECT_URL='/' # welcome page
# Custom user profile
#AUTH_PROFILE_MODULE = "cog.UserProfile"

# makes 'request' object available in templates
TEMPLATE_CONTEXT_PROCESSORS += (
     'django.core.context_processors.request',
)

#CONTACT_EMAIL = 'Luca.Cinquini@jpl.nasa.gov'

# site-specific settings for grid directories
SRCGRID_DIR = '/export/shared/grids'
DSTGRID_DIR = '/export/shared/grids'
WEIGHTS_DIR = '/tmp'

# default search configuration
DEFAULT_SEARCH_URL = 'http://hydra.fsl.noaa.gov/esg-search/search/'
#DEFAULT_SEARCH_URL = 'http://esg-datanode.jpl.nasa.gov/esg-search/search/'
DEFAULT_SEARCH_FACETS = { 'project':'Project',
                          'realm':'Realm',
                          'variable':'Variable',
                          'cf_variable':'CF Variable'  }

COG_MAILING_LIST = "cog_info@list.woc.noaa.gov"

# CSS styles
#COLOR_DARK_TEAL = "#358C92"
#COLOR_LIGHT_TEAL = "#B9E0E3"

#COLOR_DARK_YELLOW = "#FAC2A4";
#COLOR_LIGHT_YELLOW = "#FCE79F";

#COLOR_DARK_GRAY = "#666666";

# ATOM feed information
ATOM_FEED_DIR = '/esg/content/atom_feed_home'

# Filebrowser (IMPORTANT: must have traiing slash)
FILEBROWSER_DIRECTORY = "images/"<|endoftext|>