<commit_before>from pymel.util.common import path
import os
import json
import pymel.core as pm

# Patch for the broken getTransform in older pymel versions
pm.nt.Shape.getTransform = lambda x: x.getParent(generations=1)

title = 'Unreal Pipeline'
version = '1.06'

def UI():
    if pm.window('pbudk', exists=True):
        pm.deleteUI('pbudk')

    optspath = '%s/pbUDK.json' % pm.internalVar(usd=True)
    defaultdata = {'phyType': 1,
                   'maxVerts': 32,
                   'center': True,
                   'child': True,
                   'fbxPath': '%sdata/' % pm.workspace(q=True, rd=True),
                   'presetFile': '%s/UDKexport/UDK-FBX.fbxexportpreset' % pm.internalVar(usd=True),
                   'version': version,
                   'prefix': False,
                   'prefix_text': '',
                   'suffix': False,
                   'suffix_text': ''}

    with pm.window('pbudk', title="{0} - {1}".format(title, version), width=250, sizeable=True) as window:
        with pm.columnLayout():
            opts = JSONDict(optspath, defaultdata)
            PhyUI(opts)
            FbxUI(opts)
        window.show()

class PhyUI(object):
    def __init__(self, opts):
        self.opts = opts
        with pm.frameLayout('Physics', collapsable=True, cl=False, bs='out'):
            with pm.columnLayout(width=250):
                pm.text(l='Collision Type:')
                self.phyType = pm.radioButtonGrp(labelArray3=['Convex Hull', 'Box', 'Sphere'],
                                                 sl=self.opts['phyType'], nrb=3, cc=self.save,
                                                 cw3=[94, 64, 64], width=250)
                self.maxVerts = pm.intSliderGrp(field=True, l='Max Vertices:', v=self.opts['maxVerts'],
                                                cl3=['left', 'left', 'left'], cw3=[64, 48, 128], cc=self.save)
                pm.button(l='Add Hull', w=250, c=self._addHull)
        self.save()

    def _addHull(self, *args):
        if not pm.selected():
            return
        i = self.phyType.getSelect()
        if i == 1:
            self.convexHull()
        elif i == 2:
            self.boxHull()
        elif i == 3:
            self.sphereHull()

    def convexHull(self):
        sel = pm.selected()
        if not isinstance(sel[0], pm.nt.Transform):
            oldSel = sel
            sel = sel[0].node().getParent()
        else:
            oldSel = sel
            sel = sel[0]

        inputMesh = sel.getShape()
        hullNode = pm.createNode('DDConvexHull')
        outputNode = pm.createNode('mesh', n='UCX_{0}Shape_01'.format(sel))

        pm.connectAttr('%s.outMesh' % inputMesh, '%s.input[0].inputPolymesh' % hullNode)
        pm.connectAttr('%s.output' % hullNode, '%s.inMesh' % outputNode)

        hullNode.maxVertices.set(self.maxVerts.getValue())
        outputNode.getParent().setParent(sel)
        outputNode.getParent().translate.set(0, 0, 0)
        if not isinstance(oldSel[0], pm.nt.Transform):
            self.setComponents(oldSel, hullNode)

    def setComponents(self, sel, hullNode):
        coms = [str(i.name().split('.')[1]) for i in sel]
        hullNode.input[0].inputComponents.set(len(coms), *coms, type='componentList')

    @staticmethod
    def _get_bounds(sel):
        if sel > 1 and isinstance(sel[0], pm.Component):
            transform = sel[0].node().getTransform()
            t = pm.polyEvaluate(bc=True)
            bb = pm.dt.BoundingBox(pm.dt.Point(t[0][0], t[1][0], t[2][0]), pm.dt.Point(t[0][1], t[1][1], t[2][1]))
            verts = [i.getPosition() for i in pm.ls(pm.polyListComponentConversion(sel, tv=True), fl=True)]
            center = sum(verts) / len(verts)
        else:
            transform = sel[0]
            bb = sel[0].getBoundingBox()
            center = pm.objectCenter(sel[0])
        return bb, center, transform

    def boxHull(self):
        sel = pm.ls(sl=True, fl=True)
        bb, cnt, transform = self._get_bounds(sel)
        hull = pm.polyCube(w=bb.width(), h=bb.height(), d=bb.depth(), n='UCX_{0}_01'.format(transform))
        hull[0].setTranslation(cnt)
        sg = pm.PyNode('initialShadingGroup')
        sg.remove(hull[0].getShape())
        hull[0].setParent(transform)

    def sphereHull(self):
        sel = pm.ls(sl=True, fl=True)
        bb, cnt, transform = self._get_bounds(sel)
        hull = pm.polySphere(radius=max(bb.width(), bb.height(), bb.depth()) / 2, sx=8, sy=8,
                             n='UCX_{0}_01'.format(transform))
        hull[0].setTranslation(cnt)
        sg = pm.PyNode('initialShadingGroup')
        sg.remove(hull[0].getShape())
        hull[0].setParent(transform)

    def save(self, *args):
        self.opts['phyType'] = self.phyType.getSelect()
        self.opts['maxVerts'] = self.maxVerts.getValue()
        if self.phyType.getSelect() == 1:
            self.maxVerts.setEnable(True)
        else:
            self.maxVerts.setEnable(False)

class FbxUI(object):
    def __init__(self, opts):
        self.opts = opts
        with pm.frameLayout('Export Meshes (.FBX)', collapsable=True, cl=False, bs='out'):
            with pm.columnLayout(width=250):
                pm.text(l='Export List:')
                pm.separator(height=4)
                self.meshList = pm.textScrollList(height=250, width=250, ams=True, dkc=self._remove)
                with pm.rowColumnLayout(nc=3, cw=[(1, 82), (2, 82), (3, 82)]):
                    pm.button(l='Add', c=self._add)
                    pm.button(l='Remove', c=self._remove)
                    pm.button(l='Clear', c=self._clear)

                with pm.rowColumnLayout(nc=2, cw=[(1, 124), (2,124)]):
                    self.prefix = pm.checkBox(label='Prefix', value=self.opts['prefix'], cc=self.save)
                    self.suffix = pm.checkBox(label='Suffix', value=self.opts['suffix'], cc=self.save)
                    self.prefix_text = pm.textField(en=self.prefix.getValue(), text=self.opts['prefix_text'], cc=self.save)
                    self.suffix_text = pm.textField(en=self.suffix.getValue(), text=self.opts['suffix_text'], cc=self.save)

                pm.text(l='Export Path:')
                with pm.rowColumnLayout(nc=2, cw=[(1, 215), (2, 32)]):
                    self.fbxPath = pm.textField(text=self.opts['fbxPath'], cc=self._pathRefreash)
                    pm.button(l='...', c=self._path)

                with pm.rowColumnLayout(nc=3):
                    self.center = pm.checkBox(label='Move to Orgin', v=self.opts['center'], cc=self.save)
                    self.child = pm.checkBox(label='Export Childern', v=self.opts['child'], cc=self.save)
                    pm.button(l='FBXPreset', c=self._fbxPreset)

                with pm.rowColumnLayout(nc=2, cw=[(1, 124), (2, 124)]):
                    pm.button(l='Selected', c=self._selected)
                    pm.button(l='All', c=self._all)

        self._refresh()

    def _path(self, *args):
        exportPath = pm.fileDialog2(dir=path, fm=3, okc='Select Folder', cap='Select Export Folder')
        if exportPath:
            self.fbxPath.setText(exportPath[0])
            self.opts['fbxPath'] = exportPath[0]

    def _pathRefreash(self, text):
        self.opts['fbxPath'] = text

    def _selected(self, *args):
        self.export(self.fbxPath.getText(), all=False, center=self.center.getValue(), child=self.child.getValue())

    def _all(self, *args):
        self.export(self.fbxPath.getText(), all=True, center=self.center.getValue(), child=self.child.getValue())

    def _fbxPreset(self, *args):
        tempData = pm.fileDialog2(dir=path(self.opts['presetFile']).parent, fm=1, okc='Select Preset File',
                                  cap='Sselect FPXExportPreset File', ff='FBX Export Presets (*.fbxexportpreset)')
        if tempData:
            self.opts['presetFile'] = tempData[0]

    def _add(self, *args):
        sel = pm.selected()
        for i in sel:
            if isinstance(i, pm.nt.Transform):
                try:
                    i.pbExport.set(True)
                except:
                    i.addAttr('pbExport', at='bool')
                    i.pbExport.set(True)
        self._refresh()

    def _remove(self, *args):
        sel = self.meshList.getSelectItem()
        for i in sel:
            i = pm.PyNode(i)
            i.pbExport.delete()
        self._refresh()

    def _clear(self, *args):
        sel = pm.ls()
        for i in sel:
            if hasattr(i, 'pbExport'):
                i.pbExport.delete()
        self._refresh()

    def _refresh(self):
        self.meshList.removeAll()
        sel = pm.ls(type=pm.nt.Transform)
        for i in sel:
            if hasattr(i.getParent(), 'pbExport') and i.getParent().pbExport.get() is True:
                self.meshList.append(i.getParent())

    def export(self, dirpath, all=False, center=True, child=True):
        if os.path.isfile(self.opts['presetFile']):
            pm.mel.FBXLoadExportPresetFile(f=self.opts['presetFile'])
        ext = '.fbx'

        if all:
            pm.select(self.meshList.getAllItems())
            sel = pm.selected()
        else:
            sel = pm.selected()

        if len(sel) == 0:
            pm.warning('Nothing is selected!')
        else:
            for obj in sel:
                pm.select(obj)
                if center:
                    oldLoc = obj.getRotatePivot()
                    self.centerPiv(obj)
                exportPath = self._get_filename(dirpath, obj.name(), ext)
                if child:
                    children = obj.getChildren()
                    for i in children:
                        if isinstance(i, pm.nt.Transform):
                            pm.select(i, add=True)
                pm.mel.FBXExport(f=exportPath, s=True)
                if center:
                    obj.setTranslation([oldLoc[0], oldLoc[1], oldLoc[2]])

    def _get_filename(self, dir, name, ext):
        file_name = "{0}{1}{2}{3}".format(
            self.prefix_text.getText() if self.prefix.getValue() else "",
            name,
            self.suffix_text.getText() if self.suffix.getValue() else "",
            ext
        )
        return dir + os.sep + file_name;

    def centerPiv(self, obj):
        pm.select(obj)
        pm.makeIdentity(apply=True, t=True, r=True, s=True, n=False)
        pos = obj.getRotatePivot()
        obj.translate.set(-1 * pos.x, -1 * pos.y, -1 * pos.z)
        pm.makeIdentity(apply=True, t=True, r=True, s=True, n=False)

    def save(self, *args):
        self.prefix_text.setEnable(self.prefix.getValue())
        self.suffix_text.setEnable(self.suffix.getValue())

        self.opts['center'] = self.center.getValue()
        self.opts['child'] = self.child.getValue()
        self.opts['fbxPath'] = self.fbxPath.getText()
        self.opts['prefix'] = self.prefix.getValue()
        self.opts['prefix_text'] = self.prefix_text.getText()
        self.opts['suffix'] = self.suffix.getValue()
        self.opts['suffix_text'] = self.suffix_text.getText()

class JSONDict(dict):
    def __init__(self, filename, defaults, *args, **kwargs):
        super(JSONDict, self).__init__(**kwargs)
        self.filename = filename
        self.defaults = defaults
        self._load()
        self.update(*args, **kwargs)

    def _load(self):
        if os.path.isfile(self.filename) and os.path.getsize(self.filename) > 0:
            with open(self.filename, 'r') as f:
                data = json.load(f)
                if 'version' in data and data['version'] == version:
                    self.update(data)
                else:
                    self._dumpdefaults()
        else:
            self._dumpdefaults()

    def _dump(self):
        with open(self.filename, 'w') as f:
            json.dump(self, f, sort_keys=True, indent=4)

    def _dumpdefaults(self):
        with open(self.filename, 'w') as f:
            json.dump(self.defaults, f, sort_keys=True, indent=4)
        self._load()

    def __getitem__(self, key):
        return dict.__getitem__(self, key)

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, value)
        self._dump()

    def __repr__(self):
        dictrepr = dict.__repr__(self)
        return '%s(%s)' % (type(self).__name__, dictrepr)

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).items():
            self[k] = v
        self._dump()

if __name__ == "__main__":
    UI()
<commit_msg>incremented version number<commit_after>from pymel.util.common import path
import os
import json
import pymel.core as pm

# Patch for the broken getTransform in older pymel versions
pm.nt.Shape.getTransform = lambda x: x.getParent(generations=1)

title = 'Unreal Pipeline'
version = '1.07'

def UI():
    if pm.window('pbudk', exists=True):
        pm.deleteUI('pbudk')

    optspath = '%s/pbUDK.json' % pm.internalVar(usd=True)
    defaultdata = {'phyType': 1,
                   'maxVerts': 32,
                   'center': True,
                   'child': True,
                   'fbxPath': '%sdata/' % pm.workspace(q=True, rd=True),
                   'presetFile': '%s/UDKexport/UDK-FBX.fbxexportpreset' % pm.internalVar(usd=True),
                   'version': version,
                   'prefix': False,
                   'prefix_text': '',
                   'suffix': False,
                   'suffix_text': ''}

    with pm.window('pbudk', title="{0} - {1}".format(title, version), width=250, sizeable=True) as window:
        with pm.columnLayout():
            opts = JSONDict(optspath, defaultdata)
            PhyUI(opts)
            FbxUI(opts)
        window.show()

class PhyUI(object):
    def __init__(self, opts):
        self.opts = opts
        with pm.frameLayout('Physics', collapsable=True, cl=False, bs='out'):
            with pm.columnLayout(width=250):
                pm.text(l='Collision Type:')
                self.phyType = pm.radioButtonGrp(labelArray3=['Convex Hull', 'Box', 'Sphere'],
                                                 sl=self.opts['phyType'], nrb=3, cc=self.save,
                                                 cw3=[94, 64, 64], width=250)
                self.maxVerts = pm.intSliderGrp(field=True, l='Max Vertices:', v=self.opts['maxVerts'],
                                                cl3=['left', 'left', 'left'], cw3=[64, 48, 128], cc=self.save)
                pm.button(l='Add Hull', w=250, c=self._addHull)
        self.save()

    def _addHull(self, *args):
        if not pm.selected():
            return
        i = self.phyType.getSelect()
        if i == 1:
            self.convexHull()
        elif i == 2:
            self.boxHull()
        elif i == 3:
            self.sphereHull()

    def convexHull(self):
        sel = pm.selected()
        if not isinstance(sel[0], pm.nt.Transform):
            oldSel = sel
            sel = sel[0].node().getParent()
        else:
            oldSel = sel
            sel = sel[0]

        inputMesh = sel.getShape()
        hullNode = pm.createNode('DDConvexHull')
        outputNode = pm.createNode('mesh', n='UCX_{0}Shape_01'.format(sel))

        pm.connectAttr('%s.outMesh' % inputMesh, '%s.input[0].inputPolymesh' % hullNode)
        pm.connectAttr('%s.output' % hullNode, '%s.inMesh' % outputNode)

        hullNode.maxVertices.set(self.maxVerts.getValue())
        outputNode.getParent().setParent(sel)
        outputNode.getParent().translate.set(0, 0, 0)
        if not isinstance(oldSel[0], pm.nt.Transform):
            self.setComponents(oldSel, hullNode)

    def setComponents(self, sel, hullNode):
        coms = [str(i.name().split('.')[1]) for i in sel]
        hullNode.input[0].inputComponents.set(len(coms), *coms, type='componentList')

    @staticmethod
    def _get_bounds(sel):
        if sel > 1 and isinstance(sel[0], pm.Component):
            transform = sel[0].node().getTransform()
            t = pm.polyEvaluate(bc=True)
            bb = pm.dt.BoundingBox(pm.dt.Point(t[0][0], t[1][0], t[2][0]), pm.dt.Point(t[0][1], t[1][1], t[2][1]))
            verts = [i.getPosition() for i in pm.ls(pm.polyListComponentConversion(sel, tv=True), fl=True)]
            center = sum(verts) / len(verts)
        else:
            transform = sel[0]
            bb = sel[0].getBoundingBox()
            center = pm.objectCenter(sel[0])
        return bb, center, transform

    def boxHull(self):
        sel = pm.ls(sl=True, fl=True)
        bb, cnt, transform = self._get_bounds(sel)
        hull = pm.polyCube(w=bb.width(), h=bb.height(), d=bb.depth(), n='UCX_{0}_01'.format(transform))
        hull[0].setTranslation(cnt)
        sg = pm.PyNode('initialShadingGroup')
        sg.remove(hull[0].getShape())
        hull[0].setParent(transform)

    def sphereHull(self):
        sel = pm.ls(sl=True, fl=True)
        bb, cnt, transform = self._get_bounds(sel)
        hull = pm.polySphere(radius=max(bb.width(), bb.height(), bb.depth()) / 2, sx=8, sy=8,
                             n='UCX_{0}_01'.format(transform))
        hull[0].setTranslation(cnt)
        sg = pm.PyNode('initialShadingGroup')
        sg.remove(hull[0].getShape())
        hull[0].setParent(transform)

    def save(self, *args):
        self.opts['phyType'] = self.phyType.getSelect()
        self.opts['maxVerts'] = self.maxVerts.getValue()
        if self.phyType.getSelect() == 1:
            self.maxVerts.setEnable(True)
        else:
            self.maxVerts.setEnable(False)

class FbxUI(object):
    def __init__(self, opts):
        self.opts = opts
        with pm.frameLayout('Export Meshes (.FBX)', collapsable=True, cl=False, bs='out'):
            with pm.columnLayout(width=250):
                pm.text(l='Export List:')
                pm.separator(height=4)
                self.meshList = pm.textScrollList(height=250, width=250, ams=True, dkc=self._remove)
                with pm.rowColumnLayout(nc=3, cw=[(1, 82), (2, 82), (3, 82)]):
                    pm.button(l='Add', c=self._add)
                    pm.button(l='Remove', c=self._remove)
                    pm.button(l='Clear', c=self._clear)

                with pm.rowColumnLayout(nc=2, cw=[(1, 124), (2,124)]):
                    self.prefix = pm.checkBox(label='Prefix', value=self.opts['prefix'], cc=self.save)
                    self.suffix = pm.checkBox(label='Suffix', value=self.opts['suffix'], cc=self.save)
                    self.prefix_text = pm.textField(en=self.prefix.getValue(), text=self.opts['prefix_text'], cc=self.save)
                    self.suffix_text = pm.textField(en=self.suffix.getValue(), text=self.opts['suffix_text'], cc=self.save)

                pm.text(l='Export Path:')
                with pm.rowColumnLayout(nc=2, cw=[(1, 215), (2, 32)]):
                    self.fbxPath = pm.textField(text=self.opts['fbxPath'], cc=self._pathRefreash)
                    pm.button(l='...', c=self._path)

                with pm.rowColumnLayout(nc=3):
                    self.center = pm.checkBox(label='Move to Orgin', v=self.opts['center'], cc=self.save)
                    self.child = pm.checkBox(label='Export Childern', v=self.opts['child'], cc=self.save)
                    pm.button(l='FBXPreset', c=self._fbxPreset)

                with pm.rowColumnLayout(nc=2, cw=[(1, 124), (2, 124)]):
                    pm.button(l='Selected', c=self._selected)
                    pm.button(l='All', c=self._all)

        self._refresh()

    def _path(self, *args):
        exportPath = pm.fileDialog2(dir=path, fm=3, okc='Select Folder', cap='Select Export Folder')
        if exportPath:
            self.fbxPath.setText(exportPath[0])
            self.opts['fbxPath'] = exportPath[0]

    def _pathRefreash(self, text):
        self.opts['fbxPath'] = text

    def _selected(self, *args):
        self.export(self.fbxPath.getText(), all=False, center=self.center.getValue(), child=self.child.getValue())

    def _all(self, *args):
        self.export(self.fbxPath.getText(), all=True, center=self.center.getValue(), child=self.child.getValue())

    def _fbxPreset(self, *args):
        tempData = pm.fileDialog2(dir=path(self.opts['presetFile']).parent, fm=1, okc='Select Preset File',
                                  cap='Sselect FPXExportPreset File', ff='FBX Export Presets (*.fbxexportpreset)')
        if tempData:
            self.opts['presetFile'] = tempData[0]

    def _add(self, *args):
        sel = pm.selected()
        for i in sel:
            if isinstance(i, pm.nt.Transform):
                try:
                    i.pbExport.set(True)
                except:
                    i.addAttr('pbExport', at='bool')
                    i.pbExport.set(True)
        self._refresh()

    def _remove(self, *args):
        sel = self.meshList.getSelectItem()
        for i in sel:
            i = pm.PyNode(i)
            i.pbExport.delete()
        self._refresh()

    def _clear(self, *args):
        sel = pm.ls()
        for i in sel:
            if hasattr(i, 'pbExport'):
                i.pbExport.delete()
        self._refresh()

    def _refresh(self):
        self.meshList.removeAll()
        sel = pm.ls(type=pm.nt.Transform)
        for i in sel:
            if hasattr(i.getParent(), 'pbExport') and i.getParent().pbExport.get() is True:
                self.meshList.append(i.getParent())

    def export(self, dirpath, all=False, center=True, child=True):
        if os.path.isfile(self.opts['presetFile']):
            pm.mel.FBXLoadExportPresetFile(f=self.opts['presetFile'])
        ext = '.fbx'

        if all:
            pm.select(self.meshList.getAllItems())
            sel = pm.selected()
        else:
            sel = pm.selected()

        if len(sel) == 0:
            pm.warning('Nothing is selected!')
        else:
            for obj in sel:
                pm.select(obj)
                if center:
                    oldLoc = obj.getRotatePivot()
                    self.centerPiv(obj)
                exportPath = self._get_filename(dirpath, obj.name(), ext)
                if child:
                    children = obj.getChildren()
                    for i in children:
                        if isinstance(i, pm.nt.Transform):
                            pm.select(i, add=True)
                pm.mel.FBXExport(f=exportPath, s=True)
                if center:
                    obj.setTranslation([oldLoc[0], oldLoc[1], oldLoc[2]])

    def _get_filename(self, dir, name, ext):
        file_name = "{0}{1}{2}{3}".format(
            self.prefix_text.getText() if self.prefix.getValue() else "",
            name,
            self.suffix_text.getText() if self.suffix.getValue() else "",
            ext
        )
        return dir + os.sep + file_name;

    def centerPiv(self, obj):
        pm.select(obj)
        pm.makeIdentity(apply=True, t=True, r=True, s=True, n=False)
        pos = obj.getRotatePivot()
        obj.translate.set(-1 * pos.x, -1 * pos.y, -1 * pos.z)
        pm.makeIdentity(apply=True, t=True, r=True, s=True, n=False)

    def save(self, *args):
        self.prefix_text.setEnable(self.prefix.getValue())
        self.suffix_text.setEnable(self.suffix.getValue())

        self.opts['center'] = self.center.getValue()
        self.opts['child'] = self.child.getValue()
        self.opts['fbxPath'] = self.fbxPath.getText()
        self.opts['prefix'] = self.prefix.getValue()
        self.opts['prefix_text'] = self.prefix_text.getText()
        self.opts['suffix'] = self.suffix.getValue()
        self.opts['suffix_text'] = self.suffix_text.getText()

class JSONDict(dict):
    def __init__(self, filename, defaults, *args, **kwargs):
        super(JSONDict, self).__init__(**kwargs)
        self.filename = filename
        self.defaults = defaults
        self._load()
        self.update(*args, **kwargs)

    def _load(self):
        if os.path.isfile(self.filename) and os.path.getsize(self.filename) > 0:
            with open(self.filename, 'r') as f:
                data = json.load(f)
                if 'version' in data and data['version'] == version:
                    self.update(data)
                else:
                    self._dumpdefaults()
        else:
            self._dumpdefaults()

    def _dump(self):
        with open(self.filename, 'w') as f:
            json.dump(self, f, sort_keys=True, indent=4)

    def _dumpdefaults(self):
        with open(self.filename, 'w') as f:
            json.dump(self.defaults, f, sort_keys=True, indent=4)
        self._load()

    def __getitem__(self, key):
        return dict.__getitem__(self, key)

    def __setitem__(self, key, value):
        dict.__setitem__(self, key, value)
        self._dump()

    def __repr__(self):
        dictrepr = dict.__repr__(self)
        return '%s(%s)' % (type(self).__name__, dictrepr)

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).items():
            self[k] = v
        self._dump()

if __name__ == "__main__":
    UI()
<|endoftext|>


<commit_before><commit_msg>Remove processing of 'N' bases<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import csv
import itertools
import json
import os
import platform
import subprocess
from datetime import datetime
from functools import partial
from statistics import median, StatisticsError
from urllib.parse import urlsplit, urlunsplit, urljoin

DATE = datetime.now().strftime("%Y%m%d")
MACHINE = platform.machine()
SYSTEM = platform.system()

def load_manifest(filename):
    with open(filename, 'r') as f:
        text = f.read()
    return list(parse_manifest(text))

def parse_manifest(text):
    lines = filter(lambda x: x != "" and not x.startswith("#"),
                   map(lambda x: x.strip(), text.splitlines()))
    output = []
    for line in lines:
        if line.split(" ")[0] == "async":
            output.append((line.split(" ")[1], True))
        else:
            output.append((line.split(" ")[0], False))
    return output

def testcase_url(base, testcase):
    # The tp5 manifest hardwires http://localhost/ as the base URL,
    # which requires running the server as root in order to open
    # the server on port 80. To allow non-root users to run the test
    # case, we take the URL to be relative to a base URL.
    (scheme, netloc, path, query, fragment) = urlsplit(testcase)
    relative_url = urlunsplit(('', '', '.' + path, query, fragment))
    absolute_url = urljoin(base, relative_url)
    return absolute_url

def execute_test(url, command, timeout):
    try:
        return subprocess.check_output(
            command, stderr=subprocess.STDOUT, timeout=timeout
        )
    except subprocess.CalledProcessError as e:
        print("Unexpected Fail:")
        print(e)
        print("You may want to re-run the test manually:\n{}"
              .format(' '.join(command)))
    except subprocess.TimeoutExpired:
        print("Test FAILED due to timeout: {}".format(url))
    return ""

def run_servo_test(testcase, url, timeout, is_async):
    if is_async:
        print("Servo does not support async test!")
        # Return a placeholder
        return parse_log("", testcase, url)

    ua_script_path = "{}/user-agent-js".format(os.getcwd())
    command = [
        "../../../target/release/servo", url,
        "--userscripts", ua_script_path,
        "--headless",
        "-x", "-o", "output.png"
    ]
    log = ""
    try:
        log = subprocess.check_output(
            command, stderr=subprocess.STDOUT, timeout=timeout
        )
    except subprocess.CalledProcessError as e:
        print("Unexpected Fail:")
        print(e)
        print("You may want to re-run the test manually:\n{}".format(
            ' '.join(command)
        ))
    except subprocess.TimeoutExpired:
        print("Test FAILED due to timeout: {}".format(testcase))
    return parse_log(log, testcase, url)

def parse_log(log, testcase, url):
    blocks = []
    block = []
    copy = False
    for line_bytes in log.splitlines():
        line = line_bytes.decode()

        if line.strip() == ("[PERF] perf block start"):
            copy = True
        elif line.strip() == ("[PERF] perf block end"):
            copy = False
            blocks.append(block)
            block = []
        elif copy and line.strip().startswith("[PERF]"):
            block.append(line)

    def parse_block(block):
        timing = {}
        for line in block:
            try:
                (_, key, value) = line.split(",")
            except:
                print("[DEBUG] failed to parse the following line:")
                print(line)
                print('[DEBUG] log:')
                print('-----')
                print(log)
                print('-----')
                return None

            if key == "testcase" or key == "title":
                timing[key] = value
            else:
                timing[key] = None if (value == "undefined") else int(value)

        return timing

    def valid_timing(timing, url=None):
        if (timing is None or
                testcase is None or
                timing.get('title') == 'Error loading page' or
                timing.get('testcase') != url):
            return False
        else:
            return True

    # We need to still include the failed tests, otherwise Treeherder will
    # consider the result to be a new test series, and thus a new graph. So we
    # use a placeholder with values = -1 to make Treeherder happy, and still be
    # able to identify failed tests (successful tests have time >=0).
    def create_placeholder(testcase):
        return {
            "system": SYSTEM,
            "machine": MACHINE,
            "date": DATE,
            "testcase": testcase,
            "title": "",
            "navigationStart": 0,
            "unloadEventStart": -1,
            "domLoading": -1,
            "fetchStart": -1,
            "responseStart": -1,
            "loadEventEnd": -1,
            "connectStart": -1,
            "domainLookupStart": -1,
            "redirectStart": -1,
            "domContentLoadedEventEnd": -1,
            "requestStart": -1,
            "secureConnectionStart": -1,
            "connectEnd": -1,
            "loadEventStart": -1,
            "domInteractive": -1,
            "domContentLoadedEventStart": -1,
            "redirectEnd": -1,
            "domainLookupEnd": -1,
            "unloadEventEnd": -1,
            "responseEnd": -1,
            "domComplete": -1,
        }

    # Set the testcase field to contain the original testcase name,
    # rather than the url.
    def set_testcase(timing, testcase=None):
        timing['testcase'] = testcase
        timing['system'] = SYSTEM
        timing['machine'] = MACHINE
        timing['date'] = DATE
        return timing

    valid_timing_for_case = partial(valid_timing, url=url)
    set_testcase_for_case = partial(set_testcase, testcase=testcase)
    timings = list(map(set_testcase_for_case, filter(valid_timing_for_case, map(parse_block, blocks))))

    if len(timings) == 0:
        print("Didn't find any perf data in the log, test timeout?")
        print('[DEBUG] log:')
        print('-----')
        print(log)
        print('-----')

        return [create_placeholder(testcase)]
    else:
        return timings

def filter_result_by_manifest(result_json, manifest, base):
    filtered = []
    for name, is_async in manifest:
        url = testcase_url(base, name)
        match = [tc for tc in result_json if tc['testcase'] == url]
        if len(match) == 0:
            raise Exception(("Missing test result: {}. This will cause a "
                             "discontinuity in the treeherder graph, "
                             "so we won't submit this data.").format(name))
        filtered += match
    return filtered

def take_result_median(result_json, expected_runs):
    median_results = []
    for k, g in itertools.groupby(result_json, lambda x: x['testcase']):
        group = list(g)
        if len(group) != expected_runs:
            print(("Warning: Not enough test data for {},"
                  " maybe some runs failed?").format(k))

        median_result = {}
        for k, _ in group[0].items():
            if k == "testcase" or k == "title":
                median_result[k] = group[0][k]
            else:
                try:
                    median_result[k] = median([x[k] for x in group
                                               if x[k] is not None])
                except StatisticsError:
                    median_result[k] = -1
        median_results.append(median_result)
    return median_results

def save_result_json(results, filename, manifest, expected_runs, base):

    results = filter_result_by_manifest(results, manifest, base)
    results = take_result_median(results, expected_runs)

    if len(results) == 0:
        with open(filename, 'w') as f:
            json.dump("No test result found in the log. All tests timeout?",
                      f, indent=2)
    else:
        with open(filename, 'w') as f:
            json.dump(results, f, indent=2)
    print("Result saved to {}".format(filename))

def save_result_csv(results, filename, manifest, expected_runs, base):

    fieldnames = [
        'system',
        'machine',
        'date',
        'testcase',
        'title',
        'connectEnd',
        'connectStart',
        'domComplete',
        'domContentLoadedEventEnd',
        'domContentLoadedEventStart',
        'domInteractive',
        'domLoading',
        'domainLookupEnd',
        'domainLookupStart',
        'fetchStart',
        'loadEventEnd',
        'loadEventStart',
        'navigationStart',
        'redirectEnd',
        'redirectStart',
        'requestStart',
        'responseEnd',
        'responseStart',
        'secureConnectionStart',
        'unloadEventEnd',
        'unloadEventStart',
    ]

    with open(filename, 'w') as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames)
        writer.writeheader()
        writer.writerows(results)

def format_result_summary(results):
    failures = list(filter(lambda x: x['domComplete'] == -1, results))
    result_log = """
========================================
Total {total} tests; {suc} succeeded, {fail} failed.

Failure summary:
""".format(
        total=len(results),
        suc=len(list(filter(lambda x: x['domComplete'] != -1, results))),
        fail=len(failures)
    )
    uniq_failures = list(set(map(lambda x: x['testcase'], failures)))
    for failure in uniq_failures:
        result_log += " - {}\n".format(failure)

    result_log += "========================================\n"

    return result_log

def main():
    parser = argparse.ArgumentParser(
        description="Run page load test on servo"
    )
    parser.add_argument("tp5_manifest",
                        help="the test manifest in tp5 format")
    parser.add_argument("output_file",
                        help="filename for the output json")
    parser.add_argument("--base",
                        type=str,
                        default='http://localhost:8000/',
                        help="the base URL for tests. Default: http://localhost:8000/")
    parser.add_argument("--runs",
                        type=int,
                        default=20,
                        help="number of runs for each test case. Defult: 20")
    parser.add_argument("--timeout",
                        type=int,
                        default=300,  # 5 min
                        help=("kill the test if not finished in time (sec)."
                              " Default: 5 min"))
    parser.add_argument("--engine",
                        type=str,
                        default='servo',
                        help=("The engine to run the tests on. Currently only"
                              " servo and gecko are supported."))
    args = parser.parse_args()
    if args.engine == 'servo':
        run_test = run_servo_test
    elif args.engine == 'gecko':
        import gecko_driver  # Load this only when we need gecko test
        run_test = gecko_driver.run_gecko_test
    try:
        # Assume the server is up and running
        testcases = load_manifest(args.tp5_manifest)
        results = []
        for testcase, is_async in testcases:
            url = testcase_url(args.base, testcase)
            for run in range(args.runs):
                print("Running test {}/{} on {}".format(run + 1,
                                                        args.runs,
                                                        url))
                # results will be a mixure of timings dict and testcase strings
                # testcase string indicates a failed test
                results += run_test(testcase, url, args.timeout, is_async)
                print("Finished")
                # TODO: Record and analyze other performance.timing properties

        print(format_result_summary(results))
        if args.output_file.endswith('.csv'):
            save_result_csv(results, args.output_file, testcases, args.runs, args.base)
        else:
            save_result_json(results, args.output_file, testcases, args.runs, args.base)

    except KeyboardInterrupt:
        print("Test stopped by user, saving partial result")
        save_result_json(results, args.output_file, testcases, args.runs)

if __name__ == "__main__":
    main()
<commit_msg>Added explicit UTF-8 encoding to test-perf.<commit_after>#!/usr/bin/env python3

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import csv
import itertools
import json
import os
import platform
import subprocess
from datetime import datetime
from functools import partial
from statistics import median, StatisticsError
from urllib.parse import urlsplit, urlunsplit, urljoin

DATE = datetime.now().strftime("%Y%m%d")
MACHINE = platform.machine()
SYSTEM = platform.system()

def load_manifest(filename):
    with open(filename, 'r') as f:
        text = f.read()
    return list(parse_manifest(text))

def parse_manifest(text):
    lines = filter(lambda x: x != "" and not x.startswith("#"),
                   map(lambda x: x.strip(), text.splitlines()))
    output = []
    for line in lines:
        if line.split(" ")[0] == "async":
            output.append((line.split(" ")[1], True))
        else:
            output.append((line.split(" ")[0], False))
    return output

def testcase_url(base, testcase):
    # The tp5 manifest hardwires http://localhost/ as the base URL,
    # which requires running the server as root in order to open
    # the server on port 80. To allow non-root users to run the test
    # case, we take the URL to be relative to a base URL.
    (scheme, netloc, path, query, fragment) = urlsplit(testcase)
    relative_url = urlunsplit(('', '', '.' + path, query, fragment))
    absolute_url = urljoin(base, relative_url)
    return absolute_url

def execute_test(url, command, timeout):
    try:
        return subprocess.check_output(
            command, stderr=subprocess.STDOUT, timeout=timeout
        )
    except subprocess.CalledProcessError as e:
        print("Unexpected Fail:")
        print(e)
        print("You may want to re-run the test manually:\n{}"
              .format(' '.join(command)))
    except subprocess.TimeoutExpired:
        print("Test FAILED due to timeout: {}".format(url))
    return ""

def run_servo_test(testcase, url, timeout, is_async):
    if is_async:
        print("Servo does not support async test!")
        # Return a placeholder
        return parse_log("", testcase, url)

    ua_script_path = "{}/user-agent-js".format(os.getcwd())
    command = [
        "../../../target/release/servo", url,
        "--userscripts", ua_script_path,
        "--headless",
        "-x", "-o", "output.png"
    ]
    log = ""
    try:
        log = subprocess.check_output(
            command, stderr=subprocess.STDOUT, timeout=timeout
        )
    except subprocess.CalledProcessError as e:
        print("Unexpected Fail:")
        print(e)
        print("You may want to re-run the test manually:\n{}".format(
            ' '.join(command)
        ))
    except subprocess.TimeoutExpired:
        print("Test FAILED due to timeout: {}".format(testcase))
    return parse_log(log, testcase, url)

def parse_log(log, testcase, url):
    blocks = []
    block = []
    copy = False
    for line_bytes in log.splitlines():
        line = line_bytes.decode('utf-8')

        if line.strip() == ("[PERF] perf block start"):
            copy = True
        elif line.strip() == ("[PERF] perf block end"):
            copy = False
            blocks.append(block)
            block = []
        elif copy and line.strip().startswith("[PERF]"):
            block.append(line)

    def parse_block(block):
        timing = {}
        for line in block:
            try:
                (_, key, value) = line.split(",")
            except:
                print("[DEBUG] failed to parse the following line:")
                print(line)
                print('[DEBUG] log:')
                print('-----')
                print(log)
                print('-----')
                return None

            if key == "testcase" or key == "title":
                timing[key] = value
            else:
                timing[key] = None if (value == "undefined") else int(value)

        return timing

    def valid_timing(timing, url=None):
        if (timing is None or
                testcase is None or
                timing.get('title') == 'Error loading page' or
                timing.get('testcase') != url):
            return False
        else:
            return True

    # We need to still include the failed tests, otherwise Treeherder will
    # consider the result to be a new test series, and thus a new graph. So we
    # use a placeholder with values = -1 to make Treeherder happy, and still be
    # able to identify failed tests (successful tests have time >=0).
    def create_placeholder(testcase):
        return {
            "system": SYSTEM,
            "machine": MACHINE,
            "date": DATE,
            "testcase": testcase,
            "title": "",
            "navigationStart": 0,
            "unloadEventStart": -1,
            "domLoading": -1,
            "fetchStart": -1,
            "responseStart": -1,
            "loadEventEnd": -1,
            "connectStart": -1,
            "domainLookupStart": -1,
            "redirectStart": -1,
            "domContentLoadedEventEnd": -1,
            "requestStart": -1,
            "secureConnectionStart": -1,
            "connectEnd": -1,
            "loadEventStart": -1,
            "domInteractive": -1,
            "domContentLoadedEventStart": -1,
            "redirectEnd": -1,
            "domainLookupEnd": -1,
            "unloadEventEnd": -1,
            "responseEnd": -1,
            "domComplete": -1,
        }

    # Set the testcase field to contain the original testcase name,
    # rather than the url.
    def set_testcase(timing, testcase=None):
        timing['testcase'] = testcase
        timing['system'] = SYSTEM
        timing['machine'] = MACHINE
        timing['date'] = DATE
        return timing

    valid_timing_for_case = partial(valid_timing, url=url)
    set_testcase_for_case = partial(set_testcase, testcase=testcase)
    timings = list(map(set_testcase_for_case, filter(valid_timing_for_case, map(parse_block, blocks))))

    if len(timings) == 0:
        print("Didn't find any perf data in the log, test timeout?")
        print('[DEBUG] log:')
        print('-----')
        print(log)
        print('-----')

        return [create_placeholder(testcase)]
    else:
        return timings

def filter_result_by_manifest(result_json, manifest, base):
    filtered = []
    for name, is_async in manifest:
        url = testcase_url(base, name)
        match = [tc for tc in result_json if tc['testcase'] == url]
        if len(match) == 0:
            raise Exception(("Missing test result: {}. This will cause a "
                             "discontinuity in the treeherder graph, "
                             "so we won't submit this data.").format(name))
        filtered += match
    return filtered

def take_result_median(result_json, expected_runs):
    median_results = []
    for k, g in itertools.groupby(result_json, lambda x: x['testcase']):
        group = list(g)
        if len(group) != expected_runs:
            print(("Warning: Not enough test data for {},"
                  " maybe some runs failed?").format(k))

        median_result = {}
        for k, _ in group[0].items():
            if k == "testcase" or k == "title":
                median_result[k] = group[0][k]
            else:
                try:
                    median_result[k] = median([x[k] for x in group
                                               if x[k] is not None])
                except StatisticsError:
                    median_result[k] = -1
        median_results.append(median_result)
    return median_results

def save_result_json(results, filename, manifest, expected_runs, base):

    results = filter_result_by_manifest(results, manifest, base)
    results = take_result_median(results, expected_runs)

    if len(results) == 0:
        with open(filename, 'w') as f:
            json.dump("No test result found in the log. All tests timeout?",
                      f, indent=2)
    else:
        with open(filename, 'w') as f:
            json.dump(results, f, indent=2)
    print("Result saved to {}".format(filename))

def save_result_csv(results, filename, manifest, expected_runs, base):

    fieldnames = [
        'system',
        'machine',
        'date',
        'testcase',
        'title',
        'connectEnd',
        'connectStart',
        'domComplete',
        'domContentLoadedEventEnd',
        'domContentLoadedEventStart',
        'domInteractive',
        'domLoading',
        'domainLookupEnd',
        'domainLookupStart',
        'fetchStart',
        'loadEventEnd',
        'loadEventStart',
        'navigationStart',
        'redirectEnd',
        'redirectStart',
        'requestStart',
        'responseEnd',
        'responseStart',
        'secureConnectionStart',
        'unloadEventEnd',
        'unloadEventStart',
    ]

    with open(filename, 'w', encoding='utf-8') as csvfile:
        writer = csv.DictWriter(csvfile, fieldnames)
        writer.writeheader()
        writer.writerows(results)

def format_result_summary(results):
    failures = list(filter(lambda x: x['domComplete'] == -1, results))
    result_log = """
========================================
Total {total} tests; {suc} succeeded, {fail} failed.

Failure summary:
""".format(
        total=len(results),
        suc=len(list(filter(lambda x: x['domComplete'] != -1, results))),
        fail=len(failures)
    )
    uniq_failures = list(set(map(lambda x: x['testcase'], failures)))
    for failure in uniq_failures:
        result_log += " - {}\n".format(failure)

    result_log += "========================================\n"

    return result_log

def main():
    parser = argparse.ArgumentParser(
        description="Run page load test on servo"
    )
    parser.add_argument("tp5_manifest",
                        help="the test manifest in tp5 format")
    parser.add_argument("output_file",
                        help="filename for the output json")
    parser.add_argument("--base",
                        type=str,
                        default='http://localhost:8000/',
                        help="the base URL for tests. Default: http://localhost:8000/")
    parser.add_argument("--runs",
                        type=int,
                        default=20,
                        help="number of runs for each test case. Defult: 20")
    parser.add_argument("--timeout",
                        type=int,
                        default=300,  # 5 min
                        help=("kill the test if not finished in time (sec)."
                              " Default: 5 min"))
    parser.add_argument("--engine",
                        type=str,
                        default='servo',
                        help=("The engine to run the tests on. Currently only"
                              " servo and gecko are supported."))
    args = parser.parse_args()
    if args.engine == 'servo':
        run_test = run_servo_test
    elif args.engine == 'gecko':
        import gecko_driver  # Load this only when we need gecko test
        run_test = gecko_driver.run_gecko_test
    try:
        # Assume the server is up and running
        testcases = load_manifest(args.tp5_manifest)
        results = []
        for testcase, is_async in testcases:
            url = testcase_url(args.base, testcase)
            for run in range(args.runs):
                print("Running test {}/{} on {}".format(run + 1,
                                                        args.runs,
                                                        url))
                # results will be a mixure of timings dict and testcase strings
                # testcase string indicates a failed test
                results += run_test(testcase, url, args.timeout, is_async)
                print("Finished")
                # TODO: Record and analyze other performance.timing properties

        print(format_result_summary(results))
        if args.output_file.endswith('.csv'):
            save_result_csv(results, args.output_file, testcases, args.runs, args.base)
        else:
            save_result_json(results, args.output_file, testcases, args.runs, args.base)

    except KeyboardInterrupt:
        print("Test stopped by user, saving partial result")
        save_result_json(results, args.output_file, testcases, args.runs)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from collections import defaultdict
import itertools
import six
from enum import Enum

from ..utils import merge_list

def _update_instance_from_dict(instance, obj_dict):
    if not obj_dict:
        return
    for p_name in instance.__class__.external_properties:
        if p_name in obj_dict:
            object.__setattr__(instance, p_name, obj_dict.pop(p_name))

def _update_instance_from_obj(instance, obj):
    for p_name in instance.__class__.internal_properties:
        object.__setattr__(instance, p_name, getattr(obj, p_name))

class PropertyDictMeta(type):
    def __init__(cls, name, bases, dct):
        cls.core_properties = cp = [d_name for d_name, d_type in six.iteritems(dct) if isinstance(d_type, property)]
        cls.external_properties = ce = cp[:]
        cls.internal_properties = ci = ['_{0}'.format(d_name) for d_name in cp]
        cp.extend(ci)
        cp_bases = [base for base in bases if hasattr(base, 'core_properties')]
        merge_list(cp, itertools.chain.from_iterable(base.core_properties for base in cp_bases))
        merge_list(ce, itertools.chain.from_iterable(base.external_properties for base in cp_bases))
        merge_list(ci, itertools.chain.from_iterable(base.internal_properties for base in cp_bases))
        cls.core_property_set = set(cp)
        super(PropertyDictMeta, cls).__init__(name, bases, dct)

class AttributeMixin(six.with_metaclass(PropertyDictMeta)):
    """
    Utility class which allows access to a dictionary by attributes and keys. Also overrides the default iteration to
    return keys and values.
    """
    def __init__(self, *args, **kwargs):
        _update_instance_from_dict(self, kwargs)
        super(AttributeMixin, self).__init__(*args, **kwargs)

    def __repr__(self):
        default_repr = super(AttributeMixin, self).__repr__()
        cls = self.__class__
        props = ', '.join('{0}={1!r}'.format(p_name, getattr(self, p_name))
                          for p_name in cls.external_properties)
        return '<{0}({1}): {2}>'.format(cls.__name__, props, default_repr)

    def __getattr__(self, item):
        return self[item]

    def __setattr__(self, key, value):
        if key in self.__class__.core_property_set:
            object.__setattr__(self, key, value)
        else:
            self[key] = value

    def __delattr__(self, item):
        if hasattr(self, item):
            object.__delattr__(self, item)
        else:
            self.pop(item)

    def __iter__(self):
        return six.iteritems(self)

    def __nonzero__(self):
        return len(self) > 0 or any(getattr(self, p) for p in self.__class__.internal_properties)

    __bool__ = __nonzero__

    def __eq__(self, other):
        return super(AttributeMixin, self).__eq__(other) and all(getattr(self, p) == getattr(other, p)
                                                                 for p in self.__class__.internal_properties)

    def update(self, other=None, **kwargs):
        if other is not None:
            if isinstance(other, self.__class__):
                _update_instance_from_obj(self, other)
            elif isinstance(other, dict):
                other = other.copy()
                _update_instance_from_dict(self, other)
            else:
                raise TypeError("Expected {0} or dictionary; found '{1}'".format(type(self).__name__, type(other).__name__))
        _update_instance_from_dict(self, kwargs)
        super(AttributeMixin, self).update(other, **kwargs)

class DictMap(AttributeMixin, dict):
    def copy(self):
        new_instance = self.__class__(self)
        _update_instance_from_obj(new_instance, self)
        return new_instance

class DefaultDictMap(AttributeMixin, defaultdict):
    def copy(self):
        new_instance = self.__class__(self.default_factory, self)
        _update_instance_from_obj(new_instance, self)
        return new_instance

class FlagsMeta(type):
    def __new__(mcs, name, bases, dct):
        fields = {field_name: field_value
                  for field_name, field_value in six.iteritems(dct)
                  if isinstance(field_value, int)}
        dct['fields'] = fields
        new_cls = type.__new__(mcs, name, bases, dct)
        for field_name, field_value in six.iteritems(fields):
            setattr(new_cls, field_name, new_cls(field_value))
        return new_cls

class Flags(six.with_metaclass(FlagsMeta, int)):
    NONE = 0

    def __repr__(self):
        cls = self.__class__
        set_fields = [field_name
                      for field_name, field_value in six.iteritems(cls.fields)
                      if self & field_value]
        return '{0}({1})'.format(cls.__name__, ', '.join(set_fields))

    def __contains__(self, other):
        return self & other > 0

    def __or__(self, other):
        return self.__class__(int.__or__(self, other))

    def __xor__(self, other):
        return self.__class__(int.__xor__(self, other))

class SimpleEnum(Enum):
    # Just like regular enum, but less verbose.
    def __repr__(self):
        return '{0.__class__.__name__}.{0.name}'.format(self)
<commit_msg>Consider base clases with flags, fixes comparisons.<commit_after># -*- coding: utf-8 -*-
from collections import defaultdict
import itertools
import six
from enum import Enum

from ..utils import merge_list

def _update_instance_from_dict(instance, obj_dict):
    if not obj_dict:
        return
    for p_name in instance.__class__.external_properties:
        if p_name in obj_dict:
            object.__setattr__(instance, p_name, obj_dict.pop(p_name))

def _update_instance_from_obj(instance, obj):
    for p_name in instance.__class__.internal_properties:
        object.__setattr__(instance, p_name, getattr(obj, p_name))

class PropertyDictMeta(type):
    def __init__(cls, name, bases, dct):
        cls.core_properties = cp = [d_name for d_name, d_type in six.iteritems(dct) if isinstance(d_type, property)]
        cls.external_properties = ce = cp[:]
        cls.internal_properties = ci = ['_{0}'.format(d_name) for d_name in cp]
        cp.extend(ci)
        cp_bases = [base for base in bases if hasattr(base, 'core_properties')]
        merge_list(cp, itertools.chain.from_iterable(base.core_properties for base in cp_bases))
        merge_list(ce, itertools.chain.from_iterable(base.external_properties for base in cp_bases))
        merge_list(ci, itertools.chain.from_iterable(base.internal_properties for base in cp_bases))
        cls.core_property_set = set(cp)
        super(PropertyDictMeta, cls).__init__(name, bases, dct)

class AttributeMixin(six.with_metaclass(PropertyDictMeta)):
    """
    Utility class which allows access to a dictionary by attributes and keys. Also overrides the default iteration to
    return keys and values.
    """
    def __init__(self, *args, **kwargs):
        _update_instance_from_dict(self, kwargs)
        super(AttributeMixin, self).__init__(*args, **kwargs)

    def __repr__(self):
        default_repr = super(AttributeMixin, self).__repr__()
        cls = self.__class__
        props = ', '.join('{0}={1!r}'.format(p_name, getattr(self, p_name))
                          for p_name in cls.external_properties)
        return '<{0}({1}): {2}>'.format(cls.__name__, props, default_repr)

    def __getattr__(self, item):
        return self[item]

    def __setattr__(self, key, value):
        if key in self.__class__.core_property_set:
            object.__setattr__(self, key, value)
        else:
            self[key] = value

    def __delattr__(self, item):
        if hasattr(self, item):
            object.__delattr__(self, item)
        else:
            self.pop(item)

    def __iter__(self):
        return six.iteritems(self)

    def __nonzero__(self):
        return len(self) > 0 or any(getattr(self, p) for p in self.__class__.internal_properties)

    __bool__ = __nonzero__

    def __eq__(self, other):
        return super(AttributeMixin, self).__eq__(other) and all(getattr(self, p) == getattr(other, p)
                                                                 for p in self.__class__.internal_properties)

    def update(self, other=None, **kwargs):
        if other is not None:
            if isinstance(other, self.__class__):
                _update_instance_from_obj(self, other)
            elif isinstance(other, dict):
                other = other.copy()
                _update_instance_from_dict(self, other)
            else:
                raise TypeError("Expected {0} or dictionary; found '{1}'".format(type(self).__name__, type(other).__name__))
        _update_instance_from_dict(self, kwargs)
        super(AttributeMixin, self).update(other, **kwargs)

class DictMap(AttributeMixin, dict):
    def copy(self):
        new_instance = self.__class__(self)
        _update_instance_from_obj(new_instance, self)
        return new_instance

class DefaultDictMap(AttributeMixin, defaultdict):
    def copy(self):
        new_instance = self.__class__(self.default_factory, self)
        _update_instance_from_obj(new_instance, self)
        return new_instance

class FlagsMeta(type):
    def __new__(mcs, name, bases, dct):
        fields = {}
        for base in bases:
            if hasattr(base, 'fields'):
                fields.update(base.fields)
        fields.update((field_name, field_value)
                      for field_name, field_value in six.iteritems(dct)
                      if isinstance(field_value, int))
        dct['fields'] = fields
        new_cls = type.__new__(mcs, name, bases, dct)
        for field_name, field_value in six.iteritems(fields):
            setattr(new_cls, field_name, new_cls(field_value))

        def _get_fields(self):
            set_fields = [field_name
                          for field_name, field_value in six.iteritems(fields)
                          if self & field_value]
            return '{0}({1})'.format(name, ', '.join(set_fields))
        new_cls.__repr__ = _get_fields

        return new_cls

class Flags(six.with_metaclass(FlagsMeta, int)):
    NONE = 0

    def __contains__(self, other):
        return self & other > 0

    def __or__(self, other):
        return self.__class__(int.__or__(self, other))

    __add__ = __or__

    def __xor__(self, other):
        return self.__class__(int.__xor__(self, other))

    __sub__ = __xor__

class SimpleEnum(Enum):
    # Just like regular enum, but less verbose.
    def __repr__(self):
        return '{0.__class__.__name__}.{0.name}'.format(self)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyMoreItertools(PythonPackage):
    """Additions to the standard Python itertools package."""

    homepage = "https://github.com/erikrose/more-itertools"
    url      = "https://pypi.io/packages/source/m/more-itertools/more-itertools-7.2.0.tar.gz"

    import_modules = ['more_itertools', 'more_itertools.tests']

    version('7.2.0', sha256='409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832')
    version('5.0.0', sha256='38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4')
    version('4.3.0', sha256='c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e')
    version('4.1.0', sha256='c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44')
    version('2.2',   sha256='93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0')

    depends_on('python@3.5:', when='@7.1:', type=('build', 'run'))
    depends_on('python@3.4:', when='@6:', type=('build', 'run'))
    depends_on('python@2.7:2.8,3.2:', when='@2.3:', type=('build', 'run'))
    depends_on('python@2.6:2.8,3.2:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-six@1.0.0:1.999', when='@:5', type=('build', 'run'))
<commit_msg>py-moreitertools: added v7.0.0 (#16802)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyMoreItertools(PythonPackage):
    """Additions to the standard Python itertools package."""

    homepage = "https://github.com/erikrose/more-itertools"
    url      = "https://pypi.io/packages/source/m/more-itertools/more-itertools-7.2.0.tar.gz"

    import_modules = ['more_itertools', 'more_itertools.tests']

    version('7.2.0', sha256='409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832')
    version('7.0.0', sha256='c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a')
    version('5.0.0', sha256='38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4')
    version('4.3.0', sha256='c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e')
    version('4.1.0', sha256='c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44')
    version('2.2',   sha256='93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0')

    depends_on('python@3.5:', when='@7.1:', type=('build', 'run'))
    depends_on('python@3.4:', when='@6:', type=('build', 'run'))
    depends_on('python@2.7:2.8,3.2:', when='@2.3:', type=('build', 'run'))
    depends_on('python@2.6:2.8,3.2:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-six@1.0.0:1.999', when='@:5', type=('build', 'run'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from base import *
from json import loads
from urllib.request import urlopen
from decimal import *
import pygeoip
gi = pygeoip.GeoIP('pygeoip/GeoLiteCity.dat')

class weatherinfo:
    data = {}

    def getinfo(self, number):
        data = self.data
        name = data["name"]
        cname = data["sys"]["country"]
        if len(name) == 0:
            name = self.dname
        #checks what the request wants
        if number == "currenttemp":
            if self.outputtemp == "kel":
                return "It is " +\
                str(data["main"]["temp"]) + " Kelvin in " + name + ", " + cname
            elif self.outputtemp == "cel":
                temp = Decimal(str(data["main"]["temp"])) - Decimal('273.15')
                return "It is " + str(temp) + " Celcius in " + name + ", " + cname
            elif self.outputtemp == "far":
                temp = Decimal(str(data["main"]["temp"])) - Decimal('273.15')
                temp = Decimal(str(temp)) * Decimal('1.8') + Decimal('32.0')
                return "It is " + str(temp) + " Farenheit in " + name + + ", " + cname
            else:
                raise ValueError(
                    "Invalid Temperature Style: " + self.outputtemp)
        elif number == "currentwindspeed":
            return "The Wind is Blowing at " +\
            str(data["wind"]["speed"]) + " m/s in " + name + ", " + cname

    def __init__(self, url, style, altname):
        jsonobj = urlopen(url)
        jsonobj = jsonobj.read()
        self.data = loads(jsonobj.decode())
        self.outputtemp = style
        self.dname = altname

def msg(mp):
    if mp.cmd('weather'):
        tempstyle = "cel"
        if mp.argbool('cel'):
            tempstyle = "cel"
        elif mp.argbool('kel'):
            tempstyle = "kel"
        elif mp.argbool('far'):
            tempstyle = "far"
        returnwind = mp.argbool('wind')
        urlid = mp.argbool('id')
        urlcity = mp.argbool('name')
        usegeoip = mp.argbool('geoip')
        argument = mp.argsdef()
        if usegeoip:
            ip = mp.argsdef()
            urlid = False
            urlcity = True
            try:
                r = gi.record_by_addr(ip)
            except:
                try:
                    r = gi.record_by_name(ip)
                except:
                    r = None
            try:
                argument = r['city'] + ', ' + r['region_code'] + ', '\
                        + r['country_code']
                main.sendcmsg("Using: " + argument)
            except TypeError:
                main.sendcmsg('Cannot get GeoIP information.')
                return True
        if not urlid:
            urlcity = True
        returntemp = mp.argbool('temp')
        #funtion that gets the info to be displayed

        if not returntemp and not returnwind:
            main.sendcmsg("Specify what information you want.")
            return True
        if (urlcity and urlid):
            main.sendcmsg("You can't have both -name and -id!")
        else:
            jsonurl = 'http://api.openweathermap.org/data/2.5/weather?q=' + \
            argument
            if urlid:
                jsonurl = \
                'http://api.openweathermap.org/data/2.5/weather?id=' + \
                argument
            wxinfo = weatherinfo(jsonurl, tempstyle, argument)
            data = wxinfo.data
            if data["cod"] == 200:
                if returnwind:
                    main.sendcmsg(wxinfo.getinfo("currentwindspeed"))
                if returntemp:
                    main.sendcmsg(wxinfo.getinfo("currenttemp"))
            elif data["cod"] == "404":
                main.sendcmsg(data["cod"] + ":" + data["message"])

'''
def showhelp():
    main.sendcmsg(
        'Weather is a module that returns many usefull weather information.'
        )
    main.sendcmsg('This module uses OpenWeatherMap api (openweathermap.org)')
    main.sendcmsg("")
    main.sendcmsg('There are several options listed below:')
    main.sendcmsg('        -cel: Returns answer in celcius')
    main.sendcmsg("        -far: Returns answer in Farenheit")
    main.sendcmsg("        -kel: Returns answer in Kelvin")
    main.sendcmsg("        -name: Looks up info by city name. " +
    "Please use the closest large town")
    main.sendcmsg("        -id: Looks up by city id")
    main.sendcmsg("        -wind: Finds wind information")
'''

def showhelp():
    main.sendcmsg(
        ".weather [-temp] [-wind] [-cel -kel -far] [-name -id]" +
        "<city name/id>:" +
        " Get weather from http://openweathermap.org")
    main.sendcmsg("-temp, -wind: Get temperature, wind.")
    main.sendcmsg(
        "-cel, -kel, -far: Use celsius, kelvin, farenheit, -cel is default.")
    main.sendcmsg(
        "-name, -id, -geoip: Use city name or ID or GeoIP, -name is default.")<commit_msg>fix a [+ +]<commit_after># -*- coding: utf-8 -*-
from base import *
from json import loads
from urllib.request import urlopen
from decimal import *
import pygeoip
gi = pygeoip.GeoIP('pygeoip/GeoLiteCity.dat')

class weatherinfo:
    data = {}

    def getinfo(self, number):
        data = self.data
        name = data["name"]
        cname = data["sys"]["country"]
        if len(name) == 0:
            name = self.dname
        #checks what the request wants
        if number == "currenttemp":
            if self.outputtemp == "kel":
                return "It is " +\
                str(data["main"]["temp"]) +\
                " Kelvin in " + name + ", " + cname
            elif self.outputtemp == "cel":
                temp = Decimal(str(data["main"]["temp"])) - Decimal('273.15')
                return "It is " + str(temp) +\
                " Celcius in " + name + ", " + cname
            elif self.outputtemp == "far":
                temp = Decimal(str(data["main"]["temp"])) - Decimal('273.15')
                temp = Decimal(str(temp)) * Decimal('1.8') + Decimal('32.0')
                return "It is " + str(temp) +\
                " Farenheit in " + name + ", " + cname
            else:
                raise ValueError(
                    "Invalid Temperature Style: " + self.outputtemp)
        elif number == "currentwindspeed":
            return "The Wind is Blowing at " +\
            str(data["wind"]["speed"]) + " m/s in " + name + ", " + cname

    def __init__(self, url, style, altname):
        jsonobj = urlopen(url)
        jsonobj = jsonobj.read()
        self.data = loads(jsonobj.decode())
        self.outputtemp = style
        self.dname = altname

def msg(mp):
    if mp.cmd('weather'):
        tempstyle = "cel"
        if mp.argbool('cel'):
            tempstyle = "cel"
        elif mp.argbool('kel'):
            tempstyle = "kel"
        elif mp.argbool('far'):
            tempstyle = "far"
        returnwind = mp.argbool('wind')
        urlid = mp.argbool('id')
        urlcity = mp.argbool('name')
        usegeoip = mp.argbool('geoip')
        argument = mp.argsdef()
        if usegeoip:
            ip = mp.argsdef()
            urlid = False
            urlcity = True
            try:
                r = gi.record_by_addr(ip)
            except:
                try:
                    r = gi.record_by_name(ip)
                except:
                    r = None
            try:
                argument = r['city'] + ', ' + r['region_code'] + ', '\
                        + r['country_code']
                main.sendcmsg("Using: " + argument)
            except TypeError:
                main.sendcmsg('Cannot get GeoIP information.')
                return True
        if not urlid:
            urlcity = True
        returntemp = mp.argbool('temp')
        #funtion that gets the info to be displayed

        if not returntemp and not returnwind:
            main.sendcmsg("Specify what information you want.")
            return True
        if (urlcity and urlid):
            main.sendcmsg("You can't have both -name and -id!")
        else:
            jsonurl = 'http://api.openweathermap.org/data/2.5/weather?q=' + \
            argument
            if urlid:
                jsonurl = \
                'http://api.openweathermap.org/data/2.5/weather?id=' + \
                argument
            wxinfo = weatherinfo(jsonurl, tempstyle, argument)
            data = wxinfo.data
            if data["cod"] == 200:
                if returnwind:
                    main.sendcmsg(wxinfo.getinfo("currentwindspeed"))
                if returntemp:
                    main.sendcmsg(wxinfo.getinfo("currenttemp"))
            elif data["cod"] == "404":
                main.sendcmsg(data["cod"] + ":" + data["message"])

'''
def showhelp():
    main.sendcmsg(
        'Weather is a module that returns many usefull weather information.'
        )
    main.sendcmsg('This module uses OpenWeatherMap api (openweathermap.org)')
    main.sendcmsg("")
    main.sendcmsg('There are several options listed below:')
    main.sendcmsg('        -cel: Returns answer in celcius')
    main.sendcmsg("        -far: Returns answer in Farenheit")
    main.sendcmsg("        -kel: Returns answer in Kelvin")
    main.sendcmsg("        -name: Looks up info by city name. " +
    "Please use the closest large town")
    main.sendcmsg("        -id: Looks up by city id")
    main.sendcmsg("        -wind: Finds wind information")
'''

def showhelp():
    main.sendcmsg(
        ".weather [-temp] [-wind] [-cel -kel -far] [-name -id]" +
        "<city name/id>:" +
        " Get weather from http://openweathermap.org")
    main.sendcmsg("-temp, -wind: Get temperature, wind.")
    main.sendcmsg(
        "-cel, -kel, -far: Use celsius, kelvin, farenheit, -cel is default.")
    main.sendcmsg(
        "-name, -id, -geoip: Use city name or ID or GeoIP, -name is default.")<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2016, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import os
import io
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from .test_models import BaseModelTestCase

here = os.path.abspath(os.path.dirname(__file__))
TEST_DATA_DIR = os.path.join(here, 'data')

class ReconstituteTestCase(unittest.TestCase):

    def test(self):
        from lxml import etree
        from ..models import model_to_tree

        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.html')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)

        self.assertEqual('Desserts', desserts.metadata['title'])

        self.assertEqual({
            'id': 'book',
            'title': 'Desserts',
            'contents': [
                {
                    'id': 'subcol',
                    'title': 'Fruity',
                    'contents': [
                        {
                            'id': 'Apple',
                            'title': 'Apple',
                            },
                        {
                            'id': 'Lemon',
                            'title': 'Lemon',
                            },
                        {
                            'id': 'subcol',
                            'title': 'Citrus',
                            'contents': [
                                {
                                    'id': 'Lemon',
                                    'title': 'Lemon',
                                    },
                                ],
                            },
                        ],
                    },
                {
                    'id': u'チョコレート',
                    'title': u'チョコレート',
                    },
                {
                    'id': 'Extra Stuff',
                    'title': 'Extra Stuff',
                    },
                ],
            }, model_to_tree(desserts))

        base_metadata = {
            u'publishers': [],
            u'created': None,  # '2016/03/04 17:05:20 -0500',
            u'revised': None,  # '2013/03/05 09:35:24 -0500',
            u'authors': [
                {u'type': u'cnx-id',
                 u'name': u'Good Food',
                 u'id': u'yum'}],
            u'editors': [],
            u'copyright_holders': [],
            u'illustrators': [],
            u'subjects': [u'Humanities'],
            u'translators': [],
            u'keywords': [u'Food', u'デザート', u'Pudding'],
            u'title': u'チョコレート',
            u'license_text': u'CC-By 4.0',
            u'license_url': u'http://creativecommons.org/licenses/by/4.0/',
            # 'version': 'draft',
            u'language': None,
            u'print_style': None,
            u'cnx-archive-uri': None,
            u'derived_from_title': None,
            u'derived_from_uri': None,
            }

        fruity = desserts[0]
        self.assertEqual('TranslucentBinder', fruity.__class__.__name__)
        self.assertEqual('Fruity', fruity.metadata['title'])

        apple = fruity[0]
        self.assertEqual('Document', apple.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Apple'
        apple_metadata = apple.metadata.copy()
        summary = etree.fromstring(apple_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, apple_metadata)

        lemon = fruity[1]
        self.assertEqual('Document', lemon.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Lemon'
        lemon_metadata = lemon.metadata.copy()
        summary = etree.fromstring(lemon_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, lemon_metadata)

        citrus = fruity[2]
        self.assertEqual('TranslucentBinder', citrus.__class__.__name__)
        self.assertEqual(citrus.metadata['title'], 'Citrus')

        self.assertEqual(lemon.metadata, citrus[0].metadata)

        chocolate = desserts[1]
        self.assertEqual('Document', chocolate.__class__.__name__)
        chocolate_metadata = chocolate.metadata.copy()
        summary = etree.fromstring(chocolate_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = u'チョコレート'
        self.assertEqual(metadata, chocolate_metadata)

        extra = desserts[2]
        self.assertEqual('CompositeDocument', extra.__class__.__name__)
        extra_metadata = extra.metadata.copy()
        summary = etree.fromstring(extra_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = 'Extra Stuff'
        self.assertEqual(metadata, extra_metadata)

class CollateTestCase(BaseModelTestCase):

    @property
    def target(self):
        from cnxepub import collate
        return collate

    def test(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"
document one
",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"
document two
",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b" ")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        def mock_easybake(ruleset, in_html, out_html):
            from lxml import etree
            html = etree.parse(in_html)
            # Add in a composite-page with title "Composite One" here.
            body = html.getroot().xpath(
                '//xhtml:body',
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            comp_elm = etree.SubElement(body, 'div')
            comp_elm.attrib['data-type'] = 'composite-page'
            comp_elm.append(etree.fromstring("""

Composite One

                By:
                Edited by:
                Illustrated by:
                Translated by:

                Published By:

                Licensed:

"""))
            etree.SubElement(comp_elm, 'p').text = "composite document"
            # Add the composite-page to the table-of-contents.
            toc = html.getroot().xpath(
                "//xhtml:*[@id='toc']/xhtml:ol",
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            etree.SubElement(toc, 'li').append(etree.fromstring('Composite One'))
            out_html.write(etree.tostring(html))

        with mock.patch('cnxepub.collation.easybake') as easybake:
            easybake.side_effect = mock_easybake
            collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Composite One')

    def test_without_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        result = self.target(binder)
        self.assertIs(binder, result)
<commit_msg>Add test that uses a real ruleset<commit_after># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2016, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import os
import io
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from .test_models import BaseModelTestCase

here = os.path.abspath(os.path.dirname(__file__))
TEST_DATA_DIR = os.path.join(here, 'data')

class ReconstituteTestCase(unittest.TestCase):

    def test(self):
        from lxml import etree
        from ..models import model_to_tree

        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.html')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)

        self.assertEqual('Desserts', desserts.metadata['title'])

        self.assertEqual({
            'id': 'book',
            'title': 'Desserts',
            'contents': [
                {
                    'id': 'subcol',
                    'title': 'Fruity',
                    'contents': [
                        {
                            'id': 'Apple',
                            'title': 'Apple',
                            },
                        {
                            'id': 'Lemon',
                            'title': 'Lemon',
                            },
                        {
                            'id': 'subcol',
                            'title': 'Citrus',
                            'contents': [
                                {
                                    'id': 'Lemon',
                                    'title': 'Lemon',
                                    },
                                ],
                            },
                        ],
                    },
                {
                    'id': u'チョコレート',
                    'title': u'チョコレート',
                    },
                {
                    'id': 'Extra Stuff',
                    'title': 'Extra Stuff',
                    },
                ],
            }, model_to_tree(desserts))

        base_metadata = {
            u'publishers': [],
            u'created': None,  # '2016/03/04 17:05:20 -0500',
            u'revised': None,  # '2013/03/05 09:35:24 -0500',
            u'authors': [
                {u'type': u'cnx-id',
                 u'name': u'Good Food',
                 u'id': u'yum'}],
            u'editors': [],
            u'copyright_holders': [],
            u'illustrators': [],
            u'subjects': [u'Humanities'],
            u'translators': [],
            u'keywords': [u'Food', u'デザート', u'Pudding'],
            u'title': u'チョコレート',
            u'license_text': u'CC-By 4.0',
            u'license_url': u'http://creativecommons.org/licenses/by/4.0/',
            # 'version': 'draft',
            u'language': None,
            u'print_style': None,
            u'cnx-archive-uri': None,
            u'derived_from_title': None,
            u'derived_from_uri': None,
            }

        fruity = desserts[0]
        self.assertEqual('TranslucentBinder', fruity.__class__.__name__)
        self.assertEqual('Fruity', fruity.metadata['title'])

        apple = fruity[0]
        self.assertEqual('Document', apple.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Apple'
        apple_metadata = apple.metadata.copy()
        summary = etree.fromstring(apple_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, apple_metadata)

        lemon = fruity[1]
        self.assertEqual('Document', lemon.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Lemon'
        lemon_metadata = lemon.metadata.copy()
        summary = etree.fromstring(lemon_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, lemon_metadata)

        citrus = fruity[2]
        self.assertEqual('TranslucentBinder', citrus.__class__.__name__)
        self.assertEqual(citrus.metadata['title'], 'Citrus')

        self.assertEqual(lemon.metadata, citrus[0].metadata)

        chocolate = desserts[1]
        self.assertEqual('Document', chocolate.__class__.__name__)
        chocolate_metadata = chocolate.metadata.copy()
        summary = etree.fromstring(chocolate_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = u'チョコレート'
        self.assertEqual(metadata, chocolate_metadata)

        extra = desserts[2]
        self.assertEqual('CompositeDocument', extra.__class__.__name__)
        extra_metadata = extra.metadata.copy()
        summary = etree.fromstring(extra_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = 'Extra Stuff'
        self.assertEqual(metadata, extra_metadata)

class CollateTestCase(BaseModelTestCase):

    @property
    def target(self):
        from cnxepub import collate
        return collate

    def test(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"
document one
",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"
document two
",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b" ")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        def mock_easybake(ruleset, in_html, out_html):
            from lxml import etree
            html = etree.parse(in_html)
            # Add in a composite-page with title "Composite One" here.
            body = html.getroot().xpath(
                '//xhtml:body',
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            comp_elm = etree.SubElement(body, 'div')
            comp_elm.attrib['data-type'] = 'composite-page'
            comp_elm.append(etree.fromstring("""

Composite One

                By:
                Edited by:
                Illustrated by:
                Translated by:

                Published By:

                Licensed:

"""))
            etree.SubElement(comp_elm, 'p').text = "composite document"
            # Add the composite-page to the table-of-contents.
            toc = html.getroot().xpath(
                "//xhtml:*[@id='toc']/xhtml:ol",
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            etree.SubElement(toc, 'li').append(etree.fromstring('Composite One'))
            out_html.write(etree.tostring(html))

        with mock.patch('cnxepub.collation.easybake') as easybake:
            easybake.side_effect = mock_easybake
            collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Composite One')

    def test_without_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        result = self.target(binder)
        self.assertIs(binder, result)

    def test_with_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"document one",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"document two",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b"""\
div[data-type='page'] > div[data-type='metadata'] {
  copy-to: eob-all
}
div[data-type='page'] span {
  copy-to: eob-all
}
body::after {
  content: pending(eob-all);
  class: end-of-book;
  data-type: composite-page;
  container: div;
}
""")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Composite One')
<|endoftext|>


<commit_before>"""
MXP - Mud eXtension Protocol.

Partial implementation of the MXP protocol.
The MXP protocol allows more advanced formatting options for telnet clients
that supports it (mudlet, zmud, mushclient are a few)

This only implements the SEND tag.

More information can be found on the following links:
http://www.zuggsoft.com/zmud/mxp.htm
http://www.mushclient.com/mushclient/mxp.htm
http://www.gammon.com.au/mushclient/addingservermxp.htm

"""
from builtins import object
import re

LINKS_SUB = re.compile(r'\|lc(.*?)\|lt(.*?)\|le', re.DOTALL)

MXP = "\x5B"
MXP_TEMPSECURE = "\x1B[4z"
MXP_SEND = MXP_TEMPSECURE + \
           "<SEND HREF=\"\\1\">" + \
           "\\2" + \
           MXP_TEMPSECURE + \
           "</SEND>"

def mxp_parse(text):
    """
    Replaces links to the correct format for MXP.

    Args:
        text (str): The text to parse.

    Returns:
        parsed (str): The parsed text.

    """
    text = text.replace("&", "&") \
               .replace('"', """) \
               .replace("<", "<") \
               .replace(">", ">")

    text = LINKS_SUB.sub(MXP_SEND, text)
    return text

class Mxp(object):
    """
    Implements the MXP protocol.

    """

    def __init__(self, protocol):
        """
        Initializes the protocol by checking if the client supports it.

        Args:
            protocol (Protocol): The active protocol instance.

        """
        self.protocol = protocol
        self.protocol.protocol_flags["MXP"] = False
        self.protocol.will(MXP).addCallbacks(self.do_mxp, self.no_mxp)

    def no_mxp(self, option):
        """
        Called when the Client reports to not support MXP.

        Args:
            option (Option): Not used.

        """
        self.protocol.protocol_flags["MXP"] = False
        self.protocol.handshake_done()

    def do_mxp(self, option):
        """
        Called when the Client reports to support MXP.

        Args:
            option (Option): Not used.

        """
        self.protocol.protocol_flags["MXP"] = True
        self.protocol.requestNegotiation(MXP, '')
        self.protocol.handshake_done()
<commit_msg>Make the MXP charcode more explicit for readabilty.<commit_after>"""
MXP - Mud eXtension Protocol.

Partial implementation of the MXP protocol.
The MXP protocol allows more advanced formatting options for telnet clients
that supports it (mudlet, zmud, mushclient are a few)

This only implements the SEND tag.

More information can be found on the following links:
http://www.zuggsoft.com/zmud/mxp.htm
http://www.mushclient.com/mushclient/mxp.htm
http://www.gammon.com.au/mushclient/addingservermxp.htm

"""
from builtins import object
import re

LINKS_SUB = re.compile(r'\|lc(.*?)\|lt(.*?)\|le', re.DOTALL)

MXP = chr(91)
MXP_TEMPSECURE = "\x1B[4z"
MXP_SEND = MXP_TEMPSECURE + \
           "<SEND HREF=\"\\1\">" + \
           "\\2" + \
           MXP_TEMPSECURE + \
           "</SEND>"

def mxp_parse(text):
    """
    Replaces links to the correct format for MXP.

    Args:
        text (str): The text to parse.

    Returns:
        parsed (str): The parsed text.

    """
    text = text.replace("&", "&") \
               .replace('"', """) \
               .replace("<", "<") \
               .replace(">", ">")

    text = LINKS_SUB.sub(MXP_SEND, text)
    return text

class Mxp(object):
    """
    Implements the MXP protocol.

    """

    def __init__(self, protocol):
        """
        Initializes the protocol by checking if the client supports it.

        Args:
            protocol (Protocol): The active protocol instance.

        """
        self.protocol = protocol
        self.protocol.protocol_flags["MXP"] = False
        self.protocol.will(MXP).addCallbacks(self.do_mxp, self.no_mxp)

    def no_mxp(self, option):
        """
        Called when the Client reports to not support MXP.

        Args:
            option (Option): Not used.

        """
        self.protocol.protocol_flags["MXP"] = False
        self.protocol.handshake_done()

    def do_mxp(self, option):
        """
        Called when the Client reports to support MXP.

        Args:
            option (Option): Not used.

        """
        self.protocol.protocol_flags["MXP"] = True
        self.protocol.requestNegotiation(MXP, '')
        self.protocol.handshake_done()
<|endoftext|>