<commit_before>#!/usr/local/bin/python3

import os
os.environ['OMP_NUM_THREADS'] = '8'

import argparse
from SCNIC.annotate_correls import do_annotate_correls
from SCNIC.calculate_permutations import do_multiprocessed_perms
from SCNIC.calculate_permutation_stats import do_stats

def main():
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers(dest='subparser_name')
    annotate_correls = subparsers.add_parser("annotate", help="Annotate correls.txt file",
                                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    run_perms = subparsers.add_parser("perms", help="Run permutation generation",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    calc_stats = subparsers.add_parser("stats", help="Generate p-values from the permutations",
                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # annotate correlations
    annotate_correls.add_argument('--correls')
    annotate_correls.add_argument('--tree')
    annotate_correls.add_argument('--genome')
    annotate_correls.add_argument('--modules')
    annotate_correls.add_argument('--output')

    # run permutations multiprocessed
    run_perms.add_argument('--correls')
    run_perms.add_argument('--perms', type=int, default=2000)
    run_perms.add_argument('--procs', type=int, default=4)
    run_perms.add_argument('--modules')
    run_perms.add_argument('--output')
    run_perms.add_argument('--to_keep')

    # calculate statistics
    calc_stats.add_argument('--correls')
    calc_stats.add_argument('--modules')
    calc_stats.add_argument('--perms')
    calc_stats.add_argument('--output')

    args = parser.parse_args()

    if args.subparser_name == 'annotate':
        do_annotate_correls(args.correls, args.tree, args.genome, args.modules, args.output)
    elif args.subparser_name == 'perms':
        do_multiprocessed_perms(args.correls, args.perms, args.procs, args.modules, args.output, args.to_keep)
    elif args.subparser_name == 'stats':
        do_stats(args.correls, args.modules, args.perms, args.output)
    else:
        print('What the hell happened here?')

if __name__ == '__main__':
    main()
<commit_msg>Add to keep parameter to stats<commit_after>#!/usr/local/bin/python3

import os
os.environ['OMP_NUM_THREADS'] = '8'

import argparse
from SCNIC.annotate_correls import do_annotate_correls
from SCNIC.calculate_permutations import do_multiprocessed_perms
from SCNIC.calculate_permutation_stats import do_stats

def main():
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers(dest='subparser_name')
    annotate_correls = subparsers.add_parser("annotate", help="Annotate correls.txt file",
                                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    run_perms = subparsers.add_parser("perms", help="Run permutation generation",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    calc_stats = subparsers.add_parser("stats", help="Generate p-values from the permutations",
                                       formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # annotate correlations
    annotate_correls.add_argument('--correls')
    annotate_correls.add_argument('--tree')
    annotate_correls.add_argument('--genome')
    annotate_correls.add_argument('--modules')
    annotate_correls.add_argument('--output')

    # run permutations multiprocessed
    run_perms.add_argument('--correls')
    run_perms.add_argument('--perms', type=int, default=2000)
    run_perms.add_argument('--procs', type=int, default=4)
    run_perms.add_argument('--modules')
    run_perms.add_argument('--output')
    run_perms.add_argument('--to_keep')

    # calculate statistics
    calc_stats.add_argument('--correls')
    calc_stats.add_argument('--modules')
    calc_stats.add_argument('--perms')
    calc_stats.add_argument('--output')
    calc_stats.add_argument('--to_keep')

    args = parser.parse_args()

    if args.subparser_name == 'annotate':
        do_annotate_correls(args.correls, args.tree, args.genome, args.modules, args.output)
    elif args.subparser_name == 'perms':
        do_multiprocessed_perms(args.correls, args.perms, args.procs, args.modules, args.output, args.to_keep)
    elif args.subparser_name == 'stats':
        do_stats(args.correls, args.modules, args.perms, args.output, args.to_keep)
    else:
        print('What the hell happened here?')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pymel.core as pm

class UI(object):
    def __init__(self):
        title = 'pbRenderableCurve'
        version = 1.0

        if pm.window('pbRCurve', exists=True):
            pm.deleteUI('pbRCurve')

        with pm.window('pbRCurve', title='{0} | {1}'.format(title, version), s=False) as window:
            with pm.columnLayout():
                with pm.frameLayout(l='Selection:', cll=True, bs='out'):
                    with pm.columnLayout():
                        self.selField = pm.textFieldGrp(text='No Curves Selected', ed=False, l='Curve:', cw2=[72, 192])
                        with pm.rowLayout(nc=2):
                            self.bRenderable = pm.checkBox(l='Renderable', cc=self.bcRenderable)
                            # pm.checkBox(l='Mesh Selection')

                with pm.frameLayout(l='Mesh Settings:', cll=True, bs='out') as self.meshUI:
                    with pm.columnLayout():
                        self.meshAttrs = [AttrSlider(maxValue=128, name='Thickness', obj=getCurves, type_='float', fmn=0.0001),
                                          AttrSlider(value=3, minValue=3, maxValue=64, name='Sides', obj=getCurves, fmn=3, fmx=100),
                                          AttrSlider(minValue=1, maxValue=32, name='Samples', obj=getCurves, fmn=1, fmx=32)]

                with pm.frameLayout('Shell Settings:', cll=True, bs='out') as self.shellUI:
                    with pm.columnLayout():
                        self.bShell = pm.checkBox(l='Enable Shell', cc=self.bcShell)
                        self.shellAttrs = [AttrSlider(value=1, minValue=-64, maxValue=64, name='ShellThickness', obj=getCurves, type_='float'),
                                           AttrSlider(value=1, minValue=1, maxValue=64, name='ShellDivisions', obj=getCurves, fmn=1, fmx=32)]

        window.show()
        pm.scriptJob(event=['SelectionChanged', self.refresh], protected=True, p=window)
        self.refresh()

    def refresh(self):
        curves = getCurves()
        if len(curves) == 1:
            self.selField.setText(curves[0])
            self.getValues(curves)

        elif len(curves) > 1:
            self.selField.setText('%s Curves Selected' % len(curves))
            self.getValues(curves)

        else:
            self.selField.setText('No Curves Selected')
            self.bRenderable.setValue(False)
            self.bShell.setValue(False)
            for i in self.meshAttrs:
                i.setEnable(False)

            for i in self.shellAttrs:
                i.setEnable(False)

    def bcRenderable(self, *args):
        curves = getCurves()

        if not all(i.isRenderable() for i in curves):
            for i in curves:
                i.makeRenderable()
        else:
            for i in curves:
                i.makeNonRenderable()
        pm.select([i.curve for i in curves])
        self.refresh()

    def bcShell(self, *args):
        curves = getCurves()
        if all(i.isRenderable() for i in curves):
            if not all(i.hasShell() for i in curves):
                for i in curves:
                    i.makeShell()
            else:
                for i in curves:
                    i.makeNonShell()

        pm.select([i.curve for i in curves])
        self.refresh()

    def getValues(self, curves):
        if all(i.isRenderable() for i in curves):
            self.bRenderable.setValue(True)
            for i in self.meshAttrs:
                i.get()
                i.setEnable(True)

            if all(i.hasShell() for i in curves):
                self.bShell.setValue(True)
                for i in self.shellAttrs:
                    i.get()
                    i.setEnable(True)
            else:
                self.bShell.setValue(False)
                for i in self.shellAttrs:
                    i.setEnable(False)

        else:
            self.bRenderable.setValue(False)
            self.bShell.setValue(False)
            for i in self.meshAttrs:
                i.setEnable(False)
            for i in self.shellAttrs:
                i.setEnable(False)

class Curve(object):
    def __init__(self, curve):
        self.curve = curve

        if self.isRenderable():
            self.stroke = self.curve.getShape().connections(shapes=True, type=pm.nt.Stroke)[0]
            self.brush = self.stroke.connections(type=pm.nt.Brush)[0]

            self.thickness = self.brush.brushWidth
            self.sides = self.brush.tubeSections
            self.samples = self.stroke.sampleDensity

            self.mesh = self.stroke.worldMainMesh[0].connections()[0]
            if isinstance(self.mesh, pm.nt.PolyExtrudeFace):
                self.mesh = self.mesh.connections(type=pm.nt.Mesh)[0]

            if self.hasShell():
                self.extrudeNode = self.mesh.getShape().connections(type=pm.nt.PolyExtrudeFace)[0]
                self.shellthickness = self.extrudeNode.thickness
                self.shelldivisions = self.extrudeNode.divisions

    def __str__(self):
        return str(self.curve)

    def makeRenderable(self):
        if not self.isRenderable():
            self.createBrush()
            self.strokeToMesh()

    def makeNonRenderable(self):
        if self.isRenderable():
            if self.hasShell():
                self.makeNonShell()
            mesh = self.stroke.worldMainMesh[0].connections(shapes=True, type=pm.nt.Mesh)
            pm.delete(self.brush)
            pm.delete(self.stroke.getParent())
            pm.delete(mesh[0].getParent())

    def makeShell(self):
        if not self.hasShell():
            self.ExtrudeNode = pm.polyExtrudeFacet(self.mesh, thickness=1)

    def makeNonShell(self):
        if self.hasShell():
            pm.delete(self.extrudeNode)

    def createBrush(self):
        self.brush = pm.createNode('brush', name='{0}Brush'.format(self.curve))
        self.stroke = pm.stroke(name='{0}Stroke'.format(self.curve), seed=0, pressure=True)

        self.brush.outBrush >> self.stroke.brush
        pm.PyNode('time1').outTime >> self.brush.time

        # Brush Defualts
        self.brush.brushWidth.set(1)
        self.brush.tubeSections.set(8)
        self.brush.strokeTime.set(1)

        # Brush type mesh
        self.brush.brushType.set(5)

        # set divisions
        spans = self.curve.spans.get()
        deg = self.curve.degree.get()
        samples = (spans + 1) * deg
        if deg > 1:
            samples = (spans * 5) + 1

        self.stroke.getShape().pathCurve[0].samples.set(samples)

        self.stroke.useNormal.set(0)
        self.stroke.normalY.set(1.0)

        self.curve.ws >> self.stroke.getShape().pathCurve[0].curve

        self.stroke.perspective.set(1)
        self.stroke.displayPercent.set(100.0)

    def strokeToMesh(self):
        # Stroke settings
        self.stroke.meshVertexColorMode.set(0)
        self.stroke.meshQuadOutput.set(1)
        self.stroke.meshPolyLimit.set(100000)

        # output mesh stuff
        meshName = self.brush.replace('Brush', "Mesh")
        self.mesh = pm.createNode('mesh', n='%sShape' % meshName)
        self.stroke.getShape().worldMainMesh[0] >> self.mesh.inMesh

        # Parent Stroke to mesh to clean up outliner
        self.stroke.setParent(self.mesh.getParent())

        # Display mesh as ref
        self.mesh.overrideEnabled.set(1)
        self.mesh.overrideDisplayType.set(2)
        self.stroke.visibility.set(0)

        # lambert Shadeing
        sg = pm.PyNode('initialShadingGroup')
        sg.add(self.mesh)

    def isRenderable(self):
        strokes = self.curve.getShape().connections(shapes=True, type=pm.nt.Stroke)
        if strokes:
            return True
        else:
            return False

    def hasShell(self):
        meshShape = self.mesh.getShape()
        if isinstance(meshShape.inMesh.connections()[0], pm.nt.PolyExtrudeFace):
            return True
        else:
            return False

class AttrSlider(object):
    def __init__(self, value=0, minValue=0, maxValue=32, name=None, obj=None, type_='int', en=False, fmx=10000, fmn=-10000):
        self.name = name
        self.obj = obj

        if type_ == 'float':
            self.attr_ = pm.floatSliderGrp(field=True, l=self.name, value=value, minValue=minValue, maxValue=maxValue,
                                           cc=self.set, dc=self.set, pre=3, cw3=[72, 64, 128], en=en, fmx=fmx, fmn=fmn)
        elif type_ == 'int':
            self.attr_ = pm.intSliderGrp(field=True, l=self.name, value=value, minValue=minValue, maxValue=maxValue,
                                         cc=self.set, dc=self.set, cw3=[72, 64, 128], en=en, fmn=fmn, fmx=fmx)
        else:
            raise AttributeError('%s is not a valid type' % type_)

    def get(self, *args):
        try:
            value = getattr(self.obj()[0], self.name.lower()).get()
            self.attr_.setValue(value)
        except AttributeError:
            pass

    def set(self, *args):
        for i in self.obj():
            getattr(i, self.name.lower()).set(self.attr_.getValue())

    def setEnable(self, val):
        self.attr_.setEnable(val)

def getCurves():
    sel = pm.ls(sl=True, l=True, type=pm.nt.Transform)
    curveList = []
    if sel:
        for i in sel:
            if isinstance(i.getShape(), pm.nt.NurbsCurve):
                curveList.append(Curve(i))
    return curveList
<commit_msg>now works on component selections as well as curve objects.<commit_after>import pymel.core as pm

class UI(object):
    def __init__(self):
        title = 'pbRenderableCurve'
        version = 1.0

        if pm.window('pbRCurve', exists=True):
            pm.deleteUI('pbRCurve')

        with pm.window('pbRCurve', title='{0} | {1}'.format(title, version), s=False) as window:
            with pm.columnLayout():
                with pm.frameLayout(l='Selection:', cll=True, bs='out'):
                    with pm.columnLayout():
                        self.selField = pm.textFieldGrp(text='No Curves Selected', ed=False, l='Curve:', cw2=[72, 192])
                        with pm.rowLayout(nc=2):
                            self.bRenderable = pm.checkBox(l='Renderable', cc=self.bcRenderable)
                            # pm.checkBox(l='Mesh Selection')

                with pm.frameLayout(l='Mesh Settings:', cll=True, bs='out') as self.meshUI:
                    with pm.columnLayout():
                        self.meshAttrs = [AttrSlider(maxValue=128, name='Thickness', obj=getCurves, type_='float', fmn=0.0001),
                                          AttrSlider(value=3, minValue=3, maxValue=64, name='Sides', obj=getCurves, fmn=3, fmx=100),
                                          AttrSlider(minValue=1, maxValue=32, name='Samples', obj=getCurves, fmn=1, fmx=128)]

                with pm.frameLayout('Shell Settings:', cll=True, bs='out') as self.shellUI:
                    with pm.columnLayout():
                        self.bShell = pm.checkBox(l='Enable Shell', cc=self.bcShell)
                        self.shellAttrs = [AttrSlider(value=1, minValue=-64, maxValue=64, name='ShellThickness', obj=getCurves, type_='float'),
                                           AttrSlider(value=1, minValue=1, maxValue=64, name='ShellDivisions', obj=getCurves, fmn=1, fmx=32)]

        window.show()
        pm.scriptJob(event=['SelectionChanged', self.refresh], protected=True, p=window)
        self.refresh()

    def refresh(self):
        curves = getCurves()
        if len(curves) == 1:
            self.selField.setText(curves[0])
            self.getValues(curves)

        elif len(curves) > 1:
            self.selField.setText('%s Curves Selected' % len(curves))
            self.getValues(curves)

        else:
            self.selField.setText('No Curves Selected')
            self.bRenderable.setValue(False)
            self.bShell.setValue(False)
            for i in self.meshAttrs:
                i.setEnable(False)

            for i in self.shellAttrs:
                i.setEnable(False)

    def bcRenderable(self, *args):
        curves = getCurves()

        if not all(i.isRenderable() for i in curves):
            for i in curves:
                i.makeRenderable()
        else:
            for i in curves:
                i.makeNonRenderable()
        pm.select([i.curve for i in curves])
        self.refresh()

    def bcShell(self, *args):
        curves = getCurves()
        if all(i.isRenderable() for i in curves):
            if not all(i.hasShell() for i in curves):
                for i in curves:
                    i.makeShell()
            else:
                for i in curves:
                    i.makeNonShell()

        pm.select([i.curve for i in curves])
        self.refresh()

    def getValues(self, curves):
        if all(i.isRenderable() for i in curves):
            self.bRenderable.setValue(True)
            for i in self.meshAttrs:
                i.get()
                i.setEnable(True)

            if all(i.hasShell() for i in curves):
                self.bShell.setValue(True)
                for i in self.shellAttrs:
                    i.get()
                    i.setEnable(True)
            else:
                self.bShell.setValue(False)
                for i in self.shellAttrs:
                    i.setEnable(False)

        else:
            self.bRenderable.setValue(False)
            self.bShell.setValue(False)
            for i in self.meshAttrs:
                i.setEnable(False)
            for i in self.shellAttrs:
                i.setEnable(False)

class Curve(object):
    def __init__(self, curve):
        self.curve = curve

        if self.isRenderable():
            self.stroke = self.curve.getShape().connections(shapes=True, type=pm.nt.Stroke)[0]
            self.brush = self.stroke.connections(type=pm.nt.Brush)[0]

            self.thickness = self.brush.brushWidth
            self.sides = self.brush.tubeSections
            self.samples = self.stroke.sampleDensity

            self.mesh = self.stroke.worldMainMesh[0].connections()[0]
            if isinstance(self.mesh, pm.nt.PolyExtrudeFace):
                self.mesh = self.mesh.connections(type=pm.nt.Mesh)[0]

            if self.hasShell():
                self.extrudeNode = self.mesh.getShape().connections(type=pm.nt.PolyExtrudeFace)[0]
                self.shellthickness = self.extrudeNode.thickness
                self.shelldivisions = self.extrudeNode.divisions

    def __str__(self):
        return str(self.curve)

    def makeRenderable(self):
        if not self.isRenderable():
            self.createBrush()
            self.strokeToMesh()

    def makeNonRenderable(self):
        if self.isRenderable():
            if self.hasShell():
                self.makeNonShell()
            mesh = self.stroke.worldMainMesh[0].connections(shapes=True, type=pm.nt.Mesh)
            pm.delete(self.brush)
            pm.delete(self.stroke.getParent())
            pm.delete(mesh[0].getParent())

    def makeShell(self):
        if not self.hasShell():
            self.ExtrudeNode = pm.polyExtrudeFacet(self.mesh, thickness=1)

    def makeNonShell(self):
        if self.hasShell():
            pm.delete(self.extrudeNode)

    def createBrush(self):
        self.brush = pm.createNode('brush', name='{0}Brush'.format(self.curve))
        self.stroke = pm.stroke(name='{0}Stroke'.format(self.curve), seed=0, pressure=True)

        self.brush.outBrush >> self.stroke.brush
        pm.PyNode('time1').outTime >> self.brush.time

        # Brush Defualts
        self.brush.brushWidth.set(1)
        self.brush.tubeSections.set(8)
        self.brush.strokeTime.set(1)

        # Brush type mesh
        self.brush.brushType.set(5)

        # set divisions
        spans = self.curve.spans.get()
        deg = self.curve.degree.get()
        samples = (spans + 1) * deg
        if deg > 1:
            samples = (spans * 5) + 1

        self.stroke.getShape().pathCurve[0].samples.set(samples)

        self.stroke.useNormal.set(0)
        self.stroke.normalY.set(1.0)

        self.curve.ws >> self.stroke.getShape().pathCurve[0].curve

        self.stroke.perspective.set(1)
        self.stroke.displayPercent.set(100.0)

    def strokeToMesh(self):
        # Stroke settings
        self.stroke.meshVertexColorMode.set(0)
        self.stroke.meshQuadOutput.set(1)
        self.stroke.meshPolyLimit.set(100000)

        # output mesh stuff
        meshName = self.brush.replace('Brush', "Mesh")
        self.mesh = pm.createNode('mesh', n='%sShape' % meshName)
        self.stroke.getShape().worldMainMesh[0] >> self.mesh.inMesh

        # Parent Stroke to mesh to clean up outliner
        self.stroke.setParent(self.mesh.getParent())

        # Display mesh as ref
        self.mesh.overrideEnabled.set(1)
        self.mesh.overrideDisplayType.set(2)
        self.stroke.visibility.set(0)

        # lambert Shadeing
        sg = pm.PyNode('initialShadingGroup')
        sg.add(self.mesh)

    def isRenderable(self):
        strokes = self.curve.getShape().connections(shapes=True, type=pm.nt.Stroke)
        if strokes:
            return True
        else:
            return False

    def hasShell(self):
        meshShape = self.mesh.getShape()
        if isinstance(meshShape.inMesh.connections()[0], pm.nt.PolyExtrudeFace):
            return True
        else:
            return False

class AttrSlider(object):
    def __init__(self, value=0, minValue=0, maxValue=32, name=None, obj=None, type_='int', en=False, fmx=10000, fmn=-10000):
        self.name = name
        self.obj = obj

        if type_ == 'float':
            self.attr_ = pm.floatSliderGrp(field=True, l=self.name, value=value, minValue=minValue, maxValue=maxValue,
                                           cc=self.set, dc=self.set, pre=5, cw3=[72, 64, 128], en=en, fmx=fmx, fmn=fmn)
        elif type_ == 'int':
            self.attr_ = pm.intSliderGrp(field=True, l=self.name, value=value, minValue=minValue, maxValue=maxValue,
                                         cc=self.set, dc=self.set, cw3=[72, 64, 128], en=en, fmn=fmn, fmx=fmx)
        else:
            raise AttributeError('%s is not a valid type' % type_)

    def get(self, *args):
        try:
            value = getattr(self.obj()[0], self.name.lower()).get()
            self.attr_.setValue(value)
        except AttributeError:
            pass

    def set(self, *args):
        for i in self.obj():
            getattr(i, self.name.lower()).set(self.attr_.getValue())

    def setEnable(self, val):
        self.attr_.setEnable(val)

def getCurves():
    sel = pm.selected()
    curveList = []
    if sel:
        if isinstance(sel[0], pm.NurbsCurveCV):
            sel = list(set([i.node().getParent() for i in sel]))

        if all(isinstance(i, pm.nt.Transform) for i in sel):
            for i in sel:
                if isinstance(i.getShape(), pm.nt.NurbsCurve):
                    curveList.append(Curve(i))
    return curveList
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8:noet
from __future__ import (unicode_literals, division, absolute_import, print_function)

import argparse
import codecs
import os

from subprocess import check_output, check_call

def parse_version(s):
	if s == ('+' * len(s)):
		try:
			last_version = next(iter(sorted([
				tuple(map(int, tag[len('release-'):].split('.')))
				for tag in check_output(['git', 'tag', '-l', 'release-*']).split('\n')[:-1]
			], reverse=True)))
		except StopIteration:
			raise ValueError('No existing versions found')

		version = []
		for i in range(len(s) - 1):
			try:
				version.append(last_version[i])
			except IndexError:
				version.append(0)

		try:
			version.append(last_version[len(s) - 1] + 1)
		except IndexError:
			version.append(1)

		if len(version) == 1:
			version.append(0)

		return tuple(version)
	else:
		return tuple(map(int, s.split('.')))

p = argparse.ArgumentParser(description='Powerline release script')
p.add_argument('-v', '--version', type=parse_version, metavar='VERSION', help='Use given version for the release. If version contains only `+\' signs then it will increase latest version number: one `+\' increases major version number (e.g. 1.2.3 -> 2.0), `++\' increases minor version number (e.g. 1.2.3 -> 1.3), `+++\' increases patch level (e.g. 1.2.3 -> 1.2.4). Defaults to `+++\'', default='+++')
p.add_argument('-r', '--rev', metavar='COMMIT', help='Use given revision for the release. Defaults to `develop\'.', default='develop')

def create_release(version, rev):
	version_string = '.'.join((str(v) for v in version))
	check_call(['git', 'checkout', 'master'])
	check_call(['git', 'merge', '--no-ff', '--no-commit', '--log', rev])

	with codecs.open('.setup.py.new', 'w', encoding='utf-8') as NS:
		with codecs.open('setup.py', 'r', encoding='utf-8') as OS:
			for line in OS:
				if line.startswith('\tversion='):
					line = '\tversion=\'' + version_string + '\',\n'
				elif 'Development Status' in line:
					line = '\t\t\'Development Status :: 5 - Production/Stable\',\n'
				NS.write(line)

	os.unlink('setup.py')
	os.rename('.setup.py.new', 'setup.py')
	check_call(['git', 'add', 'setup.py'])

	check_call(['git', 'commit'])
	check_call(['git', 'tag', '-m', 'Release ' + version_string, '-a', 'release-' + version_string])
	check_call(['python', 'setup.py', 'sdist', 'upload'])
	check_call(['python', 'setup.py', 'upload_docs'])

if __name__ == '__main__':
	args = p.parse_args()
	create_release(args.version, args.rev)
<commit_msg>Push to upstream and do not use `release-` prefix<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8:noet
from __future__ import (unicode_literals, division, absolute_import, print_function)

import argparse
import codecs
import os

from subprocess import check_output, check_call

def parse_version(s):
	if s == ('+' * len(s)):
		try:
			last_version = next(iter(sorted([
				tuple(map(int, tag.split('.')))
				for tag in check_output(['git', 'tag', '-l', '[0-9]*.*']).split('\n')[:-1]
			], reverse=True)))
		except StopIteration:
			raise ValueError('No existing versions found')

		version = []
		for i in range(len(s) - 1):
			try:
				version.append(last_version[i])
			except IndexError:
				version.append(0)

		try:
			version.append(last_version[len(s) - 1] + 1)
		except IndexError:
			version.append(1)

		if len(version) == 1:
			version.append(0)

		return tuple(version)
	else:
		return tuple(map(int, s.split('.')))

p = argparse.ArgumentParser(description='Powerline release script')
p.add_argument('-v', '--version', type=parse_version, metavar='VERSION', help='Use given version for the release. If version contains only `+\' signs then it will increase latest version number: one `+\' increases major version number (e.g. 1.2.3 -> 2.0), `++\' increases minor version number (e.g. 1.2.3 -> 1.3), `+++\' increases patch level (e.g. 1.2.3 -> 1.2.4). Defaults to `+++\'', default='+++')
p.add_argument('-r', '--rev', metavar='COMMIT', help='Use given revision for the release. Defaults to `develop\'.', default='develop')

def create_release(version, rev):
	version_string = '.'.join((str(v) for v in version))
	check_call(['git', 'checkout', 'master'])
	check_call(['git', 'merge', '--no-ff', '--no-commit', '--log', rev])

	with codecs.open('.setup.py.new', 'w', encoding='utf-8') as NS:
		with codecs.open('setup.py', 'r', encoding='utf-8') as OS:
			for line in OS:
				if line.startswith('\tversion='):
					line = '\tversion=\'' + version_string + '\',\n'
				elif 'Development Status' in line:
					line = '\t\t\'Development Status :: 5 - Production/Stable\',\n'
				NS.write(line)

	os.unlink('setup.py')
	os.rename('.setup.py.new', 'setup.py')
	check_call(['git', 'add', 'setup.py'])

	check_call(['git', 'commit'])
	check_call(['git', 'tag', '-m', 'Release ' + version_string, '-a', version_string])
	check_call(['git', 'push', 'upstream', 'master'])
	check_call(['git', 'push', 'upstream', version_string])
	check_call(['python', 'setup.py', 'sdist', 'upload'])
	check_call(['python', 'setup.py', 'upload_docs'])

if __name__ == '__main__':
	args = p.parse_args()
	create_release(args.version, args.rev)
<|endoftext|>


<commit_before>import logging

logger = logging.getLogger( __name__ )

class Job(object):
    """
    A class encapsulating state about a jobTree job including its child commands and follow-on commands.

    Note that a parent Job instance does not store its children as instances of the Job class but uses 3-tuples of
    the form (jobStoreId, memory, cpu) instead.
    """

    @staticmethod
    def create(command, memory, cpu, tryCount, jobStoreID, logJobStoreFileID):
        return Job(
            remainingRetryCount=tryCount,
            jobStoreID=jobStoreID,
            children=[],
            followOnCommands=[(command, memory, cpu, 0)],
            messages=[],
            logJobStoreFileID=logJobStoreFileID)

    def __init__(self, remainingRetryCount, jobStoreID, children, followOnCommands, messages, logJobStoreFileID):
        self.remainingRetryCount = remainingRetryCount
        self.jobStoreID = jobStoreID
        self.children = children
        self.followOnCommands = followOnCommands
        self.messages = messages
        self.logJobStoreFileID = logJobStoreFileID

    def setupJobAfterFailure(self, config):
        if len(self.followOnCommands) > 0:
            self.remainingRetryCount = max(0, self.remainingRetryCount - 1)
            logger.critical("Due to failure we are reducing the remaining retry \
            count of job %s to %s" % (self.jobStoreID, self.remainingRetryCount))
            # Set the default memory to be at least as large as the default, in
            # case this was a malloc failure (we do this because of the combined
            # batch system)
            self.followOnCommands[-1] = (self.followOnCommands[-1][0], \
                                         max(self.followOnCommands[-1][1],
                                             float(config.attrib["default_memory"]))) + \
                                        self.followOnCommands[-1][2:]
            logger.critical("We have set the default memory of the failed job to %s bytes" \
                            % self.followOnCommands[-1][1])
        else:
            logger.critical("The job %s has no follow on jobs to reset" % self.jobStoreID)

    def clearLogFile(self, jobStore):
        """Clears the log file, if it is set.
        """
        if self.logJobStoreFileID is not None:
            jobStore.deleteFile(self.logJobStoreFileID)
            self.logJobStoreFileID = None

    def setLogFile(self, logFile, jobStore):
        """Sets the log file in the file store.
        """
        if self.logJobStoreFileID is not None:  # File already exists
            jobStore.updateFile(self.logJobStoreFileID, logFile)
        else:
            self.logJobStoreFileID = jobStore.writeFile(self.jobStoreID, logFile)
            assert self.logJobStoreFileID is not None

    def getLogFileHandle(self, jobStore):
        """
        Returns a context manager that yields a file handle to the log file
        """
        return jobStore.readFileStream(self.logJobStoreFileID)

    # Serialization support methods

    def toList(self):
        """
        Deprecated. Use toDict() instead.
        """
        return [
            self.remainingRetryCount,
            self.jobStoreID,
            self.children,
            self.followOnCommands,
            self.messages,
            self.logJobStoreFileID]

    @staticmethod
    def fromList(l):
        """
        Deprecated. Use fromDict() instead.
        """
        return Job(*l)

    def toDict(self):
        return self.__dict__.copy()

    @staticmethod
    def fromDict(d):
        return Job(**d)<commit_msg>Added default parameters to Job constructor<commit_after>import logging

logger = logging.getLogger( __name__ )

class Job( object ):
    """
    A class encapsulating state about a jobTree job including its child commands and follow-on commands.

    Note that a parent Job instance does not store its children as instances of the Job class but uses 3-tuples of
    the form (jobStoreId, memory, cpu) instead.
    """

    @staticmethod
    def create( command, memory, cpu, tryCount, jobStoreID, logJobStoreFileID ):
        return Job(
            remainingRetryCount=tryCount,
            jobStoreID=jobStoreID,
            followOnCommands=[ (command, memory, cpu, 0) ],
            messages=[ ],
            logJobStoreFileID=logJobStoreFileID )

    def __init__( self, remainingRetryCount, jobStoreID,
                  children=None, followOnCommands=None, messages=None, logJobStoreFileID=None ):
        self.remainingRetryCount = remainingRetryCount
        self.jobStoreID = jobStoreID
        self.children = children or [ ]
        self.followOnCommands = followOnCommands or [ ]
        self.messages = messages or [ ]
        self.logJobStoreFileID = logJobStoreFileID

    def setupJobAfterFailure( self, config ):
        if len( self.followOnCommands ) > 0:
            self.remainingRetryCount = max( 0, self.remainingRetryCount - 1 )
            logger.critical( "Due to failure we are reducing the remaining retry \
            count of job %s to %s" % (self.jobStoreID, self.remainingRetryCount) )
            # Set the default memory to be at least as large as the default, in
            # case this was a malloc failure (we do this because of the combined
            # batch system)
            self.followOnCommands[ -1 ] = (self.followOnCommands[ -1 ][ 0 ], \
                max( self.followOnCommands[ -1 ][ 1 ],
                     float( config.attrib[ "default_memory" ] ) )) + \
                                          self.followOnCommands[ -1 ][ 2: ]
            logger.critical( "We have set the default memory of the failed job to %s bytes" \
                             % self.followOnCommands[ -1 ][ 1 ] )
        else:
            logger.critical( "The job %s has no follow on jobs to reset" % self.jobStoreID )

    def clearLogFile( self, jobStore ):
        """Clears the log file, if it is set.
        """
        if self.logJobStoreFileID is not None:
            jobStore.deleteFile( self.logJobStoreFileID )
            self.logJobStoreFileID = None

    def setLogFile( self, logFile, jobStore ):
        """Sets the log file in the file store.
        """
        if self.logJobStoreFileID is not None:  # File already exists
            jobStore.updateFile( self.logJobStoreFileID, logFile )
        else:
            self.logJobStoreFileID = jobStore.writeFile( self.jobStoreID, logFile )
            assert self.logJobStoreFileID is not None

    def getLogFileHandle( self, jobStore ):
        """
        Returns a context manager that yields a file handle to the log file
        """
        return jobStore.readFileStream( self.logJobStoreFileID )

    # Serialization support methods

    def toList( self ):
        """
        Deprecated. Use toDict() instead.
        """
        return [
            self.remainingRetryCount,
            self.jobStoreID,
            self.children,
            self.followOnCommands,
            self.messages,
            self.logJobStoreFileID ]

    @staticmethod
    def fromList( l ):
        """
        Deprecated. Use fromDict() instead.
        """
        return Job( *l )

    def toDict( self ):
        return self.__dict__.copy( )

    @staticmethod
    def fromDict( d ):
        return Job( **d )

    def __hash__( self ):
        return hash( self.jobStoreID )

    def __eq__( self, other ):
        return isinstance( other, self.__class__ ) and self.__dict__ == other.__dict__

    def __ne__( self, other ):
        return not self.__eq__( other )

    def __repr__( self ):
        return 'Job( **%r )' % self.__dict__
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import datetime, random, subprocess, sys

host = 'greatwave'

uniqkey = '{}-{:08}'.format(datetime.datetime.now().strftime('%Y-%m-%d/%H-%M-%S'), random.randint(0,99999999))
tmpdir = 'work/' + uniqkey
#tmpdir = 'work/experimental'
destdir = '/home/nushio/work/' + uniqkey

srcfiles = ['3d-mhd.h', '3d-mhd*.c', '3d-mhd-main-prof.cpp']
srcdir = 'examples/'
srcpaths = [srcdir + fn for fn in srcfiles]

def cmd(str):
    print(str, file=sys.stderr)
    subprocess.call(str, shell=True)

def on_k(str):
    cmd("ssh {} '(cd {}; {})'".format(host, destdir,str))

#cmd('rm {}/3d-mhd_internal_*.c'.format(srcdir))
#cmd('stack exec formura -- {}/3d-mhd.fmr'.format(srcdir))
cmd('mkdir -p {}'.format(tmpdir))
cmd('ssh {} mkdir -p {}'.format(host,destdir))
cmd('cp {} {}'.format(' '.join(srcpaths),tmpdir))
cmd('cp cmake-for-k.sh {}/cmake.sh'.format(tmpdir))
cmd('cp CMakeLists-for-k.txt {}/CMakeLists.txt'.format(tmpdir))

submit_script_path = '{}/submit.sh'.format(tmpdir)

with(open(submit_script_path,'w')) as fp:
    fp.write("""
#!/bin/sh -x
#PJM --rsc-list "node=8"

#time limit
#PJM --rsc-list "elapse=12:00:00"
#PJM --rsc-list "rscgrp=small"
# #PJM --rsc-list "rscgrp=large"
#PJM --mpi "use-rankdir"
#PJM --stg-transfiles all

# stage in  hello.out.
#PJM --stgin "./a.out %r:./a.out"
#PJM --stgout "%r:./out/* ./out-%r-%j/"

#statistics output
#PJM -s

# config environmental variables
. /work/system/Env_base
mpiexec /work/system/bin/msh "mkdir ./out"

mpirun -n 8 fapp -C -d out/prof-01 -Hpa=1 ./a.out
mpirun -n 8 fapp -C -d out/prof-02 -Hpa=2 ./a.out
mpirun -n 8 fapp -C -d out/prof-03 -Hpa=3 ./a.out
mpirun -n 8 fapp -C -d out/prof-04 -Hpa=4 ./a.out
mpirun -n 8 fapp -C -d out/prof-05 -Hpa=5 ./a.out
mpirun -n 8 fapp -C -d out/prof-06 -Hpa=6 ./a.out
mpirun -n 8 fapp -C -d out/prof-07 -Hpa=7 ./a.out
mpirun -n 8 fapp -C -d out/prof-08 -Hpa=8 ./a.out
mpirun -n 8 fapp -C -d out/prof-09 -Hpa=9 ./a.out
mpirun -n 8 fapp -C -d out/prof-10 -Hpa=10 ./a.out
mpirun -n 8 fapp -C -d out/prof-11 -Hpa=11 ./a.out

fapppx -A -p all -l0 -d out/prof-01 -o out/prof-1.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-02 -o out/prof-2.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-03 -o out/prof-3.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-04 -o out/prof-4.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-05 -o out/prof-5.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-06 -o out/prof-6.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-07 -o out/prof-7.csv -tcsv -Hpa

""")
cmd('chmod 755 '+submit_script_path)

cmd('scp {}/*  {}:{}'.format(tmpdir, host,destdir))
on_k('mpiFCCpx 3d-mhd*.c 3d-mhd-main-prof.cpp')
#on_k('mpiFCCpx 3d-mhd*.c 3d-mhd-main-prof.cpp -o a.out -O3 -Kfast,parallel -Kocl -Klib -Koptmsg=2 -Karray_private -Kinstance=8 -Kdynamic_iteration -Kloop_fission -Kloop_part_parallel -Kloop_part_simd -Keval  -Kreduction -Ksimd=2')
on_k('pjsub submit.sh')
<commit_msg>Do at least some optimization<commit_after>#!/usr/bin/env python3

import datetime, random, subprocess, sys

host = 'greatwave'

uniqkey = '{}-{:08}'.format(datetime.datetime.now().strftime('%Y-%m-%d/%H-%M-%S'), random.randint(0,99999999))
tmpdir = 'work/' + uniqkey
#tmpdir = 'work/experimental'
destdir = '/home/nushio/work/' + uniqkey

srcfiles = ['3d-mhd.h', '3d-mhd*.c', '3d-mhd-main-prof.cpp']
srcdir = 'examples/'
srcpaths = [srcdir + fn for fn in srcfiles]

def cmd(str):
    print(str, file=sys.stderr)
    subprocess.call(str, shell=True)

def on_k(str):
    cmd("ssh {} '(cd {}; {})'".format(host, destdir,str))

#cmd('rm {}/3d-mhd_internal_*.c'.format(srcdir))
#cmd('stack exec formura -- {}/3d-mhd.fmr'.format(srcdir))
cmd('mkdir -p {}'.format(tmpdir))
cmd('ssh {} mkdir -p {}'.format(host,destdir))
cmd('cp {} {}'.format(' '.join(srcpaths),tmpdir))
cmd('cp cmake-for-k.sh {}/cmake.sh'.format(tmpdir))
cmd('cp CMakeLists-for-k.txt {}/CMakeLists.txt'.format(tmpdir))

submit_script_path = '{}/submit.sh'.format(tmpdir)

with(open(submit_script_path,'w')) as fp:
    fp.write("""
#!/bin/sh -x
#PJM --rsc-list "node=8"

#time limit
#PJM --rsc-list "elapse=12:00:00"
#PJM --rsc-list "rscgrp=small"
# #PJM --rsc-list "rscgrp=large"
#PJM --mpi "use-rankdir"
#PJM --stg-transfiles all

# stage in  hello.out.
#PJM --stgin "./a.out %r:./a.out"
#PJM --stgout "%r:./out/* ./out-%r-%j/"

#statistics output
#PJM -s

# config environmental variables
. /work/system/Env_base
mpiexec /work/system/bin/msh "mkdir ./out"

mpirun -n 8 fapp -C -d out/prof-01 -Hpa=1 ./a.out
mpirun -n 8 fapp -C -d out/prof-02 -Hpa=2 ./a.out
mpirun -n 8 fapp -C -d out/prof-03 -Hpa=3 ./a.out
mpirun -n 8 fapp -C -d out/prof-04 -Hpa=4 ./a.out
mpirun -n 8 fapp -C -d out/prof-05 -Hpa=5 ./a.out
mpirun -n 8 fapp -C -d out/prof-06 -Hpa=6 ./a.out
mpirun -n 8 fapp -C -d out/prof-07 -Hpa=7 ./a.out
mpirun -n 8 fapp -C -d out/prof-08 -Hpa=8 ./a.out
mpirun -n 8 fapp -C -d out/prof-09 -Hpa=9 ./a.out
mpirun -n 8 fapp -C -d out/prof-10 -Hpa=10 ./a.out
mpirun -n 8 fapp -C -d out/prof-11 -Hpa=11 ./a.out

fapppx -A -p all -l0 -d out/prof-01 -o out/prof-1.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-02 -o out/prof-2.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-03 -o out/prof-3.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-04 -o out/prof-4.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-05 -o out/prof-5.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-06 -o out/prof-6.csv -tcsv -Hpa
fapppx -A -p all -l0 -d out/prof-07 -o out/prof-7.csv -tcsv -Hpa

""")
cmd('chmod 755 '+submit_script_path)

cmd('scp {}/*  {}:{}'.format(tmpdir, host,destdir))
on_k('mpiFCCpx -O1 3d-mhd*.c 3d-mhd-main-prof.cpp')
#on_k('mpiFCCpx 3d-mhd*.c 3d-mhd-main-prof.cpp -o a.out -O3 -Kfast,parallel -Kocl -Klib -Koptmsg=2 -Karray_private -Kinstance=8 -Kdynamic_iteration -Kloop_fission -Kloop_part_parallel -Kloop_part_simd -Keval  -Kreduction -Ksimd=2')
on_k('pjsub submit.sh')
<|endoftext|>


<commit_before>#!/usr/bin/env python3

	#####################################################
	##    WISCONSIN BREAST CANCER MACHINE LEARNING     ##
	#####################################################

# Project by Raul Eulogio

# Project found at: https://www.inertia7.com/projects/3

"""
Model Evaluation
"""

import matplotlib.pyplot as plt
from helper_functions import test_class_set, test_class_set_scaled, print_class_report
import random_forest as rf
import knn
import neural_networks as nn
from terminaltables import AsciiTable
from sklearn.metrics import classification_report

# Creating useful dicts or lists
pred_dict = {
	'predictions': 'Kth Nearest Neighbor',
	'predictions_rf': 'Random Forest',
	'predictions_nn': 'Neural Networks'
	}

# Calling up metrics from the model scripts
fpr, tpr, auc_knn, predictions, test_error_rate = knn.return_knn()

fpr2, tpr2, auc_rf, predictions_rf, test_error_rate_rf = rf.return_rf()

fpr3, tpr3, auc_nn, predictions_nn, test_error_rate_nn =nn.return_nn()

if __name__ == '__main__':
	# Populate list for human readable table from terminal line
	table_data = [
		[ 'Model/Algorithm', 'Test Error Rate',
		'False Negative for Test Set', 'Area under the Curve for ROC',
		'Cross Validation Score'],
		['Kth Nearest Neighbor',  round(test_error_rate, 3), 5,
		round(auc_knn, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(knn.mean_cv_knn, knn.std_error_knn)
		],
		[ 'Random Forest', round(test_error_rate_rf, 3), 3,
		round(auc_rf, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(rf.mean_cv_rf, rf.std_error_rf)
		],
		[ 'Neural Networks' ,  round(test_error_rate_nn, 3),  1,
		round(auc_nn, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(nn.mean_cv_nn, nn.std_error_nn)
		]]

	# convert to AsciiTable from terminaltables package
	table = AsciiTable(table_data)

	target_names = ['Benign', 'Malignant']

	for key, value in pred_dict.items():
		hf.print_class_report(key, value)

	print("Comparison of different logistics relating to model evaluation:")
	print(table.table)

	# Plotting ROC Curves
	f, ax = plt.subplots(figsize=(10, 10))

	plt.plot(fpr, tpr, label='Kth Nearest Neighbor ROC Curve (area = {0: .3f})'\
		.format(auc_knn),
         	color = 'deeppink',
         	linewidth=1)
	plt.plot(fpr2, tpr2,label='Random Forest ROC Curve (area = {0: .3f})'\
		.format(auc_rf),
         	color = 'red',
         	linestyle=':',
         	linewidth=2)
	plt.plot(fpr3, tpr3,label='Neural Networks ROC Curve (area = {0: .3f})'\
		.format(auc_nn),
         	color = 'purple',
         	linestyle=':',
         	linewidth=3)

	ax.set_axis_bgcolor('#fafafa')
	plt.plot([0, 1], [0, 1], 'k--', lw=2)
	plt.plot([0, 0], [1, 0], 'k--', lw=2, color = 'black')
	plt.plot([1, 0], [1, 1], 'k--', lw=2, color = 'black')
	plt.xlim([-0.01, 1.0])
	plt.ylim([0.0, 1.05])
	plt.xlabel('False Positive Rate')
	plt.ylabel('True Positive Rate')
	plt.title('ROC Curve Comparison For All Models')
	plt.legend(loc="lower right")
	plt.show()

	# Zoomed in
	f, ax = plt.subplots(figsize=(10, 10))
	plt.plot(fpr, tpr, label='Kth Nearest Neighbor ROC Curve  (area = {0: .3f})'\
		.format(auc_knn),
         	color = 'deeppink',
         	linewidth=1)
	plt.plot(fpr2, tpr2,label='Random Forest ROC Curve  (area = {0: .3f})'\
		.format(auc_rf),
         	color = 'red',
         	linestyle=':',
         	linewidth=3)
	plt.plot(fpr3, tpr3,label='Neural Networks ROC Curve  (area = {0: .3f})'\
		.format(auc_nn),
         	color = 'purple',
         	linestyle=':',
         	linewidth=3)

	ax.set_axis_bgcolor('#fafafa')
	plt.plot([0, 1], [0, 1], 'k--', lw=2) # Add Diagonal line
	plt.plot([0, 0], [1, 0], 'k--', lw=2, color = 'black')
	plt.plot([1, 0], [1, 1], 'k--', lw=2, color = 'black')
	plt.xlim([-0.001, 0.2])
	plt.ylim([0.7, 1.05])
	plt.xlabel('False Positive Rate')
	plt.ylabel('True Positive Rate')
	plt.title('ROC Curve Comparison For All Models (Zoomed)')
	plt.legend(loc="lower right")
	plt.show()

	print('fin \n:)')
<commit_msg>Update model_eval.py<commit_after>#!/usr/bin/env python3

	#####################################################
	##    WISCONSIN BREAST CANCER MACHINE LEARNING     ##
	#####################################################

# Project by Raul Eulogio

# Project found at: https://www.inertia7.com/projects/3

"""
Model Evaluation
"""

import matplotlib.pyplot as plt
from helper_functions import test_class_set, print_class_report
import random_forest as rf
import knn
import neural_networks as nn
from terminaltables import AsciiTable
from sklearn.metrics import classification_report

# Creating useful dicts or lists
pred_dict = {
	'predictions': 'Kth Nearest Neighbor',
	'predictions_rf': 'Random Forest',
	'predictions_nn': 'Neural Networks'
	}

# Calling up metrics from the model scripts
fpr, tpr, auc_knn, predictions, test_error_rate = knn.return_knn()

fpr2, tpr2, auc_rf, predictions_rf, test_error_rate_rf = rf.return_rf()

fpr3, tpr3, auc_nn, predictions_nn, test_error_rate_nn =nn.return_nn()

if __name__ == '__main__':
	# Populate list for human readable table from terminal line
	table_data = [
		[ 'Model/Algorithm', 'Test Error Rate',
		'False Negative for Test Set', 'Area under the Curve for ROC',
		'Cross Validation Score'],
		['Kth Nearest Neighbor',  round(test_error_rate, 3), 5,
		round(auc_knn, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(knn.mean_cv_knn, knn.std_error_knn)
		],
		[ 'Random Forest', round(test_error_rate_rf, 3), 3,
		round(auc_rf, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(rf.mean_cv_rf, rf.std_error_rf)
		],
		[ 'Neural Networks' ,  round(test_error_rate_nn, 3),  1,
		round(auc_nn, 3), "Accuracy: {0: 0.3f} (+/- {1: 0.3f})"\
				.format(nn.mean_cv_nn, nn.std_error_nn)
		]]

	# convert to AsciiTable from terminaltables package
	table = AsciiTable(table_data)

	target_names = ['Benign', 'Malignant']

	for key, value in pred_dict.items():
		hf.print_class_report(key, value)

	print("Comparison of different logistics relating to model evaluation:")
	print(table.table)

	# Plotting ROC Curves
	f, ax = plt.subplots(figsize=(10, 10))

	plt.plot(fpr, tpr, label='Kth Nearest Neighbor ROC Curve (area = {0: .3f})'\
		.format(auc_knn),
         	color = 'deeppink',
         	linewidth=1)
	plt.plot(fpr2, tpr2,label='Random Forest ROC Curve (area = {0: .3f})'\
		.format(auc_rf),
         	color = 'red',
         	linestyle=':',
         	linewidth=2)
	plt.plot(fpr3, tpr3,label='Neural Networks ROC Curve (area = {0: .3f})'\
		.format(auc_nn),
         	color = 'purple',
         	linestyle=':',
         	linewidth=3)

	ax.set_axis_bgcolor('#fafafa')
	plt.plot([0, 1], [0, 1], 'k--', lw=2)
	plt.plot([0, 0], [1, 0], 'k--', lw=2, color = 'black')
	plt.plot([1, 0], [1, 1], 'k--', lw=2, color = 'black')
	plt.xlim([-0.01, 1.0])
	plt.ylim([0.0, 1.05])
	plt.xlabel('False Positive Rate')
	plt.ylabel('True Positive Rate')
	plt.title('ROC Curve Comparison For All Models')
	plt.legend(loc="lower right")
	plt.show()

	# Zoomed in
	f, ax = plt.subplots(figsize=(10, 10))
	plt.plot(fpr, tpr, label='Kth Nearest Neighbor ROC Curve  (area = {0: .3f})'\
		.format(auc_knn),
         	color = 'deeppink',
         	linewidth=1)
	plt.plot(fpr2, tpr2,label='Random Forest ROC Curve  (area = {0: .3f})'\
		.format(auc_rf),
         	color = 'red',
         	linestyle=':',
         	linewidth=3)
	plt.plot(fpr3, tpr3,label='Neural Networks ROC Curve  (area = {0: .3f})'\
		.format(auc_nn),
         	color = 'purple',
         	linestyle=':',
         	linewidth=3)

	ax.set_axis_bgcolor('#fafafa')
	plt.plot([0, 1], [0, 1], 'k--', lw=2) # Add Diagonal line
	plt.plot([0, 0], [1, 0], 'k--', lw=2, color = 'black')
	plt.plot([1, 0], [1, 1], 'k--', lw=2, color = 'black')
	plt.xlim([-0.001, 0.2])
	plt.ylim([0.7, 1.05])
	plt.xlabel('False Positive Rate')
	plt.ylabel('True Positive Rate')
	plt.title('ROC Curve Comparison For All Models (Zoomed)')
	plt.legend(loc="lower right")
	plt.show()

	print('fin \n:)')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import hashlib
import socket

import six

from wechatpy.utils import to_binary, to_text

def format_url(params, api_key=None):
    data = [to_binary('{0}={1}'.format(k, params[k])) for k in sorted(params)]
    if api_key:
        data.append(to_binary('key={0}'.format(api_key)))
    return b"&".join(data)

def calculate_signature(params, api_key):
    url = format_url(params, api_key)
    return to_text(hashlib.md5(url).hexdigest().upper())

def dict_to_xml(d, sign):
    xml = ['<xml>\n']
    for k in sorted(d):
        # use sorted to avoid test error on Py3k
        v = d[k]
        if isinstance(v, six.integer_types) or v.isdigit():
            xml.append('<{0}>{1}</{0}>\n'.format(to_text(k), to_text(v)))
        else:
            xml.append(
                '<{0}><![CDATA[{1}]]></{0}>\n'.format(to_text(k), to_text(v))
            )
    xml.append('<sign><![CDATA[{0}]]></sign>\n</xml>'.format(to_text(sign)))
    return ''.join(xml)

def get_external_ip():
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        wechat_ip = socket.gethostbyname('api.mch.weixin.qq.com')
        sock.connect((wechat_ip, 80))
        addr, port = sock.getsockname()
        sock.close()
        return addr
    except socket.error:
        return '127.0.0.1'
<commit_msg>Fix Bug: format_url error when some params' value is empty. See: https://pay.weixin.qq.com/wiki/doc/api/app/app.php?chapter=4_3<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
import hashlib
import socket

import six

from wechatpy.utils import to_binary, to_text

def format_url(params, api_key=None):
    data = [to_binary('{0}={1}'.format(k, params[k])) for k in sorted(params) if params[k]]
    if api_key:
        data.append(to_binary('key={0}'.format(api_key)))
    return b"&".join(data)

def calculate_signature(params, api_key):
    url = format_url(params, api_key)
    return to_text(hashlib.md5(url).hexdigest().upper())

def dict_to_xml(d, sign):
    xml = ['<xml>\n']
    for k in sorted(d):
        # use sorted to avoid test error on Py3k
        v = d[k]
        if isinstance(v, six.integer_types) or v.isdigit():
            xml.append('<{0}>{1}</{0}>\n'.format(to_text(k), to_text(v)))
        else:
            xml.append(
                '<{0}><![CDATA[{1}]]></{0}>\n'.format(to_text(k), to_text(v))
            )
    xml.append('<sign><![CDATA[{0}]]></sign>\n</xml>'.format(to_text(sign)))
    return ''.join(xml)

def get_external_ip():
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        wechat_ip = socket.gethostbyname('api.mch.weixin.qq.com')
        sock.connect((wechat_ip, 80))
        addr, port = sock.getsockname()
        sock.close()
        return addr
    except socket.error:
        return '127.0.0.1'
<|endoftext|>


<commit_before>import sys, os, click, MySQLdb, csv

''' Connect to DB '''
db = MySQLdb.connect(host="localhost",
                     user="root",
                     db="dataviva2")
db.autocommit(1)
cursor = db.cursor()

def frange(x, y, jump):
    while x < y:
        yield x
        x += jump

@click.command()
@click.option('output_path', '--output', '-o', help='Path to save files to.', type=click.Path(), required=True, prompt="Output path")
def main(output_path):
    cursor.execute("select distinct cnae_id from rais_ii where length(cnae_id) = 6;")
    cnaes = [r[0] for r in cursor.fetchall()]

    edges = set([])
    for cnae in cnaes:
        nada = True
        cutoffs = list(frange(0.4, 0.75, 0.01))
        cutoffs.reverse()
        # for cutoff in [0.8, 0.75, 0.725, 0.7, 0.675, 0.65, 0.6, 0.55, 0.5]:
        for cutoff in cutoffs:
            cursor.execute("select * from rais_ii where cnae_id = %s and proximity < 1 and proximity >= %s;", [cnae, cutoff])
            conns = [list(c) for c in cursor.fetchall()]
            if len(conns) >= 3:
                # print c, len(conns)
                for c in conns:
                    c.sort(reverse=True)
                    edges.add(tuple(c))
                # edges += conns
                nothing = False
                break
            # if nothing:

    # print len(edges)

    with open(os.path.join(output_path,'cnae_network.tsv'), 'w') as fp:
        a = csv.writer(fp, delimiter='\t')
        a.writerow(('cnae_source', 'cnae_target', 'proximity'))
        a.writerows(edges)

if __name__ == "__main__":
    main()<commit_msg>update thresholds for cnae space<commit_after>import sys, os, click, MySQLdb, csv

''' Connect to DB '''
db = MySQLdb.connect(host="localhost",
                     user="root",
                     db="dataviva2")
db.autocommit(1)
cursor = db.cursor()

def frange(x, y, jump):
    while x < y:
        yield x
        x += jump

@click.command()
@click.option('output_path', '--output', '-o', help='Path to save files to.', type=click.Path(), required=True, prompt="Output path")
def main(output_path):
    cursor.execute("select distinct cnae_id from rais_ii where length(cnae_id) = 6;")
    cnaes = [r[0] for r in cursor.fetchall()]

    edges = set([])
    for cnae in cnaes:
        nada = True
        cutoffs = list(frange(0.5, 0.81, 0.01))
        cutoffs.reverse()
        # for cutoff in [0.8, 0.75, 0.725, 0.7, 0.675, 0.65, 0.6, 0.55, 0.5]:
        for cutoff in cutoffs:
            cursor.execute("select * from rais_ii where cnae_id = %s and proximity < 1 and proximity >= %s;", [cnae, cutoff])
            conns = [list(c) for c in cursor.fetchall()]
            if len(conns) >= 3:
                # print c, len(conns)
                for c in conns:
                    c.sort(reverse=True)
                    edges.add(tuple(c))
                # edges += conns
                nothing = False
                break
            # if nothing:

    # print len(edges)

    with open(os.path.join(output_path,'cnae_network.tsv'), 'w') as fp:
        a = csv.writer(fp, delimiter='\t')
        a.writerow(('cnae_source', 'cnae_target', 'proximity'))
        a.writerows(edges)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>import json
import re
import os

from flask import abort
from jsonschema import ValidationError, FormatChecker
from jsonschema.validators import validator_for

MINIMUM_SERVICE_ID_LENGTH = 10
MAXIMUM_SERVICE_ID_LENGTH = 20

JSON_SCHEMAS_PATH = './json_schemas'
SCHEMA_NAMES = [
    'services-g4',
    'services-g5',
    'services-g6-scs',
    'services-g6-saas',
    'services-g6-paas',
    'services-g6-iaas',
    'services-update',
    'users',
    'users-auth',
]
FORMAT_CHECKER = FormatChecker()

def load_schemas(schemas_path, schema_names):
    loaded_schemas = {}
    for schema_name in schema_names:
        schema_path = os.path.join(schemas_path, '{}.json'.format(schema_name))

        with open(schema_path) as f:
            schema = json.load(f)
            validator = validator_for(schema)
            validator.check_schema(schema)
            loaded_schemas[schema_name] = validator(
                schema, format_checker=FORMAT_CHECKER)
    return loaded_schemas

SCHEMAS = load_schemas(JSON_SCHEMAS_PATH, SCHEMA_NAMES)

def validate_updater_json_or_400(submitted_json):
    if not validates_against_schema('services-update', submitted_json):
        abort(400, "JSON was not a valid format")

def validate_user_json_or_400(submitted_json):
    if not validates_against_schema('users', submitted_json):
        abort(400, "JSON was not a valid format")

def validate_user_auth_json_or_400(submitted_json):
    try:
        validates_against_schema('users-auth', submitted_json)
    except ValidationError as e:
        abort(400, "JSON was not a valid format. {}".format(e.message))

def detect_framework_or_400(submitted_json):
    framework = detect_framework(submitted_json)
    if not framework:
        abort(400, "JSON was not a valid format. {}".format(
            reason_for_failure(submitted_json))
        )
    return framework

def detect_framework(submitted_json):
    if validates_against_schema('services-g4', submitted_json):
        return 'G-Cloud 4'
    elif validates_against_schema('services-g5', submitted_json):
        return 'G-Cloud 5'
    elif validates_against_schema('services-g6-scs', submitted_json) or \
            validates_against_schema('services-g6-saas', submitted_json) or \
            validates_against_schema('services-g6-paas', submitted_json) or \
            validates_against_schema('services-g6-iaas', submitted_json):
        return 'G-Cloud 6'
    else:
        return False

def validates_against_schema(validator_name, submitted_json):
    try:
        validator = SCHEMAS[validator_name]
        validator.validate(submitted_json)
    except ValidationError:
        return False
    else:
        return True

def reason_for_failure(submitted_json):
    response = []
    try:
        SCHEMAS['services-g6-scs'].validate(submitted_json)
    except ValidationError as e1:
        response.append('Not SCS: %s' % e1.message)

    try:
        SCHEMAS['services-g6-saas'].validate(submitted_json)
    except ValidationError as e2:
        response.append('Not SaaS: %s' % e2.message)

    try:
        SCHEMAS['services-g6-paas'].validate(submitted_json)
    except ValidationError as e3:
        response.append('Not PaaS: %s' % e3.message)

    try:
        SCHEMAS['services-g6-iaas'].validate(submitted_json)
    except ValidationError as e4:
        response.append('Not IaaS: %s' % e4.message)

    return '. '.join(response)

def is_valid_service_id(service_id):
    """
    Validate that service ids contain only letters
    numbers and dashes. [A-z0-9-]
    :param service_id:
    :return True|False:
    """

    if len(service_id) > MAXIMUM_SERVICE_ID_LENGTH or \
            len(service_id) < MINIMUM_SERVICE_ID_LENGTH or \
            re.search(r"[^A-z0-9-]", service_id):
        return False
    return True

def is_valid_service_id_or_400(service_id):
    if is_valid_service_id(service_id):
        return True
    else:
        abort(400, "Invalid service ID supplied: %s" % service_id)
<commit_msg>Small optimization to regex that validates `service_id`s<commit_after>import json
import re
import os

from flask import abort
from jsonschema import ValidationError, FormatChecker
from jsonschema.validators import validator_for

MINIMUM_SERVICE_ID_LENGTH = 10
MAXIMUM_SERVICE_ID_LENGTH = 20

JSON_SCHEMAS_PATH = './json_schemas'
SCHEMA_NAMES = [
    'services-g4',
    'services-g5',
    'services-g6-scs',
    'services-g6-saas',
    'services-g6-paas',
    'services-g6-iaas',
    'services-update',
    'users',
    'users-auth',
]
FORMAT_CHECKER = FormatChecker()

def load_schemas(schemas_path, schema_names):
    loaded_schemas = {}
    for schema_name in schema_names:
        schema_path = os.path.join(schemas_path, '{}.json'.format(schema_name))

        with open(schema_path) as f:
            schema = json.load(f)
            validator = validator_for(schema)
            validator.check_schema(schema)
            loaded_schemas[schema_name] = validator(
                schema, format_checker=FORMAT_CHECKER)
    return loaded_schemas

SCHEMAS = load_schemas(JSON_SCHEMAS_PATH, SCHEMA_NAMES)

def validate_updater_json_or_400(submitted_json):
    if not validates_against_schema('services-update', submitted_json):
        abort(400, "JSON was not a valid format")

def validate_user_json_or_400(submitted_json):
    if not validates_against_schema('users', submitted_json):
        abort(400, "JSON was not a valid format")

def validate_user_auth_json_or_400(submitted_json):
    try:
        validates_against_schema('users-auth', submitted_json)
    except ValidationError as e:
        abort(400, "JSON was not a valid format. {}".format(e.message))

def detect_framework_or_400(submitted_json):
    framework = detect_framework(submitted_json)
    if not framework:
        abort(400, "JSON was not a valid format. {}".format(
            reason_for_failure(submitted_json))
        )
    return framework

def detect_framework(submitted_json):
    if validates_against_schema('services-g4', submitted_json):
        return 'G-Cloud 4'
    elif validates_against_schema('services-g5', submitted_json):
        return 'G-Cloud 5'
    elif validates_against_schema('services-g6-scs', submitted_json) or \
            validates_against_schema('services-g6-saas', submitted_json) or \
            validates_against_schema('services-g6-paas', submitted_json) or \
            validates_against_schema('services-g6-iaas', submitted_json):
        return 'G-Cloud 6'
    else:
        return False

def validates_against_schema(validator_name, submitted_json):
    try:
        validator = SCHEMAS[validator_name]
        validator.validate(submitted_json)
    except ValidationError:
        return False
    else:
        return True

def reason_for_failure(submitted_json):
    response = []
    try:
        SCHEMAS['services-g6-scs'].validate(submitted_json)
    except ValidationError as e1:
        response.append('Not SCS: %s' % e1.message)

    try:
        SCHEMAS['services-g6-saas'].validate(submitted_json)
    except ValidationError as e2:
        response.append('Not SaaS: %s' % e2.message)

    try:
        SCHEMAS['services-g6-paas'].validate(submitted_json)
    except ValidationError as e3:
        response.append('Not PaaS: %s' % e3.message)

    try:
        SCHEMAS['services-g6-iaas'].validate(submitted_json)
    except ValidationError as e4:
        response.append('Not IaaS: %s' % e4.message)

    return '. '.join(response)

def is_valid_service_id(service_id):
    """
    Validate that service ids contain only letters
    numbers and dashes ([A-z0-9-]) and that they're
    less than | equal to `MINIMUM_SERVICE_ID_LENGTH`
    greater than | equal to `MAXIMUM_SERVICE_ID_LENGTH`
    :param service_id:
    :return True|False:
    """

    regex_match_valid_service_id = r"^[A-z0-9-]{%s,%s}$" % (
        MINIMUM_SERVICE_ID_LENGTH,
        MAXIMUM_SERVICE_ID_LENGTH
    )

    if re.search(regex_match_valid_service_id, service_id):
        return True

    return False<|endoftext|>


<commit_before>import datetime
from flask import abort, Blueprint, redirect, request, render_template, url_for
from flask_login import current_user, login_required
from playhouse.flask_utils import get_object_or_404
from werkzeug.datastructures import CombinedMultiDict
from app import app, images
from app.forms import EditPostForm
from app.models import Post

mod = Blueprint('blog', __name__)

def url_for_other_page(page):
    args = request.view_args.copy()
    args['page'] = page
    return url_for(request.endpoint, **args)

app.jinja_env.globals['url_for_other_page'] = url_for_other_page
app.jinja_env.globals['image_url'] = images.url

@mod.route('/', defaults={'page': 1})
@mod.route('/page/<int:page>/')
def overview(page):
    if current_user.is_authenticated:
        posts = Post.select().order_by(Post.timestamp.desc())
    else:
        posts = Post.select().where(Post.published == True
                                    ).order_by(Post.timestamp.desc())

    pagination = posts.paginate(page, 5)

    if not pagination and posts:
        last_page = len(posts) // 5
        if len(posts) % 5:
            last_page += 1
        return redirect(url_for('.overview', page=last_page))

    has_next = True if posts.paginate(page+1, 5) else False

    return render_template('blog/overview.html',
                           pagination=pagination,
                           page=page,
                           has_next=has_next)

@mod.route('/new-post/', methods=['GET', 'POST'])
@login_required
def new_post():
    form = EditPostForm(CombinedMultiDict((request.form, request.files)))
    if form.validate_on_submit():
        post = Post.create(
                title=form.title.data,
                content=form.content.data,
                published=form.published.data,
                timestamp=datetime.datetime.now(),
                author=current_user.id,
                image=images.save(form.upload.data)
                )
        return redirect(post.slug)

    return render_template('blog/edit-post.html', form=form)

@mod.route('/<slug>/')
def view_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)

    if not post.published and not current_user.is_authenticated:
        return abort(404)

    return render_template('blog/view-post.html', post=post)

@mod.route('/<slug>/edit/', methods=['GET', 'POST'])
@login_required
def edit_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)
    form = EditPostForm(CombinedMultiDict((request.form, request.files)), post)

    if form.validate_on_submit():
        if form.upload.has_file():
            post.image = images.save(form.upload.data)
        post.title = form.title.data
        post.content = form.content.data
        post.published = form.published.data
        post.save()
        return redirect(post.slug)

    return render_template('blog/edit-post.html', form=form)

@mod.route('/<slug>/remove/', methods=['GET'])
@login_required
def remove_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)
    post.delete_instance()
    return redirect(url_for('blog.overview'))
<commit_msg>Fix new post post if no image uploaded<commit_after>import datetime
from flask import abort, Blueprint, redirect, request, render_template, url_for
from flask_login import current_user, login_required
from playhouse.flask_utils import get_object_or_404
from werkzeug.datastructures import CombinedMultiDict
from app import app, images
from app.forms import EditPostForm
from app.models import Post

mod = Blueprint('blog', __name__)

def url_for_other_page(page):
    args = request.view_args.copy()
    args['page'] = page
    return url_for(request.endpoint, **args)

app.jinja_env.globals['url_for_other_page'] = url_for_other_page
app.jinja_env.globals['image_url'] = images.url

@mod.route('/', defaults={'page': 1})
@mod.route('/page/<int:page>/')
def overview(page):
    if current_user.is_authenticated:
        posts = Post.select().order_by(Post.timestamp.desc())
    else:
        posts = Post.select().where(Post.published == True
                                    ).order_by(Post.timestamp.desc())

    pagination = posts.paginate(page, 5)

    if not pagination and posts:
        last_page = len(posts) // 5
        if len(posts) % 5:
            last_page += 1
        return redirect(url_for('.overview', page=last_page))

    has_next = True if posts.paginate(page+1, 5) else False

    return render_template('blog/overview.html',
                           pagination=pagination,
                           page=page,
                           has_next=has_next)

@mod.route('/new-post/', methods=['GET', 'POST'])
@login_required
def new_post():
    form = EditPostForm(CombinedMultiDict((request.form, request.files)))
    if form.validate_on_submit():
        if form.upload.has_file():
            image = images.save(form.upload.data)
        else:
            image = None

        post = Post.create(
                title=form.title.data,
                content=form.content.data,
                published=form.published.data,
                timestamp=datetime.datetime.now(),
                author=current_user.id,
                image=image
                )
        return redirect(post.slug)

    return render_template('blog/edit-post.html', form=form)

@mod.route('/<slug>/')
def view_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)

    if not post.published and not current_user.is_authenticated:
        return abort(404)

    return render_template('blog/view-post.html', post=post)

@mod.route('/<slug>/edit/', methods=['GET', 'POST'])
@login_required
def edit_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)
    form = EditPostForm(CombinedMultiDict((request.form, request.files)), post)

    if form.validate_on_submit():
        if form.upload.has_file():
            post.image = images.save(form.upload.data)
        post.title = form.title.data
        post.content = form.content.data
        post.published = form.published.data
        post.save()
        return redirect(post.slug)

    return render_template('blog/edit-post.html', form=form)

@mod.route('/<slug>/remove/', methods=['GET'])
@login_required
def remove_post(slug):
    post = get_object_or_404(Post, Post.slug == slug)
    post.delete_instance()
    return redirect(url_for('blog.overview'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import json
from requests import post
import logging
import boto3

from sm.engine.util import SMConfig, sm_log_formatters, sm_log_config, init_logger
from sm.engine.search_job import SearchJob
from sm.engine.queue import QueueConsumer
from sm.engine.db import DB

def configure_loggers():
    log_config = sm_log_config
    log_config['loggers']['sm-engine']['handlers'] = ['console_warn', 'file']
    init_logger(log_config)

def post_to_slack(emoji, msg):
    slack_conf = SMConfig.get_conf()['slack']

    if slack_conf['webhook_url']:
        m = {"channel": slack_conf['channel'],
             "username": "webhookbot",
             "text": ":{}:{}".format(emoji, msg),
             "icon_emoji": ":robot_face:"}
        post(slack_conf['webhook_url'], json=m)

def fetch_ds_metadata(ds_id):
    db = DB(SMConfig.get_conf()['db'])
    return db.select_one('SELECT name, metadata FROM dataset WHERE id = %s', ds_id)

def send_email(email, subj, body):
    ses = boto3.client('ses', 'eu-west-1')
    resp = ses.send_email(
        # Source='metaspace2020@gmail.com',
        Source='contact@metaspace2020.eu',
        Destination={
            'ToAddresses': [email]
        },
        Message={
            'Subject': {
                'Data': subj
            },
            'Body': {
                'Text': {
                    'Data': body
                }
            }
        }
    )
    if resp['ResponseMetadata']['HTTPStatusCode'] == 200:
        logger.info('Email with "{}" subject was sent to {}'.format(subj, email))
    else:
        logger.warn('SEM failed to send email to {}'.format(email))

def on_job_succeeded(msg):
    ds_name, ds_meta = fetch_ds_metadata(msg['ds_id'])

    base_url = SMConfig.get_conf()['services']['web_app_url']
    import urllib
    url_params = urllib.quote(msg['ds_id'])
    msg['web_app_link'] = '{}/#/annotations?ds={}'.format(base_url, url_params)
    post_to_slack('dart', ' [v] Finished: {}'.format(json.dumps(msg)))

    submitter = ds_meta['Submitted_By']['Submitter']

    if 'Email' in submitter and ds_meta['metaspace_options']['notify_submitter']:
        email_body = (
            'Dear {} {},\n\n'
            'Thank you for uploading the dataset {} to the METASPACE  annotation service. '
            'We are pleased to inform you that the dataset has been processed and is available on alpha.metaspace2020.eu.\n\n'
            'Best regards,\n'
            'METASPACE Team\n\n'
            '---\n'
            'The online annotation engine is being developed as part of the METASPACE Horizon2020 project (grant number: 634402).'
        ).format(submitter.get('First_Name', ''), submitter.get('Surname', ''), ds_name)
        send_email(submitter['Email'],
                   'METASPACE service notification (SUCCESS)',
                   email_body)

def on_job_failed(msg):
    post_to_slack('hankey', ' [x] Failed: {}'.format(json.dumps(msg)))

    ds_name, ds_meta = fetch_ds_metadata(msg['ds_id'])
    submitter = ds_meta['Submitted_By']['Submitter']

    if 'Email' in submitter and ds_meta['metaspace_options']['notify_submitter']:
        email_body = (
            'Dear {} {},\n\n'
            'Thank you for uploading the dataset "{}" to the METASPACE  annotation service. '
            'We are sorry to inform you that there were issues with processing your dataset. '
            'We are already working on it. '
            'In case you have any questions, please do not hesitate to write us at contact@metaspace2020.eu\n\n'
            'Best regards,\n'
            'METASPACE Team\n\n'
            '---\n'
            'The online annotation engine is being developed as part of the METASPACE Horizon2020 project (grant number: 634402).'
        ).format(submitter.get('First_Name', ''), submitter.get('Surname', ''), ds_name)
        send_email(submitter['Email'],
                   'METASPACE service notification (FAILED)',
                   email_body)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=('Daemon for reading messages from the '
                                                  'queue and starting annotation jobs'))
    parser.add_argument('--config', dest='sm_config_path', type=str, help='SM config path')

    args = parser.parse_args()
    SMConfig.set_path(args.sm_config_path)
    rabbit_config = SMConfig.get_conf()['rabbitmq']

    configure_loggers()
    logger = logging.getLogger('sm-queue')

    def run_job_callback(msg):
        log_msg = " [v] Received: {}".format(msg)
        logger.info(log_msg)
        post_to_slack('new', " [v] Received: {}".format(json.dumps(msg)))
        job = SearchJob(msg['ds_id'], args.sm_config_path)
        job.run()

    annotation_queue = QueueConsumer(rabbit_config, 'sm_annotate',
                                     run_job_callback, on_job_succeeded, on_job_failed)
    annotation_queue.run()
<commit_msg>bug fix: email submitter if metaspace_options.notify_submitter is not provided in metadata<commit_after>#!/usr/bin/env python
import argparse
import json
from requests import post
import logging
import boto3

from sm.engine.util import SMConfig, sm_log_formatters, sm_log_config, init_logger
from sm.engine.search_job import SearchJob
from sm.engine.queue import QueueConsumer
from sm.engine.db import DB

def configure_loggers():
    log_config = sm_log_config
    log_config['loggers']['sm-engine']['handlers'] = ['console_warn', 'file']
    init_logger(log_config)

def post_to_slack(emoji, msg):
    slack_conf = SMConfig.get_conf()['slack']

    if slack_conf['webhook_url']:
        m = {"channel": slack_conf['channel'],
             "username": "webhookbot",
             "text": ":{}:{}".format(emoji, msg),
             "icon_emoji": ":robot_face:"}
        post(slack_conf['webhook_url'], json=m)

def fetch_ds_metadata(ds_id):
    db = DB(SMConfig.get_conf()['db'])
    return db.select_one('SELECT name, metadata FROM dataset WHERE id = %s', ds_id)

def send_email(email, subj, body):
    ses = boto3.client('ses', 'eu-west-1')
    resp = ses.send_email(
        # Source='metaspace2020@gmail.com',
        Source='contact@metaspace2020.eu',
        Destination={
            'ToAddresses': [email]
        },
        Message={
            'Subject': {
                'Data': subj
            },
            'Body': {
                'Text': {
                    'Data': body
                }
            }
        }
    )
    if resp['ResponseMetadata']['HTTPStatusCode'] == 200:
        logger.info('Email with "{}" subject was sent to {}'.format(subj, email))
    else:
        logger.warn('SEM failed to send email to {}'.format(email))

def on_job_succeeded(msg):
    ds_name, ds_meta = fetch_ds_metadata(msg['ds_id'])

    base_url = SMConfig.get_conf()['services']['web_app_url']
    import urllib
    url_params = urllib.quote(msg['ds_id'])
    msg['web_app_link'] = '{}/#/annotations?ds={}'.format(base_url, url_params)
    post_to_slack('dart', ' [v] Finished: {}'.format(json.dumps(msg)))

    submitter = ds_meta['Submitted_By'].get('Submitter', '')

    if 'Email' in submitter and ds_meta['metaspace_options'].get('notify_submitter', True):
        email_body = (
            'Dear {} {},\n\n'
            'Thank you for uploading the dataset {} to the METASPACE  annotation service. '
            'We are pleased to inform you that the dataset has been processed and is available on alpha.metaspace2020.eu.\n\n'
            'Best regards,\n'
            'METASPACE Team\n\n'
            '---\n'
            'The online annotation engine is being developed as part of the METASPACE Horizon2020 project (grant number: 634402).'
        ).format(submitter.get('First_Name', ''), submitter.get('Surname', ''), ds_name)
        send_email(submitter['Email'],
                   'METASPACE service notification (SUCCESS)',
                   email_body)

def on_job_failed(msg):
    post_to_slack('hankey', ' [x] Failed: {}'.format(json.dumps(msg)))

    ds_name, ds_meta = fetch_ds_metadata(msg['ds_id'])
    submitter = ds_meta['Submitted_By'].get('Submitter', '')

    if 'Email' in submitter and ds_meta['metaspace_options'].get('notify_submitter', True):
        email_body = (
            'Dear {} {},\n\n'
            'Thank you for uploading the dataset "{}" to the METASPACE  annotation service. '
            'We are sorry to inform you that there were issues with processing your dataset. '
            'We are already working on it. '
            'In case you have any questions, please do not hesitate to write us at contact@metaspace2020.eu\n\n'
            'Best regards,\n'
            'METASPACE Team\n\n'
            '---\n'
            'The online annotation engine is being developed as part of the METASPACE Horizon2020 project (grant number: 634402).'
        ).format(submitter.get('First_Name', ''), submitter.get('Surname', ''), ds_name)
        send_email(submitter['Email'],
                   'METASPACE service notification (FAILED)',
                   email_body)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=('Daemon for reading messages from the '
                                                  'queue and starting annotation jobs'))
    parser.add_argument('--config', dest='sm_config_path', type=str, help='SM config path')

    args = parser.parse_args()
    SMConfig.set_path(args.sm_config_path)
    rabbit_config = SMConfig.get_conf()['rabbitmq']

    configure_loggers()
    logger = logging.getLogger('sm-queue')

    def run_job_callback(msg):
        log_msg = " [v] Received: {}".format(msg)
        logger.info(log_msg)
        post_to_slack('new', " [v] Received: {}".format(json.dumps(msg)))
        job = SearchJob(msg['ds_id'], args.sm_config_path)
        job.run()

    annotation_queue = QueueConsumer(rabbit_config, 'sm_annotate',
                                     run_job_callback, on_job_succeeded, on_job_failed)
    annotation_queue.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import curses
import threading

from panel import Panel, PanelManager
from postponer import Postponer
import rungit

class GitermPanelManager(PanelManager):

    def __init__(self, stdscr):
        super(GitermPanelManager, self).__init__(stdscr)
        self.create_panels()

    def create_panels(self):
        """Creates a graphical-like UI:
    ┌────────┐┌────────────────────────────────┐
    │Branches││Log history                     │
    │> master││                                │
    │> devel ││                                │
    │        ││                                │
    │Remotes ││                                │
    │> origin│└────────────────────────────────┘
    │        │┌───────────────┐┌───────────────┐
    │Tags    ││ Staged files  ││               │
    │        │└───────────────┘│ Diff of       │
    │Stashes │┌───────────────┐│ selected file │
    │        ││ Changed files ││               │
    └────────┘└───────────────┘└───────────────┘
        """
        height, width = self.stdscr.getmaxyx()
        if height < 8 or width < 40:
            raise Exception("Height and width must be at least 8x40.\
                Currently: %sx%s" % (height, width))
        # Following sizes are percentages (e.g. w_30 is 30% of screen width)
        w_20 = min(max(width // 5, 20), 25)
        w_30 = width // 3
        w_50 = width - w_30 - w_20
        h_49 = height // 2
        h_51 = height - h_49
        h_25 = h_51 // 2
        h_26 = h_51 - h_25
        h_l = height // 5
        h_br = height - 4 * h_l
        self['branches'] = StateLinePanel(self.stdscr, h_br, w_20, 0, 0, title='Branches')
        self['remotes'] = Panel(self.stdscr, h_l, w_20, h_br, 0, title='Remotes')
        self['stashes'] = Panel(self.stdscr, h_l, w_20, h_br+h_l, 0, title='Stashes')
        self['submodules'] = Panel(self.stdscr, h_l, w_20, h_br+2*h_l, 0, title='Submodules')
        self['tags'] = StateLinePanel(self.stdscr, h_l, w_20, h_br+3*h_l, 0, title='Tags')
        self['log'] = StateLinePanel(self.stdscr, h_49, w_30 + w_50, 0, w_20, title='History')
        self['stage'] = StagerUnstager(self, self.stdscr, h_25, w_30, h_49, w_20, title='Staging Area')
        self['changes'] = StagerUnstager(self, self.stdscr, h_26, w_30, h_49 + h_25, w_20, title='Local Changes')
        self['diff'] = Diff(self.stdscr, h_51, w_50, h_49, w_20 + w_30, title='Diff View')

        self['changes'].rungit = rungit.git_changed
        self['stage'].rungit = rungit.git_staged
        self['log'].rungit = rungit.git_history
        self['branches'].rungit = rungit.git_branches
        self['remotes'].rungit = rungit.git_remotes
        self['stashes'].rungit = rungit.git_stashes
        self['submodules'].rungit = rungit.git_submodules
        self['tags'].rungit = rungit.git_tags
        self['diff'].rungit = rungit.git_diff

        self['changes'].action = self.stage_file
        self['stage'].action = self.unstage_file

    def stage_file(self, path):
        rungit.git_stage_file(path)
        self['stage'].handle_event(None)
        self['changes'].handle_event(None)

    def unstage_file(self, path):
        rungit.git_unstage_file(path)
        self['stage'].handle_event(None)
        self['changes'].handle_event(None)

class Diff(Panel):

    def __init__(self, *args, **kwargs):
        super(Diff, self).__init__(*args, **kwargs)
        self.default_title = self.title
        self.running = threading.Lock()

    def handle_event(self, filepath):
        if type(filepath) is not str:
            return
        self.running.acquire()
        self.content = self.rungit(filepath)
        self.topLineNum = 0
        self.selected_line = -1
        self.hovered_line = 0
        self.title = ": " + filepath if type(filepath) == str else ''
        self.title = self.default_title + self.title
        self.display()
        self.running.release()

class StagerUnstager(Panel):

    def __init__(self, parent, *args, **kwargs):
        super(StagerUnstager, self).__init__(*args, **kwargs)
        self.parent = parent
        self.default_title = self.title
        self.postponer = Postponer(timeout_in_seconds=0.3)

    def filename_from_linenum(self, linenum):
        if len(self.content) <= linenum:
            # TODO: log error and raise exception
            return 'error: l.%s>%s' % (linenum, str(len(self.content)+1))
        return self.content[linenum].split()[1]

    def move_cursor(self):
        super(StagerUnstager, self).move_cursor()
        self.request_diff_in_diff_view()

    def request_diff_in_diff_view(self):
        if self.content:
            self.postponer.set(
                action=self.parent['diff'].handle_event,
                args=[self.filename_from_linenum(self.hovered_line)])

    def select(self):
        if self.selected_line == self.hovered_line:
            self.selected_line = -1
        else:
            self.selected_line = self.hovered_line
        if self.selected_line != -1:
            self.selected_file = self.filename_from_linenum(self.selected_line)
            self.cursor_y = max(self.cursor_y - 1, self.CT)
            self.action(self.selected_file)
            self.unselect()
        self.display()

class StateLinePanel(Panel):

    def handle_event(self, event):
        self.content = self.rungit()
        for i, line in enumerate(self.content):
            if line.startswith('*'):
                self.decorations[i] = curses.A_BOLD
                self.content[i] = line[1:]
        self.display()
<commit_msg>fix: Diff view does not update when staging #2<commit_after># -*- coding: utf-8 -*-
import curses
import threading

from panel import Panel, PanelManager
from postponer import Postponer
import rungit

class GitermPanelManager(PanelManager):

    def __init__(self, stdscr):
        super(GitermPanelManager, self).__init__(stdscr)
        self.create_panels()

    def create_panels(self):
        """Creates a graphical-like UI:
    ┌────────┐┌────────────────────────────────┐
    │Branches││Log history                     │
    │> master││                                │
    │> devel ││                                │
    │        ││                                │
    │Remotes ││                                │
    │> origin│└────────────────────────────────┘
    │        │┌───────────────┐┌───────────────┐
    │Tags    ││ Staged files  ││               │
    │        │└───────────────┘│ Diff of       │
    │Stashes │┌───────────────┐│ selected file │
    │        ││ Changed files ││               │
    └────────┘└───────────────┘└───────────────┘
        """
        height, width = self.stdscr.getmaxyx()
        if height < 8 or width < 40:
            raise Exception("Height and width must be at least 8x40.\
                Currently: %sx%s" % (height, width))
        # Following sizes are percentages (e.g. w_30 is 30% of screen width)
        w_20 = min(max(width // 5, 20), 25)
        w_30 = width // 3
        w_50 = width - w_30 - w_20
        h_49 = height // 2
        h_51 = height - h_49
        h_25 = h_51 // 2
        h_26 = h_51 - h_25
        h_l = height // 5
        h_br = height - 4 * h_l
        self['branches'] = StateLinePanel(self.stdscr, h_br, w_20, 0, 0, title='Branches')
        self['remotes'] = Panel(self.stdscr, h_l, w_20, h_br, 0, title='Remotes')
        self['stashes'] = Panel(self.stdscr, h_l, w_20, h_br+h_l, 0, title='Stashes')
        self['submodules'] = Panel(self.stdscr, h_l, w_20, h_br+2*h_l, 0, title='Submodules')
        self['tags'] = StateLinePanel(self.stdscr, h_l, w_20, h_br+3*h_l, 0, title='Tags')
        self['log'] = StateLinePanel(self.stdscr, h_49, w_30 + w_50, 0, w_20, title='History')
        self['stage'] = StagerUnstager(self, self.stdscr, h_25, w_30, h_49, w_20, title='Staging Area')
        self['changes'] = StagerUnstager(self, self.stdscr, h_26, w_30, h_49 + h_25, w_20, title='Local Changes')
        self['diff'] = Diff(self.stdscr, h_51, w_50, h_49, w_20 + w_30, title='Diff View')

        self['changes'].rungit = rungit.git_changed
        self['stage'].rungit = rungit.git_staged
        self['log'].rungit = rungit.git_history
        self['branches'].rungit = rungit.git_branches
        self['remotes'].rungit = rungit.git_remotes
        self['stashes'].rungit = rungit.git_stashes
        self['submodules'].rungit = rungit.git_submodules
        self['tags'].rungit = rungit.git_tags
        self['diff'].rungit = rungit.git_diff

        self['changes'].action = self.stage_file
        self['stage'].action = self.unstage_file

    def stage_file(self, path):
        rungit.git_stage_file(path)
        self['stage'].handle_event(None)
        self['changes'].handle_event(None)
        self['changes'].request_diff_in_diff_view()

    def unstage_file(self, path):
        rungit.git_unstage_file(path)
        self['stage'].handle_event(None)
        self['changes'].handle_event(None)
        self['changes'].request_diff_in_diff_view()

class Diff(Panel):

    def __init__(self, *args, **kwargs):
        super(Diff, self).__init__(*args, **kwargs)
        self.default_title = self.title
        self.running = threading.Lock()

    def handle_event(self, filepath):
        if type(filepath) is not str:
            return
        self.running.acquire()
        self.content = self.rungit(filepath)
        self.topLineNum = 0
        self.selected_line = -1
        self.hovered_line = 0
        self.title = ": " + filepath if type(filepath) == str else ''
        self.title = self.default_title + self.title
        self.display()
        self.running.release()

class StagerUnstager(Panel):

    def __init__(self, parent, *args, **kwargs):
        super(StagerUnstager, self).__init__(*args, **kwargs)
        self.parent = parent
        self.default_title = self.title
        self.postponer = Postponer(timeout_in_seconds=0.3)

    def filename_from_linenum(self, linenum):
        if len(self.content) <= linenum:
            # TODO: log error and raise exception
            return 'error: l.%s>%s' % (linenum, str(len(self.content)+1))
        return self.content[linenum].split()[1]

    def move_cursor(self):
        super(StagerUnstager, self).move_cursor()
        self.request_diff_in_diff_view()

    def request_diff_in_diff_view(self):
        if self.content:
            self.postponer.set(
                action=self.parent['diff'].handle_event,
                args=[self.filename_from_linenum(self.hovered_line)])

    def select(self):
        if self.selected_line == self.hovered_line:
            self.selected_line = -1
        else:
            self.selected_line = self.hovered_line
        if self.selected_line != -1:
            self.selected_file = self.filename_from_linenum(self.selected_line)
            self.cursor_y = max(self.cursor_y - 1, self.CT)
            self.action(self.selected_file)
            self.unselect()
        self.display()

class StateLinePanel(Panel):

    def handle_event(self, event):
        self.content = self.rungit()
        for i, line in enumerate(self.content):
            if line.startswith('*'):
                self.decorations[i] = curses.A_BOLD
                self.content[i] = line[1:]
        self.display()
<|endoftext|>


<commit_before>from django.contrib import admin

from .models import ArticleImage, Sidebar, Destination, Category, Brief, Article
from .models import supports_video, supports_polls

from tango_shared.admin import TextCounterWidget

if supports_video:
    from video.admin import VideoInline

class ArticleImagesInline(admin.TabularInline):
    model = ArticleImage
    extra = 3

class SidebarInline(admin.TabularInline):
    model = Sidebar
    extra = 2

class ArticleAdmin(admin.ModelAdmin):
    class Media:
        js = ('/static/js/admin/inline_reorder.js',)

    ordering = ['-created']
    list_display = ('title', 'author', 'destination', 'created',)
    list_filter = ('created', 'last_modified', 'enable_comments', 'publication', 'destination')
    prepopulated_fields = {'slug': ('title',)}
    search_fields = ['body', 'title']
    filter_horizontal = ('sections', 'articles', 'galleries')
    inlines = [
        ArticleImagesInline,
        SidebarInline,
    ]
    if supports_video:
        inlines.append(VideoInline)
    """
    if NEWS_SITE:
        opinion         = models.BooleanField("Opinion/Editorial", default=False)
        source          = models.CharField(max_length=200, default=SOURCE, blank=True, null=True)
        dateline        = models.CharField(max_length=200, blank=True, null=True)
    """
    related = ('Related', {
        'fields': ['galleries', 'articles'],
        'description': 'Other content directly related to this article'
    })
    if supports_polls:
        related[1]['fields'].insert(0, 'polls')

    fieldsets = (
        ('Routing', {'fields': ('destination', 'sections')}),
        ('Author info', {'fields': (('author', 'guest_author'))}),
        ('Header', {'fields': ('kicker', 'title', 'subhead')}),
        ('Content', {'fields': ('summary', 'body', 'pull_quote', 'endnote')}),
        ('Admin fields', {
            'description': 'You should rarely, if ever, need to touch these fields.',
            'fields': ('slug', 'enable_comments', 'sites', 'override_url'),
            'classes': ['collapse']
        }),
        related,
        ('Meta',     {
            'fields': (('publication'), 'featured'),
            'description': 'Additional information about this story'
        }),
    )

class BriefAdmin(admin.ModelAdmin):
    list_display = ('pub_date', 'text')

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'text':
            kwargs['widget'] = TextCounterWidget()
        return super(BriefAdmin, self).formfield_for_dbfield(db_field, **kwargs)

admin.site.register(Destination)
admin.site.register(Category)
admin.site.register(Brief, BriefAdmin)
admin.site.register(Article, ArticleAdmin)
<commit_msg>Import problem in articles text counter widget<commit_after>from django.contrib import admin

from .models import ArticleImage, Sidebar, Destination, Category, Brief, Article
from .models import supports_video, supports_polls

from tango_admin.admin import TextCounterWidget

if supports_video:
    from video.admin import VideoInline

class ArticleImagesInline(admin.TabularInline):
    model = ArticleImage
    extra = 3

class SidebarInline(admin.TabularInline):
    model = Sidebar
    extra = 2

class ArticleAdmin(admin.ModelAdmin):
    class Media:
        js = ('/static/js/admin/inline_reorder.js',)

    ordering = ['-created']
    list_display = ('title', 'author', 'destination', 'created',)
    list_filter = ('created', 'last_modified', 'enable_comments', 'publication', 'destination')
    prepopulated_fields = {'slug': ('title',)}
    search_fields = ['body', 'title']
    filter_horizontal = ('sections', 'articles', 'galleries')
    inlines = [
        ArticleImagesInline,
        SidebarInline,
    ]
    if supports_video:
        inlines.append(VideoInline)
    """
    if NEWS_SITE:
        opinion         = models.BooleanField("Opinion/Editorial", default=False)
        source          = models.CharField(max_length=200, default=SOURCE, blank=True, null=True)
        dateline        = models.CharField(max_length=200, blank=True, null=True)
    """
    related = ('Related', {
        'fields': ['galleries', 'articles'],
        'description': 'Other content directly related to this article'
    })
    if supports_polls:
        related[1]['fields'].insert(0, 'polls')

    fieldsets = (
        ('Routing', {'fields': ('destination', 'sections')}),
        ('Author info', {'fields': (('author', 'guest_author'))}),
        ('Header', {'fields': ('kicker', 'title', 'subhead')}),
        ('Content', {'fields': ('summary', 'body', 'pull_quote', 'endnote')}),
        ('Admin fields', {
            'description': 'You should rarely, if ever, need to touch these fields.',
            'fields': ('slug', 'enable_comments', 'sites', 'override_url'),
            'classes': ['collapse']
        }),
        related,
        ('Meta',     {
            'fields': (('publication'), 'featured'),
            'description': 'Additional information about this story'
        }),
    )

class BriefAdmin(admin.ModelAdmin):
    list_display = ('pub_date', 'text')

    def formfield_for_dbfield(self, db_field, **kwargs):
        if db_field.name == 'text':
            kwargs['widget'] = TextCounterWidget()
        return super(BriefAdmin, self).formfield_for_dbfield(db_field, **kwargs)

admin.site.register(Destination)
admin.site.register(Category)
admin.site.register(Brief, BriefAdmin)
admin.site.register(Article, ArticleAdmin)
<|endoftext|>