<commit_before>from portality import models
from portality.core import app

from portality.tasks.redis_huey import main_queue, schedule
from portality.decorators import write_required

from portality.background import BackgroundTask, BackgroundApi, BackgroundException
from portality.journalcsv import make_journals_csv

import os, codecs
from datetime import datetime
from operator import itemgetter

class JournalCSVBackgroundTask(BackgroundTask):

    __action__ = "journal_csv"

    def run(self):
        """
        Execute the task as specified by the background_job
        :return:
        """
        job = self.background_job

        cdir = app.config.get("CACHE_DIR")
        if cdir is None:
            raise BackgroundException("You must set CACHE_DIR in the config")
        csvdir = os.path.join(cdir, "csv")

        # save it into the cache directory
        attachment_name = 'doaj_' + datetime.strftime(datetime.now(), '%Y%m%d_%H%M') + '_utf8.csv'
        out = os.path.join(csvdir, attachment_name)

        # write the csv file
        with codecs.open(out, 'wb', encoding='utf-8') as csvfile:
            make_journals_csv(csvfile)

        # update the ES record to point to the new file
        models.Cache.cache_csv(attachment_name)

        # remove all but the two latest csvs
        csvs = [(c, os.path.getmtime(os.path.join(csvdir, c))) for c in os.listdir(csvdir) if c.endswith(".csv")]
        sorted_csvs = sorted(csvs, key=itemgetter(1), reverse=True)

        if len(sorted_csvs) > 2:
            for c, lm in sorted_csvs[2:]:
                os.remove(os.path.join(csvdir, c))

    def cleanup(self):
        """
        Cleanup after a successful OR failed run of the task
        :return:
        """
        pass

    @classmethod
    def prepare(cls, username, **kwargs):
        """
        Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob,
        or fail with a suitable exception

        :param kwargs: arbitrary keyword arguments pertaining to this task type
        :return: a BackgroundJob instance representing this task
        """
        cdir = app.config.get("CACHE_DIR")
        if cdir is None:
            raise BackgroundException("You must set CACHE_DIR in the config")

        # first prepare a job record
        job = models.BackgroundJob()
        job.user = username
        job.action = cls.__action__
        return job

    @classmethod
    def submit(cls, background_job):
        """
        Submit the specified BackgroundJob to the background queue

        :param background_job: the BackgroundJob instance
        :return:
        """
        background_job.save()
        journal_csv.schedule(args=(background_job.id,), delay=10)

@main_queue.periodic_task(schedule("journal_csv"))
@write_required(script=True)
def scheduled_journal_csv():
    user = app.config.get("SYSTEM_USERNAME")
    job = JournalCSVBackgroundTask.prepare(user)
    JournalCSVBackgroundTask.submit(job)

@main_queue.task()
@write_required(script=True)
def journal_csv(job_id):
    job = models.BackgroundJob.pull(job_id)
    task = JournalCSVBackgroundTask(job)
    BackgroundApi.execute(task)
<commit_msg>Fix csv task to create folder in appdata/cache<commit_after>from portality import models
from portality.core import app

from portality.tasks.redis_huey import main_queue, schedule
from portality.decorators import write_required

from portality.background import BackgroundTask, BackgroundApi, BackgroundException
from portality.journalcsv import make_journals_csv

import os, codecs
from datetime import datetime
from operator import itemgetter

class JournalCSVBackgroundTask(BackgroundTask):

    __action__ = "journal_csv"

    def run(self):
        """
        Execute the task as specified by the background_job
        :return:
        """
        job = self.background_job

        cdir = app.config.get("CACHE_DIR")
        if cdir is None:
            raise BackgroundException("You must set CACHE_DIR in the config")
        csvdir = os.path.join(cdir, "csv")

        if not os.path.exists(csvdir):
            os.makedirs(csvdir)

        # save it into the cache directory
        attachment_name = 'doaj_' + datetime.strftime(datetime.now(), '%Y%m%d_%H%M') + '_utf8.csv'
        out = os.path.join(csvdir, attachment_name)

        # write the csv file
        with codecs.open(out, 'wb', encoding='utf-8') as csvfile:
            make_journals_csv(csvfile)

        # update the ES record to point to the new file
        models.Cache.cache_csv(attachment_name)

        # remove all but the two latest csvs
        csvs = [(c, os.path.getmtime(os.path.join(csvdir, c))) for c in os.listdir(csvdir) if c.endswith(".csv")]
        sorted_csvs = sorted(csvs, key=itemgetter(1), reverse=True)

        if len(sorted_csvs) > 2:
            for c, lm in sorted_csvs[2:]:
                os.remove(os.path.join(csvdir, c))

    def cleanup(self):
        """
        Cleanup after a successful OR failed run of the task
        :return:
        """
        pass

    @classmethod
    def prepare(cls, username, **kwargs):
        """
        Take an arbitrary set of keyword arguments and return an instance of a BackgroundJob,
        or fail with a suitable exception

        :param kwargs: arbitrary keyword arguments pertaining to this task type
        :return: a BackgroundJob instance representing this task
        """
        cdir = app.config.get("CACHE_DIR")
        if cdir is None:
            raise BackgroundException("You must set CACHE_DIR in the config")

        # first prepare a job record
        job = models.BackgroundJob()
        job.user = username
        job.action = cls.__action__
        return job

    @classmethod
    def submit(cls, background_job):
        """
        Submit the specified BackgroundJob to the background queue

        :param background_job: the BackgroundJob instance
        :return:
        """
        background_job.save()
        journal_csv.schedule(args=(background_job.id,), delay=10)

@main_queue.periodic_task(schedule("journal_csv"))
@write_required(script=True)
def scheduled_journal_csv():
    user = app.config.get("SYSTEM_USERNAME")
    job = JournalCSVBackgroundTask.prepare(user)
    JournalCSVBackgroundTask.submit(job)

@main_queue.task()
@write_required(script=True)
def journal_csv(job_id):
    job = models.BackgroundJob.pull(job_id)
    task = JournalCSVBackgroundTask(job)
    BackgroundApi.execute(task)
<|endoftext|>


<commit_before>import time
import urllib
import socket
from xml.dom import minidom

from twisted.web import resource

from pydirector import Version
from pydirector.web import css
from pydirector.web import template

class StyleSheet(resource.Resource):
    """

    """
    def render_GET(self, request):
        """

        """
        return css.adminCSS

class BasePage(resource.Resource):
    """

    """
    def __init__(self, parent):
        resource.Resource.__init__(self)
        self.parent = parent

    def getHeader(self, refreshURL=''):
        """

        """
        refresh = ''
        if refreshURL:
            # XXX add an admin configuration option for setting the refresh
            # rate
            refreshRate = 30
            refresh = template.refresh % (refreshRate, refreshURL)
        return template.header % (
            refresh, self.parent.serverVersion, socket.gethostname())

    def getBody(self):
        """
        Subclasses must override this.
        """
        raise NotImplemented

    def getFooter(self, message=''):
        """

        """
        # XXX put the project URL in the admin config
        projectURL ='http://pythondirector.sf.net'
        if message:
            message = template.message % urllib.unquote(message)
        return template.footer % (projectURL, message)

    def getPage(self):
        """
        Subclasses must override this.
        """
        raise NotImplemented

    def render_GET(self, request):
        """

        """
        return str(self.getPage(request))

class RunningPage(BasePage):
    """

    """
    def getPage(self, request):
        """
        Don't look at me; this craziness is a modified version of the original.
        """
        verbose = False
        resultMessage = ''
        content = ''
        if request.args.has_key('refresh'):
            refresh = bool(request.args['refresh'][0])
            url = '/running?refresh=1&ignore=%s' % time.time()
            content += self.getHeader(refreshURL=url)
            stopStart = template.stopRefresh % time.time()
        else:
            content += self.getHeader()
            stopStart = template.startRefresh % time.time()
        content += template.refreshButtons % (
            time.ctime(time.time()), time.time(), stopStart)
        for service in self.parent.director.conf.getServices():
            content += template.serviceName % service.name
            for l in service.listen:
                content += template.listeningService % l
            eg = service.getEnabledGroup()
            groups = service.getGroups()
            for group in groups:
                sch = self.parent.director.getScheduler(
                    service.name, group.name)
                stats = sch.getStats(verbose=verbose)
                hdict = sch.getHostNames()
                if group is eg:
                    klass = 'enabled'
                    desc = template.groupDescEnabled
                else:
                    klass = 'inactive'
                    desc = template.groupDescDisabled % (
                        service.name, group.name)
                content += template.groupName % (klass, group.name)
                content += desc
                content += template.groupHeaderForm % (
                    service.name, group.name, klass)
                counts = stats['open']
                totals = stats['totals']
                k = counts.keys()
                k.sort()
                for h in k:
                    if counts.has_key(h):
                        oc = counts[h]
                    else:
                        oc = '--'
                    if totals.has_key(h):
                        tc = totals[h]
                    else:
                        tc = '--'
                    content += template.hostInfo % (
                        klass, hdict[h], h, oc, tc, urllib.quote(service.name),
                        urllib.quote(group.name), urllib.quote(h))
                bad = stats['bad']
                if bad:
                    content += template.badHostGroup % klass
                for k in bad.keys():
                    host = '%s:%s' % k
                    when, what = bad[k]
                    content += template.badHostInfo % (
                        klass, hdict[host], host, what)
            content += template.serviceClose
        content += self.getFooter(resultMessage)
        return content

class RunningConfig(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        request.setHeader('Content-type', 'text/plain')
        verbose = False
        conf = self.parent.director.conf
        doc = minidom.Document()
        top = doc.createElement("pdconfig")
        doc.appendChild(top)
        for service in conf.getServices():
            top.appendChild(doc.createTextNode("\n    "))
            serv = doc.createElement("service")
            serv.setAttribute('name', service.name)
            top.appendChild(serv)
            for l in service.listen:
                serv.appendChild(doc.createTextNode("\n        "))
                lobj = doc.createElement("listen")
                lobj.setAttribute('ip', l)
                serv.appendChild(lobj)
            groups = service.getGroups()
            for group in groups:
                serv.appendChild(doc.createTextNode("\n        "))
                sch = self.parent.director.getScheduler(service.name, group.name)
                xg = doc.createElement("group")
                xg.setAttribute('name', group.name)
                xg.setAttribute('scheduler', sch.schedulerName)
                serv.appendChild(xg)
                stats = sch.getStats(verbose=verbose)
                hosts = group.getHosts()
                hdict = sch.getHostNames()
                counts = stats['open']
                ahosts = counts.keys() # ahosts is now a list of active hosts
                # now add disabled hosts.
                for k in stats['bad'].keys():
                    ahosts.append('%s:%s'%k)
                ahosts.sort()
                for h in ahosts:
                    xg.appendChild(doc.createTextNode("\n            "))
                    xh = doc.createElement("host")
                    xh.setAttribute('name', hdict[h])
                    xh.setAttribute('ip', h)
                    xg.appendChild(xh)
                xg.appendChild(doc.createTextNode("\n        "))
            serv.appendChild(doc.createTextNode("\n        "))
            eg = service.getEnabledGroup()
            xeg = doc.createElement("enable")
            xeg.setAttribute("group", eg.name)
            serv.appendChild(xeg)
            serv.appendChild(doc.createTextNode("\n    "))
        top.appendChild(doc.createTextNode("\n    "))
        # now the admin block
        admin = self.parent.director.conf.admin
        if admin is not None:
            xa = doc.createElement("admin")
            xa.setAttribute("listen", "%s:%s"%admin.listen)
            top.appendChild(xa)
            for user in admin.getUsers():
                xa.appendChild(doc.createTextNode("\n        "))
                xu = doc.createElement("user")
                xu.setAttribute("name", user.name)
                xu.setAttribute("password", user.password)
                xu.setAttribute("access", user.access)
                xa.appendChild(xu)
            xa.appendChild(doc.createTextNode("\n    "))
            top.appendChild(doc.createTextNode("\n    "))
        # finally, the logging section (if set)
        #if logger.logfile is not None:
        #    xl = doc.createElement("logging")
        #    xl.setAttribute("file", logger.logfile)
        #    top.appendChild(xl)
        # final newline
        top.appendChild(doc.createTextNode("\n"))
        # and spit out the XML
        return doc.toxml()

class StoredConfig(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        request.setHeader('Content-type', 'text/plain')
        return self.parent.director.conf.dom.toxml()

class DeleteHost(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        service = request.args['service'][0]
        group = request.args['group'][0]
        ip = request.args['ip'][0]
        sched = self.parent.director.getScheduler(serviceName=service, groupName=group)
        service = self.parent.director.conf.getService(service)
        eg = service.getEnabledGroup()
        if group == eg.name:
            if sched.delHost(ip=ip, activegroup=1):
                msg = 'host %s deleted (from active group!)' % ip
            else:
                msg = 'host %s not deleted from active group' % ip
        else:
            if sched.delHost(ip=ip):
                msg = 'host %s deleted from inactive group' % ip
            else:
                msg = 'host %s not deleted from inactive group' % ip
        request.redirect('/running?resultMessage=%s' % urllib.quote(msg))
        return "OK"

class AddHost(BasePage):
    """

    """

class AdminServer(resource.Resource):
    """

    """
    def __init__(self, director):
        resource.Resource.__init__(self)
        self.director = director
        self.config = director.conf.admin
        self.starttime = time.time()
        self.serverVersion = "pythondirector/%s"%Version

    def getChild(self, name, request):
        """

        """
        if name == 'running' or name == '':
            return RunningPage(self)
        elif name == 'pydirector.css':
            return StyleSheet()
        elif name == 'running.xml':
            return RunningConfig(self)
        elif name == 'config.xml':
            return StoredConfig(self)
        elif name == 'delHost':
            return DeleteHost(self)
        elif name == 'addHost':
            return AddHost(self)
        return resource.Resource.getChild(self, name, request)
<commit_msg>Added support for adding hosts through the twisted admin web ui.<commit_after>import time
import urllib
import socket
from xml.dom import minidom

from twisted.web import resource

from pydirector import Version
from pydirector.web import css
from pydirector.web import template

class StyleSheet(resource.Resource):
    """

    """
    def render_GET(self, request):
        """

        """
        return css.adminCSS

class BasePage(resource.Resource):
    """

    """
    def __init__(self, parent):
        resource.Resource.__init__(self)
        self.parent = parent

    def getHeader(self, refreshURL=''):
        """

        """
        refresh = ''
        if refreshURL:
            # XXX add an admin configuration option for setting the refresh
            # rate
            refreshRate = 30
            refresh = template.refresh % (refreshRate, refreshURL)
        return template.header % (
            refresh, self.parent.serverVersion, socket.gethostname())

    def getBody(self):
        """
        Subclasses must override this.
        """
        raise NotImplemented

    def getFooter(self, message=''):
        """

        """
        # XXX put the project URL in the admin config
        projectURL ='http://pythondirector.sf.net'
        if message:
            message = template.message % urllib.unquote(message)
        return template.footer % (projectURL, message)

    def getPage(self):
        """
        Subclasses must override this.
        """
        raise NotImplemented

    def render_GET(self, request):
        """

        """
        return str(self.getPage(request))

class RunningPage(BasePage):
    """

    """
    def getPage(self, request):
        """
        Don't look at me; this craziness is a modified version of the original.
        """
        verbose = False
        resultMessage = ''
        content = ''
        if request.args.has_key('refresh'):
            refresh = bool(request.args['refresh'][0])
            url = '/running?refresh=1&ignore=%s' % time.time()
            content += self.getHeader(refreshURL=url)
            stopStart = template.stopRefresh % time.time()
        else:
            content += self.getHeader()
            stopStart = template.startRefresh % time.time()
        content += template.refreshButtons % (
            time.ctime(time.time()), time.time(), stopStart)
        for service in self.parent.director.conf.getServices():
            content += template.serviceName % service.name
            for l in service.listen:
                content += template.listeningService % l
            eg = service.getEnabledGroup()
            groups = service.getGroups()
            for group in groups:
                sch = self.parent.director.getScheduler(
                    service.name, group.name)
                stats = sch.getStats(verbose=verbose)
                hdict = sch.getHostNames()
                if group is eg:
                    klass = 'enabled'
                    desc = template.groupDescEnabled
                else:
                    klass = 'inactive'
                    desc = template.groupDescDisabled % (
                        service.name, group.name)
                content += template.groupName % (klass, group.name)
                content += desc
                content += template.groupHeaderForm % (
                    service.name, group.name, klass)
                counts = stats['open']
                totals = stats['totals']
                k = counts.keys()
                k.sort()
                for h in k:
                    if counts.has_key(h):
                        oc = counts[h]
                    else:
                        oc = '--'
                    if totals.has_key(h):
                        tc = totals[h]
                    else:
                        tc = '--'
                    content += template.hostInfo % (
                        klass, hdict[h], h, oc, tc, urllib.quote(service.name),
                        urllib.quote(group.name), urllib.quote(h))
                bad = stats['bad']
                if bad:
                    content += template.badHostGroup % klass
                for k in bad.keys():
                    host = '%s:%s' % k
                    when, what = bad[k]
                    content += template.badHostInfo % (
                        klass, hdict[host], host, what)
            content += template.serviceClose
        content += self.getFooter(resultMessage)
        return content

class RunningConfig(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        request.setHeader('Content-type', 'text/plain')
        verbose = False
        conf = self.parent.director.conf
        doc = minidom.Document()
        top = doc.createElement("pdconfig")
        doc.appendChild(top)
        for service in conf.getServices():
            top.appendChild(doc.createTextNode("\n    "))
            serv = doc.createElement("service")
            serv.setAttribute('name', service.name)
            top.appendChild(serv)
            for l in service.listen:
                serv.appendChild(doc.createTextNode("\n        "))
                lobj = doc.createElement("listen")
                lobj.setAttribute('ip', l)
                serv.appendChild(lobj)
            groups = service.getGroups()
            for group in groups:
                serv.appendChild(doc.createTextNode("\n        "))
                sch = self.parent.director.getScheduler(service.name, group.name)
                xg = doc.createElement("group")
                xg.setAttribute('name', group.name)
                xg.setAttribute('scheduler', sch.schedulerName)
                serv.appendChild(xg)
                stats = sch.getStats(verbose=verbose)
                hosts = group.getHosts()
                hdict = sch.getHostNames()
                counts = stats['open']
                ahosts = counts.keys() # ahosts is now a list of active hosts
                # now add disabled hosts.
                for k in stats['bad'].keys():
                    ahosts.append('%s:%s'%k)
                ahosts.sort()
                for h in ahosts:
                    xg.appendChild(doc.createTextNode("\n            "))
                    xh = doc.createElement("host")
                    xh.setAttribute('name', hdict[h])
                    xh.setAttribute('ip', h)
                    xg.appendChild(xh)
                xg.appendChild(doc.createTextNode("\n        "))
            serv.appendChild(doc.createTextNode("\n        "))
            eg = service.getEnabledGroup()
            xeg = doc.createElement("enable")
            xeg.setAttribute("group", eg.name)
            serv.appendChild(xeg)
            serv.appendChild(doc.createTextNode("\n    "))
        top.appendChild(doc.createTextNode("\n    "))
        # now the admin block
        admin = self.parent.director.conf.admin
        if admin is not None:
            xa = doc.createElement("admin")
            xa.setAttribute("listen", "%s:%s"%admin.listen)
            top.appendChild(xa)
            for user in admin.getUsers():
                xa.appendChild(doc.createTextNode("\n        "))
                xu = doc.createElement("user")
                xu.setAttribute("name", user.name)
                xu.setAttribute("password", user.password)
                xu.setAttribute("access", user.access)
                xa.appendChild(xu)
            xa.appendChild(doc.createTextNode("\n    "))
            top.appendChild(doc.createTextNode("\n    "))
        # finally, the logging section (if set)
        #if logger.logfile is not None:
        #    xl = doc.createElement("logging")
        #    xl.setAttribute("file", logger.logfile)
        #    top.appendChild(xl)
        # final newline
        top.appendChild(doc.createTextNode("\n"))
        # and spit out the XML
        return doc.toxml()

class StoredConfig(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        request.setHeader('Content-type', 'text/plain')
        return self.parent.director.conf.dom.toxml()

class DeleteHost(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        service = request.args['service'][0]
        group = request.args['group'][0]
        ip = request.args['ip'][0]
        sched = self.parent.director.getScheduler(
            serviceName=service, groupName=group)
        service = self.parent.director.conf.getService(service)
        eg = service.getEnabledGroup()
        if group == eg.name:
            if sched.delHost(ip=ip, activegroup=1):
                msg = 'host %s deleted (from active group!)' % ip
            else:
                msg = 'host %s not deleted from active group' % ip
        else:
            if sched.delHost(ip=ip):
                msg = 'host %s deleted from inactive group' % ip
            else:
                msg = 'host %s not deleted from inactive group' % ip
        request.redirect('/running?resultMessage=%s' % urllib.quote(msg))
        return "OK"

class AddHost(BasePage):
    """

    """
    def getPage(self, request):
        """

        """
        service = request.args['service'][0]
        group = request.args['group'][0]
        name = request.args['name'][0]
        ip = request.args['ip'][0]
        sched = self.parent.director.getScheduler(
            serviceName=service, groupName=group)
        sched.newHost(name=name, ip=ip)
        # also add to conf DOM object
        msg = 'Host %s(%s) added to %s / %s' % (name, ip, group, service)
        request.redirect('/running?resultMessage=%s' % urllib.quote(msg))
        return "OK"

class AdminServer(resource.Resource):
    """

    """
    def __init__(self, director):
        resource.Resource.__init__(self)
        self.director = director
        self.config = director.conf.admin
        self.starttime = time.time()
        self.serverVersion = "pythondirector/%s"%Version

    def getChild(self, name, request):
        """

        """
        if name == 'running' or name == '':
            return RunningPage(self)
        elif name == 'pydirector.css':
            return StyleSheet()
        elif name == 'running.xml':
            return RunningConfig(self)
        elif name == 'config.xml':
            return StoredConfig(self)
        elif name == 'delHost':
            return DeleteHost(self)
        elif name == 'addHost':
            return AddHost(self)
        return resource.Resource.getChild(self, name, request)
<|endoftext|>


<commit_before><commit_msg>fix(test): Make test less dependent on driver string.<commit_after><|endoftext|>


<commit_before>from .shape import Shape
from .shape import Pane

from .stroke_info import StrokeInfoFactory

strokeInfoFactory=StrokeInfoFactory()

class Stroke(Shape):
	def __init__(self, startPoint, strokeInfo, statePane=None):
		self.startPoint=startPoint
		self.strokeInfo=strokeInfo

		strokePath=strokeInfo.getStrokePath()
		boundary=strokePath.computeBoundaryWithStartPoint(startPoint)

		self.infoPane = Pane(*boundary)
		if statePane:
			self.statePane=statePane
		else:
			self.statePane=self.infoPane

	def getStartPoint(self):
		return self.startPoint

	def getStrokeInfo(self):
		return self.strokeInfo

	def getName(self):
		return self.getStrokeInfo().getName()

	def getStrokePath(self):
		return self.getStrokeInfo().getStrokePath()

	def getInfoPane(self):
		return self.infoPane

	def getStatePane(self):
		return self.statePane

	def draw(self, drawingSystem):
		startPoint = self.getStartPoint()

		stroke=self
		drawingSystem.onPreDrawStroke(stroke)
		drawingSystem.setPane(stroke.getInfoPane(), stroke.getStatePane())

		drawingSystem.startDrawing(startPoint)
		drawingSystem.draw(stroke.getStrokePath())
		drawingSystem.endDrawing()
		drawingSystem.onPostDrawStroke(stroke)

	def computeBoundary(self):
		startPoint=self.getStartPoint()
		strokePath=self.getStrokePath()
		return strokePath.computeBoundaryWithStartPoint(startPoint)

	def generateCopyToApplyNewPane(self, sgTargetPane, newSgTargetPane):
		sTargetPane=self.getStatePane()
		newSTargetPane=sgTargetPane.transformRelativePaneByTargetPane(sTargetPane, newSgTargetPane)

		strokeCopy=Stroke(self.startPoint, self.strokeInfo, newSTargetPane)
		return strokeCopy

def generateStroke(name, startPoint, parameterList):
	strokeInfo = strokeInfoFactory.generateStrokeInfo(name, parameterList)
	return Stroke(startPoint, strokeInfo)

class StrokeGroupInfo:
	def __init__(self, strokeList, statePane=None):
		self.strokeList=strokeList

		if not statePane:
			def mergeBoundaryList(boundaryList):
				from xie.graphics.shape import mergeBoundary
				r = boundaryList[0]
				for b in boundaryList[1:]:
					r = mergeBoundary(r, b)
				return r

			boundaryList=[stroke.computeBoundary() for stroke in strokeList]
			bBox=mergeBoundaryList(boundaryList)
			statePane=Pane(*bBox)

		self.statePane=statePane

	def getStrokeList(self):
		return self.strokeList

	def getStatePane(self):
		return self.statePane

	@classmethod
	def generateInstance(cls, strokeList, pane=None):
		return StrokeGroupInfo(strokeList, pane)

class StrokeGroup(Shape):
	def __init__(self, strokeGroupInfo, infoPane):
		self.strokeGroupInfo=strokeGroupInfo

		self.infoPane=infoPane
		self.statePane=infoPane

	def getInfoPane(self):
		return self.infoPane

	def getStrokeList(self):
		return self.strokeGroupInfo.getStrokeList()

	def getCount(self):
		return len(self.getStrokeList())

	def draw(self, drawingSystem):
		strokeList = self.getStrokeList();

		for stroke in strokeList:
			stroke.draw(drawingSystem)

	@classmethod
	def generateInstanceByInfo(cls, strokeGroupInfo, infoPane=None):
		if not infoPane:
			infoPane=strokeGroupInfo.getStatePane()
		return StrokeGroup(strokeGroupInfo, infoPane)

	@classmethod
	def generateInstanceByStrokeList(cls, strokeList, pane=None):
		strokeGroupInfo=StrokeGroupInfo.generateInstance(strokeList, pane)
		return StrokeGroup.generateInstanceByInfo(strokeGroupInfo)

	@classmethod
	def generateInstanceByStrokeGroupPane(cls, sg, pane):
		newSgTargetPane=pane
		sgInfoPane=sg.getInfoPane()

		strokeList=[s.generateCopyToApplyNewPane(sgInfoPane, newSgTargetPane) for s in sg.getStrokeList()]

		strokeGroupInfo=StrokeGroupInfo.generateInstance(strokeList)

		infoPane=newSgTargetPane
		strokeGroup=StrokeGroup.generateInstanceByInfo(strokeGroupInfo, infoPane)

		return strokeGroup

	@classmethod
	def generateInstanceByStrokeGroupPanePairList(cls, strokeGroupPanePairList):
		def computeBBox(paneList):
			left=min(map(lambda pane: pane.getLeft(), paneList))
			top=min(map(lambda pane: pane.getTop(), paneList))
			right=max(map(lambda pane: pane.getRight(), paneList))
			bottom=max(map(lambda pane: pane.getBottom(), paneList))
			return Pane(left, top, right, bottom)

		resultStrokeList=[]
		paneList=[]
		for strokeGroup, pane in strokeGroupPanePairList:
			strokeGroup=StrokeGroup.generateInstanceByStrokeGroupPane(strokeGroup, pane)
			resultStrokeList.extend(strokeGroup.getStrokeList())
			paneList.append(strokeGroup.getInfoPane())

		pane=computeBBox(paneList)
		strokeGroupInfo=StrokeGroupInfo.generateInstance(resultStrokeList, pane)
		strokeGroup=StrokeGroup.generateInstanceByInfo(strokeGroupInfo)

		return strokeGroup

class Character(Shape):
	def __init__(self, name, strokeGroup, tag=None):
		self.name = name
		self.strokeGroup = strokeGroup
		self.tag = tag

	def getName(self):
		return self.name

	def getStrokeGroup(self):
		return self.strokeGroup

	def getTag(self):
		return self.tag

	def draw(self, drawingSystem):
		character=self

		drawingSystem.clear()
		drawingSystem.onPreDrawCharacter(character)
		drawingSystem.draw(character.getStrokeGroup())
		drawingSystem.onPostDrawCharacter(character)

<commit_msg>Refactoring. StrokeGroupInfo.generateInstance() by removing the parameter, pane.<commit_after>from .shape import Shape
from .shape import Pane

from .stroke_info import StrokeInfoFactory

strokeInfoFactory=StrokeInfoFactory()

class Stroke(Shape):
	def __init__(self, startPoint, strokeInfo, statePane=None):
		self.startPoint=startPoint
		self.strokeInfo=strokeInfo

		strokePath=strokeInfo.getStrokePath()
		boundary=strokePath.computeBoundaryWithStartPoint(startPoint)

		self.infoPane = Pane(*boundary)
		if statePane:
			self.statePane=statePane
		else:
			self.statePane=self.infoPane

	def getStartPoint(self):
		return self.startPoint

	def getStrokeInfo(self):
		return self.strokeInfo

	def getName(self):
		return self.getStrokeInfo().getName()

	def getStrokePath(self):
		return self.getStrokeInfo().getStrokePath()

	def getInfoPane(self):
		return self.infoPane

	def getStatePane(self):
		return self.statePane

	def draw(self, drawingSystem):
		startPoint = self.getStartPoint()

		stroke=self
		drawingSystem.onPreDrawStroke(stroke)
		drawingSystem.setPane(stroke.getInfoPane(), stroke.getStatePane())

		drawingSystem.startDrawing(startPoint)
		drawingSystem.draw(stroke.getStrokePath())
		drawingSystem.endDrawing()
		drawingSystem.onPostDrawStroke(stroke)

	def computeBoundary(self):
		startPoint=self.getStartPoint()
		strokePath=self.getStrokePath()
		return strokePath.computeBoundaryWithStartPoint(startPoint)

	def generateCopyToApplyNewPane(self, sgTargetPane, newSgTargetPane):
		sTargetPane=self.getStatePane()
		newSTargetPane=sgTargetPane.transformRelativePaneByTargetPane(sTargetPane, newSgTargetPane)

		strokeCopy=Stroke(self.startPoint, self.strokeInfo, newSTargetPane)
		return strokeCopy

def generateStroke(name, startPoint, parameterList):
	strokeInfo = strokeInfoFactory.generateStrokeInfo(name, parameterList)
	return Stroke(startPoint, strokeInfo)

class StrokeGroupInfo:
	def __init__(self, strokeList, statePane=None):
		self.strokeList=strokeList

		if not statePane:
			def mergeBoundaryList(boundaryList):
				from xie.graphics.shape import mergeBoundary
				r = boundaryList[0]
				for b in boundaryList[1:]:
					r = mergeBoundary(r, b)
				return r

			boundaryList=[stroke.computeBoundary() for stroke in strokeList]
			bBox=mergeBoundaryList(boundaryList)
			statePane=Pane(*bBox)

		self.statePane=statePane

	def getStrokeList(self):
		return self.strokeList

	def getStatePane(self):
		return self.statePane

	@classmethod
	def generateInstance(cls, strokeList, pane=None):
		return StrokeGroupInfo(strokeList, pane)

class StrokeGroup(Shape):
	def __init__(self, strokeGroupInfo, infoPane):
		self.strokeGroupInfo=strokeGroupInfo

		self.infoPane=infoPane
		self.statePane=infoPane

	def getInfoPane(self):
		return self.infoPane

	def getStrokeList(self):
		return self.strokeGroupInfo.getStrokeList()

	def getCount(self):
		return len(self.getStrokeList())

	def draw(self, drawingSystem):
		strokeList = self.getStrokeList();

		for stroke in strokeList:
			stroke.draw(drawingSystem)

	@classmethod
	def generateInstanceByInfo(cls, strokeGroupInfo, infoPane=None):
		if not infoPane:
			infoPane=strokeGroupInfo.getStatePane()
		return StrokeGroup(strokeGroupInfo, infoPane)

	@classmethod
	def generateInstanceByStrokeList(cls, strokeList, pane=None):
		strokeGroupInfo=StrokeGroupInfo.generateInstance(strokeList)
		return StrokeGroup.generateInstanceByInfo(strokeGroupInfo, pane)

	@classmethod
	def generateInstanceByStrokeGroupPane(cls, sg, pane):
		newSgTargetPane=pane
		sgInfoPane=sg.getInfoPane()

		strokeList=[s.generateCopyToApplyNewPane(sgInfoPane, newSgTargetPane) for s in sg.getStrokeList()]

		strokeGroupInfo=StrokeGroupInfo.generateInstance(strokeList)

		infoPane=newSgTargetPane
		strokeGroup=StrokeGroup.generateInstanceByInfo(strokeGroupInfo, infoPane)

		return strokeGroup

	@classmethod
	def generateInstanceByStrokeGroupPanePairList(cls, strokeGroupPanePairList):
		def computeBBox(paneList):
			left=min(map(lambda pane: pane.getLeft(), paneList))
			top=min(map(lambda pane: pane.getTop(), paneList))
			right=max(map(lambda pane: pane.getRight(), paneList))
			bottom=max(map(lambda pane: pane.getBottom(), paneList))
			return Pane(left, top, right, bottom)

		resultStrokeList=[]
		paneList=[]
		for strokeGroup, pane in strokeGroupPanePairList:
			strokeGroup=StrokeGroup.generateInstanceByStrokeGroupPane(strokeGroup, pane)
			resultStrokeList.extend(strokeGroup.getStrokeList())
			paneList.append(strokeGroup.getInfoPane())

		pane=computeBBox(paneList)
		strokeGroupInfo=StrokeGroupInfo.generateInstance(resultStrokeList)
		strokeGroup=StrokeGroup.generateInstanceByInfo(strokeGroupInfo, pane)

		return strokeGroup

class Character(Shape):
	def __init__(self, name, strokeGroup, tag=None):
		self.name = name
		self.strokeGroup = strokeGroup
		self.tag = tag

	def getName(self):
		return self.name

	def getStrokeGroup(self):
		return self.strokeGroup

	def getTag(self):
		return self.tag

	def draw(self, drawingSystem):
		character=self

		drawingSystem.clear()
		drawingSystem.onPreDrawCharacter(character)
		drawingSystem.draw(character.getStrokeGroup())
		drawingSystem.onPostDrawCharacter(character)

<|endoftext|>


<commit_before><commit_msg>Linting fixes<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function, unicode_literals
import datetime as dt
from Tkinter import Tk
import tkFileDialog
import openpyxl as op
import argparse
import os.path
import sys
import re
import csv

__metaclass__ = type

class RFC4180(csv.Dialect):
    def __init__(self):
        csv.Dialect.__init__(self)
    delimiter = b','
    doublequote = True
    escapechar = None
    lineterminator = b'\r\n'
    quotechar = b'"'
    quoting = csv.QUOTE_MINIMAL
    skipinitialspace = False
    stric = True

def parseargs():
    pa = argparse.ArgumentParser(description=
            'Exports multiple CSV files from an Excel *.xlsx Workbook')
    pa.add_argument('-f', metavar='EXCELFILE',
                    help='The Excel file to export')
    pa.add_argument('-o', metavar='OUTPUTDIRECTORY',
                    help='The output directory, default is the current directory')
    args = pa.parse_args(sys.argv[1:])
    return vars(args)

def _stringify(dat):
    if not isinstance(dat, basestring):
        return str(dat).encode('utf-8')
    else:
        return dat.encode('utf-8')

def _transmap(dat):
    transmap = {
        # empty cells are going to be empty strings
        None: '',
        # workaround for bug in openpyxl
        # https://bitbucket.org/openpyxl/openpyxl/issues/674/
        dt.datetime(1899, 12, 30, 0, 0): dt.time(0, 0),
        dt.datetime(1899, 12, 31, 0, 0): dt.datetime(1900, 1, 1, 0, 0),
    }
    return transmap[dat] if dat in transmap else dat

def _datefix(dat):
    # if typ is datetime.datetime and time-part is 0:0:0,
    # covert to datetime.date (assume xlsx cell-type is "Date").
    if (type(dat) == dt.datetime and
        (dat.hour, dat.minute, dat.second) == (0, 0, 0)):
        dat = dat.date()
    return dat

def transform(l):
    l = [_transmap(f) for f in l]
    l = [_datefix(f) for f in l]
    l = [_stringify(f) for f in l]
    return l

def write_csv(data, outfile):
    with open(outfile, 'wb') as fout:
        writer = csv.writer(fout, dialect='RFC4180')
        writer.writerows(data)

def main():
    csv.register_dialect(u'RFC4180', RFC4180)
    home = os.path.expanduser('~')
    xlsxfile = parseargs()['f']
    out_dir = parseargs()['o']
    if xlsxfile is None:
        root = Tk()
        root.withdraw()
        with tkFileDialog.askopenfile(title='Choose file to convert',
                                      filetypes=[('xlsx', '*.xlsx')],
                                      initialdir=home) as f:
            xlsxfile = f.name
        if out_dir is None:
            out_dir = tkFileDialog.askdirectory(title='Choose output directory',
                                                initialdir=home)
        root.destroy()
    if out_dir is None:
        out_dir = os.getcwd()
    out_prefix = os.path.splitext(os.path.basename(xlsxfile))[0]
    wb = op.load_workbook(xlsxfile, data_only=True)
    for sn in wb.sheetnames:
        outfile = os.path.join(out_dir, out_prefix + '_' +
                               re.sub(r'\s+', '_', sn) + '.csv')
        data = []
        sheet = wb.get_sheet_by_name(sn)
        for l in sheet.values:
            data.append(transform(l))
        write_csv(data, outfile)

if __name__ == '__main__':
    main()

<commit_msg>fix cancel - behaviour<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import, division, print_function, unicode_literals
import datetime as dt
from Tkinter import Tk
import tkFileDialog
import openpyxl as op
import argparse
import os.path
import sys
import re
import csv

__metaclass__ = type

class RFC4180(csv.Dialect):
    def __init__(self):
        csv.Dialect.__init__(self)
    delimiter = b','
    doublequote = True
    escapechar = None
    lineterminator = b'\r\n'
    quotechar = b'"'
    quoting = csv.QUOTE_MINIMAL
    skipinitialspace = False
    stric = True

def parseargs():
    pa = argparse.ArgumentParser(description=
            'Exports multiple CSV files from an Excel *.xlsx Workbook')
    pa.add_argument('-f', metavar='EXCELFILE',
                    help='The Excel file to export. ' +
                    'If omitted, a graphical file chooser will be used.')
    pa.add_argument('-o', metavar='OUTPUTDIRECTORY',
                    help='The output directory. Default is the current ' +
                    'directory if EXCELFILE was given, otherwise a ' +
                    'file chooser will be used as well.')
    args = pa.parse_args(sys.argv[1:])
    return vars(args)

def _stringify(dat):
    if not isinstance(dat, basestring):
        return str(dat).encode('utf-8')
    else:
        return dat.encode('utf-8')

def _transmap(dat):
    transmap = {
        # empty cells are going to be empty strings
        None: '',
        # workaround for bug in openpyxl
        # https://bitbucket.org/openpyxl/openpyxl/issues/674/
        dt.datetime(1899, 12, 30, 0, 0): dt.time(0, 0),
        dt.datetime(1899, 12, 31, 0, 0): dt.datetime(1900, 1, 1, 0, 0),
    }
    return transmap[dat] if dat in transmap else dat

def _datefix(dat):
    # if typ is datetime.datetime and time-part is 0:0:0,
    # covert to datetime.date (assume xlsx cell-type is "Date").
    if (type(dat) == dt.datetime and
        (dat.hour, dat.minute, dat.second) == (0, 0, 0)):
        dat = dat.date()
    return dat

def transform(l):
    l = [_transmap(f) for f in l]
    l = [_datefix(f) for f in l]
    l = [_stringify(f) for f in l]
    return l

def write_csv(data, outfile):
    with open(outfile, 'wb') as fout:
        writer = csv.writer(fout, dialect='RFC4180')
        writer.writerows(data)

def main():
    csv.register_dialect(u'RFC4180', RFC4180)
    home = os.path.expanduser('~')
    xlsxfile = parseargs()['f']
    out_dir = parseargs()['o']
    if xlsxfile is None:
        root = Tk()
        root.withdraw()
        f = tkFileDialog.askopenfile(title='Choose file to convert',
                                      filetypes=[('xlsx', '*.xlsx')],
                                      initialdir=home)
        if f:
            xlsxfile = f.name
            f.close()
        else:
            sys.exit()
        if out_dir is None:
            out_dir = tkFileDialog.askdirectory(title='Choose output directory',
                                                initialdir=home)
            if not out_dir:
                sys.exit()
        root.destroy()
    if not out_dir:
        out_dir = os.getcwd()
    out_prefix = os.path.splitext(os.path.basename(xlsxfile))[0]
    wb = op.load_workbook(xlsxfile, data_only=True)
    for sn in wb.sheetnames:
        outfile = os.path.join(out_dir, out_prefix + '_' +
                               re.sub(r'\s+', '_', sn) + '.csv')
        data = []
        sheet = wb.get_sheet_by_name(sn)
        for l in sheet.values:
            data.append(transform(l))
        write_csv(data, outfile)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from .conftest import read_json_sample

steam_ids = [
    "76561198043212328",  # shire
    "76561198257384619",  # HanzoHasashiSan
    "76561197985202252",  # carecry
    "76561198308265738",  # lookaround
    "76561198257327401",  # Jalepeno
    "76561198005116094",  # Xaero
    "76561198077231066",  # Mike_Litoris
    "76561198346199541",  # Zigurun
    "76561198257338624",  # indie
    "76561198260599288",  # eugene
]

def test_player_json(service):
    for steam_id in steam_ids:
        resp = service.get("/player/{0}.json".format(steam_id))

        obj_defacto = resp.json()
        obj_expected = read_json_sample("player_{}".format(steam_id))
        assert obj_defacto == obj_expected

        resp = service.get("/player/{0}".format(steam_id))
        assert resp.template.name == "player_stats.html"
        context = resp.context
        assert "request" in context
        assert "steam_id" in context
        assert context["steam_id"] == steam_id

        del context["request"]
        del context["steam_id"]
        obj_defacto = context
        assert obj_defacto == obj_expected

# TODO: use steam_ids as params
def test_deprecated_player_json(service):
    for steam_id in steam_ids:
        resp = service.get("/deprecated/player/{0}.json".format(steam_id))
        assert resp.json() == \
            read_json_sample("deprecated_player_{0}".format(steam_id))
<commit_msg>tests: parametrized tests in test_player<commit_after>from pytest import mark

from .conftest import read_json_sample

steam_ids = [
    "76561198043212328",  # shire
    "76561198257384619",  # HanzoHasashiSan
    "76561197985202252",  # carecry
    "76561198308265738",  # lookaround
    "76561198257327401",  # Jalepeno
    "76561198005116094",  # Xaero
    "76561198077231066",  # Mike_Litoris
    "76561198346199541",  # Zigurun
    "76561198257338624",  # indie
    "76561198260599288",  # eugene
]

@mark.parametrize("steam_id", steam_ids)
def test_player_json(service, steam_id):
    resp = service.get("/player/{0}.json".format(steam_id))

    obj_defacto = resp.json()
    obj_expected = read_json_sample("player_{}".format(steam_id))
    assert obj_defacto == obj_expected

    resp = service.get("/player/{0}".format(steam_id))
    assert resp.template.name == "player_stats.html"
    context = resp.context
    assert "request" in context
    assert "steam_id" in context
    assert context["steam_id"] == steam_id

    del context["request"]
    del context["steam_id"]
    obj_defacto = context
    assert obj_defacto == obj_expected

@mark.parametrize("steam_id", steam_ids)
def test_deprecated_player_json(service, steam_id):
    resp = service.get("/deprecated/player/{0}.json".format(steam_id))
    assert resp.json() == read_json_sample("deprecated_player_{0}".format(steam_id))
<|endoftext|>


<commit_before>import os
import sys
import platform
import shutil
import subprocess as sp
import warnings
import socket
import time
from datetime import datetime
from ..pyemu_warnings import PyemuWarning

def remove_readonly(func, path, excinfo):
    """remove readonly dirs, apparently only a windows issue
    add to all rmtree calls: shutil.rmtree(**,onerror=remove_readonly), wk"""
    os.chmod(path, 128) #stat.S_IWRITE==128==normal
    func(path)

def run(cmd_str,cwd='.',verbose=False):
    """ an OS agnostic function to execute a command line

    Parameters
    ----------
    cmd_str : str
        the str to execute with os.system()

    cwd : str
        the directory to execute the command in

    verbose : bool
        flag to echo to stdout complete cmd str

    Note
    ----
    uses platform to detect OS and adds .exe suffix or ./ prefix as appropriate

    for Windows, if os.system returns non-zero, raises exception

    Example
    -------
    ``>>>import pyemu``

    ``>>>pyemu.helpers.run("pestpp pest.pst")``

    """
    bwd = os.getcwd()
    os.chdir(cwd)
    try:
        exe_name = cmd_str.split()[0]
        if "window" in platform.platform().lower():
            if not exe_name.lower().endswith("exe"):
                raw = cmd_str.split()
                raw[0] = exe_name + ".exe"
                cmd_str = ' '.join(raw)
        else:
            if exe_name.lower().endswith('exe'):
                raw = cmd_str.split()
                exe_name = exe_name.replace('.exe','')
                raw[0] = exe_name
                cmd_str = '{0} {1} '.format(*raw)
            if os.path.exists(exe_name) and not exe_name.startswith('./'):
                cmd_str = "./" + cmd_str

    except Exception as e:
        os.chdir(bwd)
        raise Exception("run() error preprocessing command line :{0}".format(str(e)))
    if verbose:
        print("run():{0}".format(cmd_str))

    try:
        ret_val = os.system(cmd_str)
    except Exception as e:
        os.chdir(bwd)
        raise Exception("run() raised :{0}".format(str(e)))
    os.chdir(bwd)
    if "window" in platform.platform().lower():
        if ret_val != 0:
            raise Exception("run() returned non-zero")

def start_slaves(slave_dir,exe_rel_path,pst_rel_path,num_slaves=None,slave_root="..",
                 port=4004,rel_path=None,local=True,cleanup=True,master_dir=None,
                 verbose=False,silent_master=False):
    """ start a group of pest(++) slaves on the local machine

    Parameters
    ----------
    slave_dir :  str
        the path to a complete set of input files
    exe_rel_path : str
        the relative path to the pest(++) executable from within the slave_dir
    pst_rel_path : str
        the relative path to the pst file from within the slave_dir
    num_slaves : int
        number of slaves to start. defaults to number of cores
    slave_root : str
        the root to make the new slave directories in
    rel_path: str
        the relative path to where pest(++) should be run from within the
        slave_dir, defaults to the uppermost level of the slave dir
    local: bool
        flag for using "localhost" instead of hostname on slave command line
    cleanup: bool
        flag to remove slave directories once processes exit
    master_dir: str
        name of directory for master instance.  If master_dir
        exists, then it will be removed.  If master_dir is None,
        no master instance will be started
    verbose : bool
        flag to echo useful information to stdout
    silent_master : bool
        flag to pipe master output to devnull.  This is only for
        pestpp Travis testing. Default is False

    Note
    ----
    if all slaves (and optionally master) exit gracefully, then the slave
    dirs will be removed unless cleanup is false

    Example
    -------
    ``>>>import pyemu``

    start 10 slaves using the directory "template" as the base case and
    also start a master instance in a directory "master".

    ``>>>pyemu.helpers.start_slaves("template","pestpp","pest.pst",10,master_dir="master")``

    """

    assert os.path.isdir(slave_dir)
    assert os.path.isdir(slave_root)
    if num_slaves is None:
        num_slaves = mp.cpu_count()
    else:
        num_slaves = int(num_slaves)
    #assert os.path.exists(os.path.join(slave_dir,rel_path,exe_rel_path))
    exe_verf = True

    if rel_path:
        if not os.path.exists(os.path.join(slave_dir,rel_path,exe_rel_path)):
            #print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
            exe_verf = False
    else:
        if not os.path.exists(os.path.join(slave_dir,exe_rel_path)):
            #print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
            exe_verf = False
    if rel_path is not None:
        assert os.path.exists(os.path.join(slave_dir,rel_path,pst_rel_path))
    else:
        assert os.path.exists(os.path.join(slave_dir,pst_rel_path))
    if local:
        hostname = "localhost"
    else:
        hostname = socket.gethostname()

    base_dir = os.getcwd()
    port = int(port)

    if os.path.exists(os.path.join(slave_dir,exe_rel_path)):
        if "window" in platform.platform().lower():
            if not exe_rel_path.lower().endswith("exe"):
                exe_rel_path = exe_rel_path + ".exe"
        else:
            if not exe_rel_path.startswith('./'):
                exe_rel_path = "./" + exe_rel_path

    if master_dir is not None:
        if master_dir != '.' and os.path.exists(master_dir):
            try:
                shutil.rmtree(master_dir,onerror=remove_readonly)#, onerror=del_rw)
            except Exception as e:
                raise Exception("unable to remove existing master dir:" + \
                                "{0}\n{1}".format(master_dir,str(e)))
        if master_dir != '.':
            try:
                shutil.copytree(slave_dir,master_dir)
            except Exception as e:
                raise Exception("unable to copy files from base slave dir: " + \
                                "{0} to master dir: {1}\n{2}".\
                                format(slave_dir,master_dir,str(e)))

        args = [exe_rel_path, pst_rel_path, "/h", ":{0}".format(port)]
        if rel_path is not None:
            cwd = os.path.join(master_dir,rel_path)
        else:
            cwd = master_dir
        if verbose:
            print("master:{0} in {1}".format(' '.join(args),cwd))
        stdout=None
        if silent_master:
            stdout = open(os.devnull,'w')
        try:
            os.chdir(cwd)
            master_p = sp.Popen(args,stdout=stdout)#,stdout=sp.PIPE,stderr=sp.PIPE)
            os.chdir(base_dir)
        except Exception as e:
            raise Exception("error starting master instance: {0}".\
                            format(str(e)))
        time.sleep(1.5) # a few cycles to let the master get ready

    tcp_arg = "{0}:{1}".format(hostname,port)
    procs = []
    slave_dirs = []
    for i in range(num_slaves):
        new_slave_dir = os.path.join(slave_root,"slave_{0}".format(i))
        if os.path.exists(new_slave_dir):
            try:
                shutil.rmtree(new_slave_dir,onerror=remove_readonly)#, onerror=del_rw)
            except Exception as e:
                raise Exception("unable to remove existing slave dir:" + \
                                "{0}\n{1}".format(new_slave_dir,str(e)))
        try:
            shutil.copytree(slave_dir,new_slave_dir)
        except Exception as e:
            raise Exception("unable to copy files from slave dir: " + \
                            "{0} to new slave dir: {1}\n{2}".format(slave_dir,new_slave_dir,str(e)))
        try:
            if exe_verf:
                # if rel_path is not None:
                #     exe_path = os.path.join(rel_path,exe_rel_path)
                # else:
                exe_path = exe_rel_path
            else:
                exe_path = exe_rel_path
            args = [exe_path, pst_rel_path, "/h", tcp_arg]
            #print("starting slave in {0} with args: {1}".format(new_slave_dir,args))
            if rel_path is not None:
                cwd = os.path.join(new_slave_dir,rel_path)
            else:
                cwd = new_slave_dir

            os.chdir(cwd)
            if verbose:
                print("slave:{0} in {1}".format(' '.join(args),cwd))
            with open(os.devnull,'w') as f:
                p = sp.Popen(args,stdout=f,stderr=f)
            procs.append(p)
            os.chdir(base_dir)
        except Exception as e:
            raise Exception("error starting slave: {0}".format(str(e)))
        slave_dirs.append(new_slave_dir)

    if master_dir is not None:
        # while True:
        #     line = master_p.stdout.readline()
        #     if line != '':
        #         print(str(line.strip())+'\r',end='')
        #     if master_p.poll() is not None:
        #         print(master_p.stdout.readlines())
        #         break
        if silent_master:
            # this keeps travis from thinking something is wrong...
            while True:
                rv = master_p.poll()
                if master_p.poll() is not None:
                    break
                print(datetime.now(), "still running")
                time.sleep(5)
        else:
            master_p.wait()
            time.sleep(1.5) # a few cycles to let the slaves end gracefully
        # kill any remaining slaves
        for p in procs:
            p.kill()
    # this waits for sweep to finish, but pre/post/model (sub)subprocs may take longer
    for p in procs:
        p.wait()
    if cleanup:
        cleanit=0
        while len(slave_dirs)>0 and cleanit<100000: # arbitrary 100000 limit
            cleanit=cleanit+1
            for d in slave_dirs:
                try:
                    shutil.rmtree(d,onerror=remove_readonly)
                    slave_dirs.pop(slave_dirs.index(d)) #if successfully removed
                except Exception as e:
                    warnings.warn("unable to remove slavr dir{0}:{1}".format(d,str(e)),PyemuWarning)

<commit_msg>initial implementation done, now for testing<commit_after>import os
import sys
import platform
import shutil
import subprocess as sp
import warnings
import socket
import time
from datetime import datetime
from ..pyemu_warnings import PyemuWarning

def remove_readonly(func, path, excinfo):
    """remove readonly dirs, apparently only a windows issue
    add to all rmtree calls: shutil.rmtree(**,onerror=remove_readonly), wk"""
    os.chmod(path, 128) #stat.S_IWRITE==128==normal
    func(path)

def run_sweep(pe,template_dir,pst_name=None,num_slaves=10,local=True,
              binary=False,master_dir="master_runsweep",cleanup=True):

    if pst_name is not  None:
        assert os.path.exists(os.path.join(template_dir,pst_name))
    else:
        pst_files = [f for f in os.listdir(template_dir) if f.lower().endswith(".pst")]
        if len(pst_files) > 1:
            raise Exception("run_sweep() error: 'pst_name' is None "+
                            "but more than one '.pst' file found in 'template_dir'")
        if len(pst_files) == 0:
            raise Exception("run_sweep() error: 'pst_name' is None and"+\
                            " no '.pst' files in 'template_dir'")
        pst_file = pst_files[0]

    # todo: add autodetect to pestpp-swp for sweep_in.jcb
    if binary:
        raise NotImplementedError("pestpp-swp doesn't support autodetect for binary yet")
        pe.to_binary(os.path.join(template_dir,"sweep_in.jcb"))
    if not binary:
        pe.to_csv(os.path.join(template_dir,"sweep_in.csv"))
    if not local:
        raise NotImplementedError("condor not supported yet")
    else:
        start_slaves(template_dir,"pestpp-swp",pst_name,num_slaves=num_slaves,slave_root=".",
                     master_dir=master_dir)

    assert os.path.exists(master_dir,"sweep_out.csv")
    df = pd.read_csv(os.path.join(master_dir,"sweep_out.csv"),index_col=0)
    df.columns = df.columns.map(str.lower)
    if cleanup:
        shutil.rmtree(master_dir)

    return df

def run(cmd_str,cwd='.',verbose=False):
    """ an OS agnostic function to execute a command line

    Parameters
    ----------
    cmd_str : str
        the str to execute with os.system()

    cwd : str
        the directory to execute the command in

    verbose : bool
        flag to echo to stdout complete cmd str

    Note
    ----
    uses platform to detect OS and adds .exe suffix or ./ prefix as appropriate

    for Windows, if os.system returns non-zero, raises exception

    Example
    -------
    ``>>>import pyemu``

    ``>>>pyemu.helpers.run("pestpp pest.pst")``

    """
    bwd = os.getcwd()
    os.chdir(cwd)
    try:
        exe_name = cmd_str.split()[0]
        if "window" in platform.platform().lower():
            if not exe_name.lower().endswith("exe"):
                raw = cmd_str.split()
                raw[0] = exe_name + ".exe"
                cmd_str = ' '.join(raw)
        else:
            if exe_name.lower().endswith('exe'):
                raw = cmd_str.split()
                exe_name = exe_name.replace('.exe','')
                raw[0] = exe_name
                cmd_str = '{0} {1} '.format(*raw)
            if os.path.exists(exe_name) and not exe_name.startswith('./'):
                cmd_str = "./" + cmd_str

    except Exception as e:
        os.chdir(bwd)
        raise Exception("run() error preprocessing command line :{0}".format(str(e)))
    if verbose:
        print("run():{0}".format(cmd_str))

    try:
        ret_val = os.system(cmd_str)
    except Exception as e:
        os.chdir(bwd)
        raise Exception("run() raised :{0}".format(str(e)))
    os.chdir(bwd)
    if "window" in platform.platform().lower():
        if ret_val != 0:
            raise Exception("run() returned non-zero")

def start_slaves(slave_dir,exe_rel_path,pst_rel_path,num_slaves=None,slave_root="..",
                 port=4004,rel_path=None,local=True,cleanup=True,master_dir=None,
                 verbose=False,silent_master=False):
    """ start a group of pest(++) slaves on the local machine

    Parameters
    ----------
    slave_dir :  str
        the path to a complete set of input files
    exe_rel_path : str
        the relative path to the pest(++) executable from within the slave_dir
    pst_rel_path : str
        the relative path to the pst file from within the slave_dir
    num_slaves : int
        number of slaves to start. defaults to number of cores
    slave_root : str
        the root to make the new slave directories in
    rel_path: str
        the relative path to where pest(++) should be run from within the
        slave_dir, defaults to the uppermost level of the slave dir
    local: bool
        flag for using "localhost" instead of hostname on slave command line
    cleanup: bool
        flag to remove slave directories once processes exit
    master_dir: str
        name of directory for master instance.  If master_dir
        exists, then it will be removed.  If master_dir is None,
        no master instance will be started
    verbose : bool
        flag to echo useful information to stdout
    silent_master : bool
        flag to pipe master output to devnull.  This is only for
        pestpp Travis testing. Default is False

    Note
    ----
    if all slaves (and optionally master) exit gracefully, then the slave
    dirs will be removed unless cleanup is false

    Example
    -------
    ``>>>import pyemu``

    start 10 slaves using the directory "template" as the base case and
    also start a master instance in a directory "master".

    ``>>>pyemu.helpers.start_slaves("template","pestpp","pest.pst",10,master_dir="master")``

    """

    assert os.path.isdir(slave_dir)
    assert os.path.isdir(slave_root)
    if num_slaves is None:
        num_slaves = mp.cpu_count()
    else:
        num_slaves = int(num_slaves)
    #assert os.path.exists(os.path.join(slave_dir,rel_path,exe_rel_path))
    exe_verf = True

    if rel_path:
        if not os.path.exists(os.path.join(slave_dir,rel_path,exe_rel_path)):
            #print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
            exe_verf = False
    else:
        if not os.path.exists(os.path.join(slave_dir,exe_rel_path)):
            #print("warning: exe_rel_path not verified...hopefully exe is in the PATH var")
            exe_verf = False
    if rel_path is not None:
        assert os.path.exists(os.path.join(slave_dir,rel_path,pst_rel_path))
    else:
        assert os.path.exists(os.path.join(slave_dir,pst_rel_path))
    if local:
        hostname = "localhost"
    else:
        hostname = socket.gethostname()

    base_dir = os.getcwd()
    port = int(port)

    if os.path.exists(os.path.join(slave_dir,exe_rel_path)):
        if "window" in platform.platform().lower():
            if not exe_rel_path.lower().endswith("exe"):
                exe_rel_path = exe_rel_path + ".exe"
        else:
            if not exe_rel_path.startswith('./'):
                exe_rel_path = "./" + exe_rel_path

    if master_dir is not None:
        if master_dir != '.' and os.path.exists(master_dir):
            try:
                shutil.rmtree(master_dir,onerror=remove_readonly)#, onerror=del_rw)
            except Exception as e:
                raise Exception("unable to remove existing master dir:" + \
                                "{0}\n{1}".format(master_dir,str(e)))
        if master_dir != '.':
            try:
                shutil.copytree(slave_dir,master_dir)
            except Exception as e:
                raise Exception("unable to copy files from base slave dir: " + \
                                "{0} to master dir: {1}\n{2}".\
                                format(slave_dir,master_dir,str(e)))

        args = [exe_rel_path, pst_rel_path, "/h", ":{0}".format(port)]
        if rel_path is not None:
            cwd = os.path.join(master_dir,rel_path)
        else:
            cwd = master_dir
        if verbose:
            print("master:{0} in {1}".format(' '.join(args),cwd))
        stdout=None
        if silent_master:
            stdout = open(os.devnull,'w')
        try:
            os.chdir(cwd)
            master_p = sp.Popen(args,stdout=stdout)#,stdout=sp.PIPE,stderr=sp.PIPE)
            os.chdir(base_dir)
        except Exception as e:
            raise Exception("error starting master instance: {0}".\
                            format(str(e)))
        time.sleep(1.5) # a few cycles to let the master get ready

    tcp_arg = "{0}:{1}".format(hostname,port)
    procs = []
    slave_dirs = []
    for i in range(num_slaves):
        new_slave_dir = os.path.join(slave_root,"slave_{0}".format(i))
        if os.path.exists(new_slave_dir):
            try:
                shutil.rmtree(new_slave_dir,onerror=remove_readonly)#, onerror=del_rw)
            except Exception as e:
                raise Exception("unable to remove existing slave dir:" + \
                                "{0}\n{1}".format(new_slave_dir,str(e)))
        try:
            shutil.copytree(slave_dir,new_slave_dir)
        except Exception as e:
            raise Exception("unable to copy files from slave dir: " + \
                            "{0} to new slave dir: {1}\n{2}".format(slave_dir,new_slave_dir,str(e)))
        try:
            if exe_verf:
                # if rel_path is not None:
                #     exe_path = os.path.join(rel_path,exe_rel_path)
                # else:
                exe_path = exe_rel_path
            else:
                exe_path = exe_rel_path
            args = [exe_path, pst_rel_path, "/h", tcp_arg]
            #print("starting slave in {0} with args: {1}".format(new_slave_dir,args))
            if rel_path is not None:
                cwd = os.path.join(new_slave_dir,rel_path)
            else:
                cwd = new_slave_dir

            os.chdir(cwd)
            if verbose:
                print("slave:{0} in {1}".format(' '.join(args),cwd))
            with open(os.devnull,'w') as f:
                p = sp.Popen(args,stdout=f,stderr=f)
            procs.append(p)
            os.chdir(base_dir)
        except Exception as e:
            raise Exception("error starting slave: {0}".format(str(e)))
        slave_dirs.append(new_slave_dir)

    if master_dir is not None:
        # while True:
        #     line = master_p.stdout.readline()
        #     if line != '':
        #         print(str(line.strip())+'\r',end='')
        #     if master_p.poll() is not None:
        #         print(master_p.stdout.readlines())
        #         break
        if silent_master:
            # this keeps travis from thinking something is wrong...
            while True:
                rv = master_p.poll()
                if master_p.poll() is not None:
                    break
                print(datetime.now(), "still running")
                time.sleep(5)
        else:
            master_p.wait()
            time.sleep(1.5) # a few cycles to let the slaves end gracefully
        # kill any remaining slaves
        for p in procs:
            p.kill()
    # this waits for sweep to finish, but pre/post/model (sub)subprocs may take longer
    for p in procs:
        p.wait()
    if cleanup:
        cleanit=0
        while len(slave_dirs)>0 and cleanit<100000: # arbitrary 100000 limit
            cleanit=cleanit+1
            for d in slave_dirs:
                try:
                    shutil.rmtree(d,onerror=remove_readonly)
                    slave_dirs.pop(slave_dirs.index(d)) #if successfully removed
                except Exception as e:
                    warnings.warn("unable to remove slavr dir{0}:{1}".format(d,str(e)),PyemuWarning)

<|endoftext|>


<commit_before>import datetime
from datetime import date
import discord
from discord.ext import commands

class xmasclock:
    """I did it!"""

    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def xmasclock(self):
        """Display days left 'til xmas"""

        now = datetime.datetime.now()
        today = date(now.year, now.month, now.day)
        xmasday = date(now.year, 12, 25)

        delta = xmasday - today

        await self.bot.say(delta.days + " days left until Xmas!")

def setup(bot):
    bot.add_cog(xmasclock(bot))
<commit_msg>cast int as string<commit_after>import datetime
from datetime import date
import discord
from discord.ext import commands

class xmasclock:
    """I did it!"""

    def __init__(self, bot):
        self.bot = bot

    @commands.command()
    async def xmasclock(self):
        """Display days left 'til xmas"""

        now = datetime.datetime.now()
        today = date(now.year, now.month, now.day)
        xmasday = date(now.year, 12, 25)

        delta = xmasday - today

        await self.bot.say(str(delta.days) + " days left until Xmas!")

def setup(bot):
    bot.add_cog(xmasclock(bot))
<|endoftext|>


<commit_before><commit_msg>black<commit_after><|endoftext|>


<commit_before>from twisted.trial import unittest

from ooni.inputunit import InputUnit
from ooni.nettest import NetTestCase
from ooni.reporter import OReporter

from ooni.runner import loadTestsAndOptions, runTestCasesWithInputUnit

class DummyTestCase(NetTestCase):
    def test_a(self):
        self.report['bar'] = 'bar'
    def test_b(self):
        self.report['foo'] = 'foo'

class DummyTestCasePP(DummyTestCase):
    def postProcessor(self, report):
        self.report['antani'] = 'sblinda'

class DummyReporter(OReporter):
    dummy_report = []
    def createReport(self, options):
        pass

    def writeReportEntry(self, entry):
        self.dummy_report.append(entry)

class TestRunner(unittest.TestCase):
    def test_load_test_and_options(self):
        input_unit = InputUnit([0,1,2,3,4])
        cmd_line_options = {}
        test_cases, options = loadTestsAndOptions([DummyTestCase],
                cmd_line_options)
        self.assertEqual(test_cases[0][1], 'test_b')
        self.assertEqual(test_cases[1][1], 'test_a')

    def test_run_testcase_with_input_unit(self):
        def done(result):
            report = oreporter.dummy_report
            self.assertEqual(len(report), 10*2)
            # XXX debug why this is failing
            # for idx, entry in enumerate(oreporter.dummy_report):
            #     if idx % 2 == 0:
            #         self.assertEqual(entry['report']['foo'], 'foo')
            #     else:
            #         self.assertEqual(entry['report']['bar'], 'bar')

        input_unit = InputUnit([0,1,2,3,4])
        cmd_line_options = {'collector': None}

        oreporter = DummyReporter(cmd_line_options)
        oreporter.dummy_report = []

        test_cases, options = loadTestsAndOptions([DummyTestCase],
                cmd_line_options)

        d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter)
        d.addBoth(done)
        return d

    def test_with_post_processing(self):
        def done(result):
            report = oreporter.dummy_report
            self.assertEqual(len(report), 6)
            for entry in report:
                if entry['test_name'] == 'summary':
                    self.assertEqual(entry['report'], {'antani': 'sblinda'})

        input_unit = InputUnit([None])
        cmd_line_options = {'collector': None}

        oreporter = DummyReporter(cmd_line_options)
        oreporter.dummy_report = []

        test_cases, options = loadTestsAndOptions([DummyTestCasePP],
                cmd_line_options)

        d = runTestCasesWithInputUnit(test_cases, input_unit, oreporter, oreporter)
        d.addBoth(done)
        return d
<commit_msg>Remove now superfluos runner unittest<commit_after><|endoftext|>