<commit_before>from ddapp import lcmUtils
from ddapp import objectmodel as om
from ddapp import visualization as vis
from ddapp.utime import getUtime
from ddapp import transformUtils
from ddapp import applogic as app

import numpy as np
import drc as lcmdrc

class FootstepsDriver(object):
    def __init__(self, jc):
        self.lastFootstepPlanMessage = None
        self._setupSubscriptions()
        self.jc = jc

    def _setupSubscriptions(self):
        lcmUtils.addSubscriber('FOOTSTEP_PLAN_RESPONSE', lcmdrc.footstep_plan_t, self.onFootstepPlan)

    def onFootstepPlan(self, msg):
        print "Got footstep plan with {:d} steps".format(msg.num_steps)

    def createWalkingGoal(self, model):
        distanceForward = 1.0

        t1 = model.getLinkFrame('l_foot')
        t2 = model.getLinkFrame('r_foot')
        pelvisT = model.getLinkFrame('pelvis')

        xaxis = [1.0, 0.0, 0.0]
        pelvisT.TransformVector(xaxis, xaxis)
        xaxis = np.array(xaxis)
        zaxis = np.array([0.0, 0.0, 1.0])
        yaxis = np.cross(zaxis, xaxis)
        xaxis = np.cross(yaxis, zaxis)

        stancePosition = np.array(t2.GetPosition()) + np.array(t1.GetPosition()) / 2.0

        footHeight=0.0817

        t = transformUtils.getTransformFromAxes(xaxis, yaxis, zaxis)
        t.PostMultiply()
        t.Translate(stancePosition)
        t.Translate([0.0, 0.0, -footHeight])
        t.Translate(xaxis*distanceForward)

        frameObj = vis.showFrame(t, 'walking goal')
        frameObj.setProperty('Edit', True)
        t.AddObserver('Modified', self.onWalkingGoalModified)
        print "created walking goal"

    def onWalkingGoalModified(self, transform, event):
        pos, wxyz = transformUtils.poseFromTransform(transform)
        print pos

    def sendFootstepPlanRequest(self):
        goalObj = om.findObjectByName('walking goal')
        assert goalObj

        msg = lcmdrc.footstep_plan_request_t()
        msg.utime = getUtime()
        pose = self.jc.getPose('q_end')
        state_msg = robotstate.drakePoseToRobotState(pose)
        msg.initial_state = state_msg

        msg.goal_pos = positionMessageFromFrame(goalObj.transform)
        msg.params = lcmdrc.footstep_plan_params_t()
        msg.params.max_num_steps = 100
        msg.params.min_num_steps = 0
        msg.params.min_step_width = 0.21
        msg.params.nom_step_width = 0.25
        msg.params.max_step_width = 0.4
        msg.params.nom_forward_step = 0.15
        msg.params.max_forward_step = 0.45
        msg.params.ignore_terrain = False
        msg.params.planning_mode = msg.MODE_AUTO
        msg.params.behavior = msg.BEHAVIOR_BDI_STEPPING
        msg.params.map_command = 2
        msg.params.leading_foot = msg.LEAD_AUTO

        msg.default_step_params = lcmdrc.footstep_params_t()
        msg.default_step_params.step_speed = 1.0
        msg.default_step_params.step_height = 0.05
        msg.default_step_params.bdi_step_duration = 2.0
        msg.default_step_params.bdi_sway_duration = 0.0
        msg.default_step_params.bdi_lift_height = 0.05
        msg.default_step_params.bdi_toe_off = 1
        msg.default_step_params.bdi_knee_nominal = 0.0
        msg.default_step_params.bdi_max_foot_vel = 0.0
        msg.default_step_params.bdi_sway_end_dist = 0.02
        msg.default_step_params.bdi_step_end_dist = 0.02
        msg.default_step_params.mu = 1.0

        lcmUtils.publish('FOOTSTEP_PLAN_REQUEST', msg)
        return msg

def init(jc):
    global driver
    driver = FootstepsDriver(jc)

<commit_msg>Add footstep plan rendering and single-step interaction to ddapp<commit_after>from ddapp import lcmUtils
from ddapp import objectmodel as om
from ddapp import visualization as vis
from ddapp.utime import getUtime
from ddapp import transformUtils
from ddapp.debugVis import DebugData
from ddapp import ioUtils
from ddapp import robotstate
from ddapp import applogic as app

import os
import numpy as np
import drc as lcmdrc
import functools

def loadFootMeshes():
    meshDir = os.path.join(app.getDRCBase(), 'software/models/mit_gazebo_models/mit_robot/meshes')
    meshes = []
    for foot in ['l', 'r']:
        d = DebugData()
        d.addPolyData(ioUtils.readPolyData(os.path.join(meshDir, '%s_talus.stl' % foot), computeNormals=True))
        d.addPolyData(ioUtils.readPolyData(os.path.join(meshDir, '%s_foot.stl' % foot), computeNormals=True))
        meshes.append(d.getPolyData())
    return meshes

def getLeftFootMesh():
    return getFootMeshes()[0]

def getRightFootMesh():
    return getFootMeshes()[1]

def getLeftFootColor():
    return [1.0, 1.0, 0.0]

def getRightFootColor():
    return [0.33, 1.0, 0.0]

_footMeshes = None

def getFootMeshes():
    global _footMeshes
    if not _footMeshes:
        _footMeshes = loadFootMeshes()
    return _footMeshes

def getFootstepsFolder():
    obj = om.findObjectByName('footstep plan')
    if obj is None:
        obj = om.getOrCreateContainer('footstep plan')
        #om.collapse(obj)
    return obj

class FootstepsDriver(object):
    def __init__(self, jc):
        self.lastFootstepPlanMessage = None
        self.has_plan = False
        self._setupSubscriptions()
        self.jc = jc

    def _setupSubscriptions(self):
        lcmUtils.addSubscriber('FOOTSTEP_PLAN_RESPONSE', lcmdrc.footstep_plan_t, self.onFootstepPlan)

    def clearFootstepPlan(self):
        self.has_plan = False
        folder = getFootstepsFolder()
        om.removeFromObjectModel(folder)

    def onFootstepPlan(self, msg):
        self.clearFootstepPlan()
        self.has_plan = True
        planFolder = getFootstepsFolder()

        for i, footstep in enumerate(msg.footsteps[2:]):
            trans = footstep.pos.translation
            trans = [trans.x, trans.y, trans.z]
            quat = footstep.pos.rotation
            quat = [quat.w, quat.x, quat.y, quat.z]
            footstepTransform = transformUtils.transformFromPose(trans, quat)

            if footstep.is_right_foot:
                mesh = getRightFootMesh()
                color = getRightFootColor()
            else:
                mesh = getLeftFootMesh()
                color = getLeftFootColor()

            obj = vis.showPolyData(mesh, 'step %d' % i, color=color, alpha=1.0, parent=planFolder)
            frameObj = vis.showFrame(footstepTransform, 'frame', parent=obj, scale=0.3, visible=False)
            frameObj.onTransformModifiedCallback = functools.partial(self.onStepModified, i)
            obj.actor.SetUserTransform(footstepTransform)

        self.lastFootstepPlanMessage = msg

    def createWalkingGoal(self, model):
        distanceForward = 1.0

        t1 = model.getLinkFrame('l_foot')
        t2 = model.getLinkFrame('r_foot')
        pelvisT = model.getLinkFrame('pelvis')

        xaxis = [1.0, 0.0, 0.0]
        pelvisT.TransformVector(xaxis, xaxis)
        xaxis = np.array(xaxis)
        zaxis = np.array([0.0, 0.0, 1.0])
        yaxis = np.cross(zaxis, xaxis)
        xaxis = np.cross(yaxis, zaxis)

        stancePosition = np.array(t2.GetPosition()) + np.array(t1.GetPosition()) / 2.0

        footHeight=0.0817

        t = transformUtils.getTransformFromAxes(xaxis, yaxis, zaxis)
        t.PostMultiply()
        t.Translate(stancePosition)
        t.Translate([0.0, 0.0, -footHeight])
        t.Translate(xaxis*distanceForward)

        frameObj = vis.showFrame(t, 'walking goal')
        frameObj.setProperty('Edit', True)

        frameObj.onTransformModifiedCallback = self.onWalkingGoalModified
        print "created walking goal"
        self.sendFootstepPlanRequest()

    def onStepModified(self, ndx, frameObj):
        self.lastFootstepPlanMessage.footsteps[ndx+2].pos = transformUtils.positionMessageFromFrame(frameObj.transform)
        self.lastFootstepPlanMessage.footsteps[ndx+2].fixed_x = True
        self.lastFootstepPlanMessage.footsteps[ndx+2].fixed_y = True
        self.lastFootstepPlanMessage.footsteps[ndx+2].fixed_yaw = True
        self.sendUpdatePlanRequest()

    def sendUpdatePlanRequest(self):
        msg = self.constructFootstepPlanRequest()
        msg.num_existing_steps = self.lastFootstepPlanMessage.num_steps
        msg.existing_steps = self.lastFootstepPlanMessage.footsteps
        lcmUtils.publish('FOOTSTEP_PLAN_REQUEST', msg)
        return msg

    def onWalkingGoalModified(self, frameObj):
        pos, wxyz = transformUtils.poseFromTransform(frameObj.transform)
        self.sendFootstepPlanRequest()

    def constructFootstepPlanRequest(self):
        goalObj = om.findObjectByName('walking goal')
        assert goalObj

        msg = lcmdrc.footstep_plan_request_t()
        msg.utime = getUtime()
        pose = self.jc.getPose('q_end')
        state_msg = robotstate.drakePoseToRobotState(pose)
        msg.initial_state = state_msg

        msg.goal_pos = transformUtils.positionMessageFromFrame(goalObj.transform)
        msg.params = lcmdrc.footstep_plan_params_t()
        msg.params.max_num_steps = 30
        msg.params.min_num_steps = 0
        msg.params.min_step_width = 0.21
        msg.params.nom_step_width = 0.25
        msg.params.max_step_width = 0.4
        msg.params.nom_forward_step = 0.15
        msg.params.max_forward_step = 0.45
        msg.params.ignore_terrain = False
        msg.params.planning_mode = msg.params.MODE_AUTO
        msg.params.behavior = msg.params.BEHAVIOR_BDI_STEPPING
        msg.params.map_command = 2
        msg.params.leading_foot = msg.params.LEAD_AUTO

        msg.default_step_params = lcmdrc.footstep_params_t()
        msg.default_step_params.step_speed = 1.0
        msg.default_step_params.step_height = 0.05
        msg.default_step_params.bdi_step_duration = 2.0
        msg.default_step_params.bdi_sway_duration = 0.0
        msg.default_step_params.bdi_lift_height = 0.05
        msg.default_step_params.bdi_toe_off = 1
        msg.default_step_params.bdi_knee_nominal = 0.0
        msg.default_step_params.bdi_max_foot_vel = 0.0
        msg.default_step_params.bdi_sway_end_dist = 0.02
        msg.default_step_params.bdi_step_end_dist = 0.02
        msg.default_step_params.mu = 1.0

        return msg

    def sendFootstepPlanRequest(self):
        msg = self.constructFootstepPlanRequest()
        lcmUtils.publish('FOOTSTEP_PLAN_REQUEST', msg)
        return msg

def init(jc):
    global driver
    driver = FootstepsDriver(jc)

<|endoftext|>


<commit_before>"""
Python job scheduling for humans.

An in-process scheduler for periodic jobs that uses the builder pattern
for configuration. Schedule lets you run Python functions (or any other
callable) periodically at pre-determined intervals using a simple,
human-friendly syntax.

Inspired by Addam Wiggins' article "Rethinking Cron" [1] and the
"clockwork" Ruby module [2][3].

Features:
    - A simple to use API for scheduling jobs.
    - Very lightweight and no external dependencies.
    - Excellent test coverage.
    - Works with Python 2.7 and 3.3

Usage:
    >>> import schedule
    >>> import time

    >>> def job(message='stuff'):
    >>>     print("I'm working on:", message)

    >>> schedule.every(10).minutes.do(job)
    >>> schedule.every().hour.do(job, message='things')
    >>> schedule.every().day.at("10:30").do(job)

    >>> while True:
    >>>     schedule.run_pending()
    >>>     time.sleep(1)

[1] http://adam.heroku.com/past/2010/4/13/rethinking_cron/
[2] https://github.com/tomykaira/clockwork
[3] http://adam.heroku.com/past/2010/6/30/replace_cron_with_clockwork/
"""
import datetime
import functools
import logging
import random
import time

logger = logging.getLogger('schedule')

class Scheduler(object):
    def __init__(self):
        self.jobs = []

    def run_pending(self):
        """Run all jobs that are scheduled to run.

        Please note that it is *intended behavior that tick() does not
        run missed jobs*. For example, if you've registered a job that
        should run every minute and you only call tick() in one hour
        increments then your job won't be run 60 times in between but
        only once.
        """
        runnable_jobs = (job for job in self.jobs if job.should_run)
        for job in sorted(runnable_jobs):
            job.run()

    def run_all(self, delay_seconds=0):
        """Run all jobs regardless if they are scheduled to run or not.

        A delay of `delay` seconds is added between each job. This helps
        distribute system load generated by the jobs more evenly
        over time."""
        logger.info('Running *all* %i jobs with %is delay inbetween',
                    len(self.jobs), delay_seconds)
        for job in self.jobs:
            job.run()
            time.sleep(delay_seconds)

    def clear(self):
        """Deletes all scheduled jobs."""
        del self.jobs[:]

    def every(self, interval=1):
        """Schedule a new periodic job."""
        job = Job(interval)
        self.jobs.append(job)
        return job

    def on(self, *days):
        """Schedule a new job to run on specific weekdays.

        See the docstring for `Job.on()`.
        """
        job = self.every()
        job.unit = 'days'
        return job.on(*days)

    @property
    def next_run(self):
        """Datetime when the next job should run."""
        if not self.jobs:
            return None
        return min(self.jobs).next_run

    @property
    def idle_seconds(self):
        """Number of seconds until `next_run`."""
        return (self.next_run - datetime.datetime.now()).total_seconds()

class Job(object):
    """A periodic job as used by `Scheduler`."""
    WEEKDAYS = {'sunday': 0, 'monday': 1, 'tuesday': 2, 'wednesday': 3,
                'thursday': 4, 'friday': 5, 'saturday': 6}

    def __init__(self, interval):
        self.interval = interval  # pause interval * unit between runs
        self.job_func = None  # the job job_func to run
        self.unit = None  # time units, e.g. 'minutes', 'hours', ...
        self.at_time = None  # optional time at which this job runs
        self.between_times = ()
        self.run_days = []
        self.start_run = None  # datetime after which this job will start
        self.last_run = None  # datetime of the last run
        self.next_run = None  # datetime of the next run
        self.period = None  # timedelta between runs, only valid for

    def __lt__(self, other):
        """PeriodicJobs are sortable based on the scheduled time
        they run next."""
        return self.next_run < other.next_run

    def __repr__(self):
        def format_time(t):
            return t.strftime("%Y-%m-%d %H:%M:%S") if t else '[never]'

        timestats = '(last run: %s, next run: %s)' % (
                    format_time(self.last_run), format_time(self.next_run))

        job_func_name = self.job_func.__name__
        args = [repr(x) for x in self.job_func.args]
        kwargs = ['%s=%s' % (k, repr(v))
                  for k, v in self.job_func.keywords.items()]
        call_repr = job_func_name + '(' + ', '.join(args + kwargs) + ')'

        if self.run_days:
            final_days = []
            for day in self.run_days:
                days_str = [k.title() for k, i in Job.WEEKDAYS.items()
                            for d in day if i == d]
                final_days.append(' or '.join(days_str))
            repr_str = 'Every %s' % ' and '.join(final_days)
        else:
            repr_str = 'Every %s %s' % (
                self.interval,
                self.unit[:-1] if self.interval == 1 else self.unit)

        if self.between_times:
            repr_str += ' between %s and %s' % (
                self.between_times[0].time(), self.between_times[1].time())
        elif self.at_time:
            repr_str += ' at %s' % self.at_time
        if self.start_run:
            repr_str += ' starting %s' % self.start_run
        repr_str += ' do %s %s' % (call_repr, timestats)
        return repr_str

    @property
    def second(self):
        assert self.interval == 1
        return self.seconds

    @property
    def seconds(self):
        self.unit = 'seconds'
        return self

    @property
    def minute(self):
        assert self.interval == 1
        return self.minutes

    @property
    def minutes(self):
        self.unit = 'minutes'
        return self

    @property
    def hour(self):
        assert self.interval == 1
        return self.hours

    @property
    def hours(self):
        self.unit = 'hours'
        return self

    @property
    def day(self):
        assert self.interval == 1
        return self.days

    @property
    def days(self):
        self.unit = 'days'
        return self

    @property
    def week(self):
        assert self.interval == 1
        return self.weeks

    @property
    def weeks(self):
        self.unit = 'weeks'
        return self

    def on(self, *days):
        """Schedule the job to run on specific weekdays.

        `days` can be a string (or sequence of strings) with the name of the
        weekday (case insensitive), e.g. 'Monday', 'sunday', etc, or a starting
        substring of the name of the weekday, e.g.  'tue', 'Sat', etc.

        If you specify multiple days, e.g. ('mon', 'wed'), the job will run
        every Monday and Wednesday.

        You can also specify OR conditions by separating the day names with a
        pipe, e.g. ('sun|mon', 'wed|thu'). In this case the job will run
        every Sunday *or* Monday, and every Wednesday *or* Thursday.
        """
        weeknums = []
        for day in days:
            day_or = set()
            for d in day.split('|'):
                for n, i in Job.WEEKDAYS.items():
                    if n.startswith(d.lower()):
                        day_or.add(i)
            if day_or:
                weeknums.append(day_or)

        self.run_days = weeknums
        return self

    def at(self, time_str):
        """Schedule the job every day at a specific time.

        Calling this is only valid for jobs scheduled to run every
        N day(s).
        """
        assert self.unit == 'days'
        hour, minute = [int(t) for t in time_str.split(':')]
        assert 0 <= hour <= 23
        assert 0 <= minute <= 59
        self.at_time = datetime.time(hour, minute)
        return self

    def between(self, time_str):
        """Schedule the job at a random time between two timestamps."""
        start, end = [datetime.datetime.strptime(t, '%H:%M')
                      for t in time_str.split('-')]
        self.between_times = (start, end)
        return self

    def starting(self, date_str):
        self.start_run = datetime.datetime.strptime(date_str, '%Y-%m-%d')
        return self

    def do(self, job_func, *args, **kwargs):
        """Specifies the job_func that should be called every time the
        job runs.

        Any additional arguments are passed on to job_func when
        the job runs.
        """
        self.job_func = functools.partial(job_func, *args, **kwargs)
        functools.update_wrapper(self.job_func, job_func)
        self._schedule_next_run()
        logger.info('Scheduled job %s', self)
        return self

    @property
    def should_run(self):
        """True if the job should be run now."""
        return datetime.datetime.now() >= self.next_run

    def run(self):
        """Run the job and immediately reschedule it."""
        logger.info('Running job %s', self)
        self.job_func()
        self.last_run = datetime.datetime.now()
        self._schedule_next_run()

    def _schedule_next_run(self):
        """Compute the instant when this job should run next."""
        # Allow *, ** magic temporarily:
        # pylint: disable=W0142
        assert self.unit in ('seconds', 'minutes', 'hours', 'days', 'weeks')
        starting = self.start_run or datetime.datetime.now()

        self.period = datetime.timedelta(**{self.unit: self.interval})
        self.next_run = starting + self.period

        if self.run_days:
            run_days = self.run_days[:]
            if self.last_run:
                starting = self.last_run
                # Don't consider this day group if it has been run already
                for day in self.run_days:
                    if self.last_run.isoweekday() in day:
                        run_days.remove(day)

            days = set()
            for day in run_days:
                days.add(random.sample(day, 1)[0])

            if not days:
                days_delta = 0
            else:
                # Calculate the closest day from the starting date
                delta_all = sorted([(i - starting.isoweekday()) % 7
                                    for i in days])
                days_delta = delta_all[0]

            if (days_delta == 0 and self.last_run and
                    self.last_run.date() == starting.date()):
                # Make sure the job doesn't run today twice
                if self.unit == 'days':
                    days_delta = 7
                elif self.unit == 'weeks':
                    days_delta = self.interval * 7
            self.next_run = starting + datetime.timedelta(days=days_delta)

        if self.between_times:
            start, end = self.between_times
            # Choose a random time between both timestamps
            self.at_time = (start + datetime.timedelta(
                seconds=random.randint(0, int(
                    (end - start).total_seconds())))).time()
        if self.at_time:
            self.next_run = self.next_run.replace(hour=self.at_time.hour,
                                                  minute=self.at_time.minute,
                                                  second=self.at_time.second,
                                                  microsecond=0)
            # If we are running for the first time, make sure we run
            # at the specified time *today* as well
            if (not self.last_run and not self.run_days and
                    self.at_time > datetime.datetime.now().time()):
                self.next_run = self.next_run - datetime.timedelta(days=1)

# The following methods are shortcuts for not having to
# create a Scheduler instance:

default_scheduler = Scheduler()
jobs = default_scheduler.jobs  # todo: should this be a copy, e.g. jobs()?

def every(interval=1):
    """Schedule a new periodic job."""
    return default_scheduler.every(interval)

def on(*days):
    """Schedule a new job to run on specific weekdays.

    See the docstring for `Job.on()`.
    """
    return default_scheduler.on(*days)

def run_pending():
    """Run all jobs that are scheduled to run.

    Please note that it is *intended behavior that run_pending()
    does not run missed jobs*. For example, if you've registered a job
    that should run every minute and you only call run_pending()
    in one hour increments then your job won't be run 60 times in
    between but only once.
    """
    default_scheduler.run_pending()

def run_all(delay_seconds=0):
    """Run all jobs regardless if they are scheduled to run or not.

    A delay of `delay` seconds is added between each job. This can help
    to distribute the system load generated by the jobs more evenly over
    time."""
    default_scheduler.run_all(delay_seconds=delay_seconds)

def clear():
    """Deletes all scheduled jobs."""
    default_scheduler.clear()

def next_run():
    """Datetime when the next job should run."""
    return default_scheduler.next_run

def idle_seconds():
    """Number of seconds until `next_run`."""
    return default_scheduler.idle_seconds
<commit_msg>Fixed logging after job scheduling<commit_after>"""
Python job scheduling for humans.

An in-process scheduler for periodic jobs that uses the builder pattern
for configuration. Schedule lets you run Python functions (or any other
callable) periodically at pre-determined intervals using a simple,
human-friendly syntax.

Inspired by Addam Wiggins' article "Rethinking Cron" [1] and the
"clockwork" Ruby module [2][3].

Features:
    - A simple to use API for scheduling jobs.
    - Very lightweight and no external dependencies.
    - Excellent test coverage.
    - Works with Python 2.7 and 3.3

Usage:
    >>> import schedule
    >>> import time

    >>> def job(message='stuff'):
    >>>     print("I'm working on:", message)

    >>> schedule.every(10).minutes.do(job)
    >>> schedule.every().hour.do(job, message='things')
    >>> schedule.every().day.at("10:30").do(job)

    >>> while True:
    >>>     schedule.run_pending()
    >>>     time.sleep(1)

[1] http://adam.heroku.com/past/2010/4/13/rethinking_cron/
[2] https://github.com/tomykaira/clockwork
[3] http://adam.heroku.com/past/2010/6/30/replace_cron_with_clockwork/
"""
import datetime
import functools
import logging
import random
import time

logger = logging.getLogger('schedule')

class Scheduler(object):
    def __init__(self):
        self.jobs = []

    def run_pending(self):
        """Run all jobs that are scheduled to run.

        Please note that it is *intended behavior that tick() does not
        run missed jobs*. For example, if you've registered a job that
        should run every minute and you only call tick() in one hour
        increments then your job won't be run 60 times in between but
        only once.
        """
        runnable_jobs = (job for job in self.jobs if job.should_run)
        for job in sorted(runnable_jobs):
            job.run()

    def run_all(self, delay_seconds=0):
        """Run all jobs regardless if they are scheduled to run or not.

        A delay of `delay` seconds is added between each job. This helps
        distribute system load generated by the jobs more evenly
        over time."""
        logger.info('Running *all* %i jobs with %is delay inbetween',
                    len(self.jobs), delay_seconds)
        for job in self.jobs:
            job.run()
            time.sleep(delay_seconds)

    def clear(self):
        """Deletes all scheduled jobs."""
        del self.jobs[:]

    def every(self, interval=1):
        """Schedule a new periodic job."""
        job = Job(interval)
        self.jobs.append(job)
        return job

    def on(self, *days):
        """Schedule a new job to run on specific weekdays.

        See the docstring for `Job.on()`.
        """
        job = self.every()
        job.unit = 'days'
        return job.on(*days)

    @property
    def next_run(self):
        """Datetime when the next job should run."""
        if not self.jobs:
            return None
        return min(self.jobs).next_run

    @property
    def idle_seconds(self):
        """Number of seconds until `next_run`."""
        return (self.next_run - datetime.datetime.now()).total_seconds()

class Job(object):
    """A periodic job as used by `Scheduler`."""
    WEEKDAYS = {'sunday': 0, 'monday': 1, 'tuesday': 2, 'wednesday': 3,
                'thursday': 4, 'friday': 5, 'saturday': 6}

    def __init__(self, interval):
        self.interval = interval  # pause interval * unit between runs
        self.job_func = None  # the job job_func to run
        self.unit = None  # time units, e.g. 'minutes', 'hours', ...
        self.at_time = None  # optional time at which this job runs
        self.between_times = ()
        self.run_days = []
        self.start_run = None  # datetime after which this job will start
        self.last_run = None  # datetime of the last run
        self.next_run = None  # datetime of the next run
        self.period = None  # timedelta between runs, only valid for

    def __lt__(self, other):
        """PeriodicJobs are sortable based on the scheduled time
        they run next."""
        return self.next_run < other.next_run

    def __repr__(self):
        def format_time(t):
            return t.strftime("%Y-%m-%d %H:%M:%S") if t else '[never]'

        timestats = '(last run: %s, next run: %s)' % (
                    format_time(self.last_run), format_time(self.next_run))

        job_func_name = self.job_func.__name__
        args = [repr(x) for x in self.job_func.args]
        kwargs = ['%s=%s' % (k, repr(v))
                  for k, v in self.job_func.keywords.items()]
        call_repr = job_func_name + '(' + ', '.join(args + kwargs) + ')'

        if self.run_days:
            final_days = []
            for day in self.run_days:
                days_str = [k.title() for k, i in Job.WEEKDAYS.items()
                            for d in day if i == d]
                final_days.append(' or '.join(days_str))
            repr_str = 'Every %s' % ' and '.join(final_days)
        else:
            repr_str = 'Every %s %s' % (
                self.interval,
                self.unit[:-1] if self.interval == 1 else self.unit)

        if self.between_times:
            repr_str += ' between %s and %s' % (
                self.between_times[0].time(), self.between_times[1].time())
        elif self.at_time:
            repr_str += ' at %s' % self.at_time
        if self.start_run:
            repr_str += ' starting %s' % self.start_run
        repr_str += ' do %s %s' % (call_repr, timestats)
        return repr_str

    @property
    def second(self):
        assert self.interval == 1
        return self.seconds

    @property
    def seconds(self):
        self.unit = 'seconds'
        return self

    @property
    def minute(self):
        assert self.interval == 1
        return self.minutes

    @property
    def minutes(self):
        self.unit = 'minutes'
        return self

    @property
    def hour(self):
        assert self.interval == 1
        return self.hours

    @property
    def hours(self):
        self.unit = 'hours'
        return self

    @property
    def day(self):
        assert self.interval == 1
        return self.days

    @property
    def days(self):
        self.unit = 'days'
        return self

    @property
    def week(self):
        assert self.interval == 1
        return self.weeks

    @property
    def weeks(self):
        self.unit = 'weeks'
        return self

    def on(self, *days):
        """Schedule the job to run on specific weekdays.

        `days` can be a string (or sequence of strings) with the name of the
        weekday (case insensitive), e.g. 'Monday', 'sunday', etc, or a starting
        substring of the name of the weekday, e.g.  'tue', 'Sat', etc.

        If you specify multiple days, e.g. ('mon', 'wed'), the job will run
        every Monday and Wednesday.

        You can also specify OR conditions by separating the day names with a
        pipe, e.g. ('sun|mon', 'wed|thu'). In this case the job will run
        every Sunday *or* Monday, and every Wednesday *or* Thursday.
        """
        weeknums = []
        for day in days:
            day_or = set()
            for d in day.split('|'):
                for n, i in Job.WEEKDAYS.items():
                    if n.startswith(d.lower()):
                        day_or.add(i)
            if day_or:
                weeknums.append(day_or)

        self.run_days = weeknums
        return self

    def at(self, time_str):
        """Schedule the job every day at a specific time.

        Calling this is only valid for jobs scheduled to run every
        N day(s).
        """
        assert self.unit == 'days'
        hour, minute = [int(t) for t in time_str.split(':')]
        assert 0 <= hour <= 23
        assert 0 <= minute <= 59
        self.at_time = datetime.time(hour, minute)
        return self

    def between(self, time_str):
        """Schedule the job at a random time between two timestamps."""
        start, end = [datetime.datetime.strptime(t, '%H:%M')
                      for t in time_str.split('-')]
        self.between_times = (start, end)
        return self

    def starting(self, date_str):
        self.start_run = datetime.datetime.strptime(date_str, '%Y-%m-%d')
        return self

    def do(self, job_func, *args, **kwargs):
        """Specifies the job_func that should be called every time the
        job runs.

        Any additional arguments are passed on to job_func when
        the job runs.
        """
        self.job_func = functools.partial(job_func, *args, **kwargs)
        functools.update_wrapper(self.job_func, job_func)
        self._schedule_next_run()
        return self

    @property
    def should_run(self):
        """True if the job should be run now."""
        return datetime.datetime.now() >= self.next_run

    def run(self):
        """Run the job and immediately reschedule it."""
        logger.info('Running job %s', self)
        self.job_func()
        self.last_run = datetime.datetime.now()
        self._schedule_next_run()

    def _schedule_next_run(self):
        """Compute the instant when this job should run next."""
        # Allow *, ** magic temporarily:
        # pylint: disable=W0142
        assert self.unit in ('seconds', 'minutes', 'hours', 'days', 'weeks')
        starting = self.start_run or datetime.datetime.now()

        self.period = datetime.timedelta(**{self.unit: self.interval})
        self.next_run = starting + self.period

        if self.run_days:
            run_days = self.run_days[:]
            if self.last_run:
                starting = self.last_run
                # Don't consider this day group if it has been run already
                for day in self.run_days:
                    if self.last_run.isoweekday() in day:
                        run_days.remove(day)

            days = set()
            for day in run_days:
                days.add(random.sample(day, 1)[0])

            if not days:
                days_delta = 0
            else:
                # Calculate the closest day from the starting date
                delta_all = sorted([(i - starting.isoweekday()) % 7
                                    for i in days])
                days_delta = delta_all[0]

            if (days_delta == 0 and self.last_run and
                    self.last_run.date() == starting.date()):
                # Make sure the job doesn't run today twice
                if self.unit == 'days':
                    days_delta = 7
                elif self.unit == 'weeks':
                    days_delta = self.interval * 7
            self.next_run = starting + datetime.timedelta(days=days_delta)

        if self.between_times:
            start, end = self.between_times
            # Choose a random time between both timestamps
            self.at_time = (start + datetime.timedelta(
                seconds=random.randint(0, int(
                    (end - start).total_seconds())))).time()
        if self.at_time:
            self.next_run = self.next_run.replace(hour=self.at_time.hour,
                                                  minute=self.at_time.minute,
                                                  second=self.at_time.second,
                                                  microsecond=0)
            # If we are running for the first time, make sure we run
            # at the specified time *today* as well
            if (not self.last_run and not self.run_days and
                    self.at_time > datetime.datetime.now().time()):
                self.next_run = self.next_run - datetime.timedelta(days=1)

        logger.info('Scheduled job %s', self)

# The following methods are shortcuts for not having to
# create a Scheduler instance:

default_scheduler = Scheduler()
jobs = default_scheduler.jobs  # todo: should this be a copy, e.g. jobs()?

def every(interval=1):
    """Schedule a new periodic job."""
    return default_scheduler.every(interval)

def on(*days):
    """Schedule a new job to run on specific weekdays.

    See the docstring for `Job.on()`.
    """
    return default_scheduler.on(*days)

def run_pending():
    """Run all jobs that are scheduled to run.

    Please note that it is *intended behavior that run_pending()
    does not run missed jobs*. For example, if you've registered a job
    that should run every minute and you only call run_pending()
    in one hour increments then your job won't be run 60 times in
    between but only once.
    """
    default_scheduler.run_pending()

def run_all(delay_seconds=0):
    """Run all jobs regardless if they are scheduled to run or not.

    A delay of `delay` seconds is added between each job. This can help
    to distribute the system load generated by the jobs more evenly over
    time."""
    default_scheduler.run_all(delay_seconds=delay_seconds)

def clear():
    """Deletes all scheduled jobs."""
    default_scheduler.clear()

def next_run():
    """Datetime when the next job should run."""
    return default_scheduler.next_run

def idle_seconds():
    """Number of seconds until `next_run`."""
    return default_scheduler.idle_seconds
<|endoftext|>


<commit_before>'''
Utilities used in the DNAnexus execution environment and test harness.
'''

import os, json, collections, logging, argparse
from functools import wraps
import dxpy

ENTRY_POINT_TABLE = {}

RUN_COUNT = 0

def run(function_name=None, function_input=None):
    '''
    Triggers the execution environment entry point processor.

    Use this function in the program entry point code:

    import dxpy

    @dxpy.entry_point('main')
    def hello(i):
        pass

    dxpy.run()

    If the environment variable *DX_JOB_ID* is set, the processor retrieves the job with that ID from the API server.
    The job's *job.function* field is used to invoke the entry point function in the module from which run() has been
    called. The function name is looked up in the table of all functions decorated with *@dxpy.entry_point('name')*.
    This is the mode of operation used in the DNAnexus execution environment.
    WARNING: The parameters *function_name* and *function_input* are disregarded in this mode of operation.

    If the environment variable *DX_JOB_ID* is not set, the function name may be given in *function_name*; if not set,
    it is assumed to be *main*. The function input may be given in *function_input*; if not set, it is set by parsing
    JSON from the environment variable *DX_JOB_INPUT*; if that is not set, no input is given to the function.

    The absence of *DX_JOB_ID* signals to run() that execution is happening in the debug harness. In this mode of
    operation, all calls to *dxpy.api.jobNew* (and higher level handler methods which use it) are intercepted, and run()
    is invoked instead with appropriate inputs. The initial invocation of *dxpy.run()* (with no arguments) need not be
    changed; instead, use the environment variable *DX_JOB_INPUT* and/or command line arguments:

        script_name --spec=path/to/dxapp.spec --input1=value1 --input2=value2 ...

    With this, no program code requires changing between the two modes.
    '''
    global RUN_COUNT
    RUN_COUNT += 1

    if dxpy.JOB_ID is not None:
        logging.basicConfig()

        try:
            logging.getLogger().addHandler(dxpy.DXLogHandler())
        except dxpy.exceptions.DXError:
            print "TODO: FIXME: the EE client should die if logging is not available"

        dx_working_dir = os.getcwd()

        job = dxpy.describe(dxpy.JOB_ID)
    else:
        if function_name is None:
            function_name = 'main'
        if function_input is None:
            function_input = json.loads(os.environ.get('DX_JOB_INPUT', '{}'))

            # Try to parse args from the command line
            args, remaining_args = None, None
            try:
                parser = argparse.ArgumentParser()
                parser.add_argument("-s", "--spec", help="Path to app metadata definition file (dxapp.json)")
                args, remaining_args = parser.parse_known_args()
            except:
                pass

            if args is not None and args.spec is not None:
                function_input.update(parse_args_as_job_input(args=remaining_args, app_spec=json.load(open(args.spec))))

        job = {'function': function_name, 'input': function_input}
    job['input'] = resolve_job_refs_in_test(job['input'])
    print "Invoking", job.get('function'), "with", job.get('input')

    try:
        result = ENTRY_POINT_TABLE[job['function']](**job['input'])
    except dxpy.AppError as e:
        if dxpy.JOB_ID is not None:
            os.chdir(dx_working_dir)
            with open("job_error.json", "w") as fh:
                fh.write(json.dumps({"error": {"type": "AppError", "message": str(e)}}) + "\n")
        raise
    except Exception as e:
        if dxpy.JOB_ID is not None:
            os.chdir(dx_working_dir)
            with open("job_error.json", "w") as fh:
                fh.write(json.dumps({"error": {"type": "AppInternalError", "message": str(e)}}) + "\n")
        raise

    result = convert_handlers_to_dxlinks(result)

    if dxpy.JOB_ID is not None:
        # TODO: protect against client removing its original working directory
        os.chdir(dx_working_dir)
        with open("job_output.json", "w") as fh:
            fh.write(json.dumps(result) + "\n")
    else:
        result = resolve_job_refs_in_test(result)

    return result

# TODO: make this less naive with respect to cycles and any other things json.dumps() can handle
def convert_handlers_to_dxlinks(x):
    if isinstance(x, dxpy.DXObject):
        x = dxpy.dxlink(x)
    elif isinstance(x, collections.Mapping):
        for key, value in x.iteritems():
            x[key] = convert_handlers_to_dxlinks(value)
    elif isinstance(x, list):
        for i in range(len(x)):
            x[i] = convert_handlers_to_dxlinks(x[i])
    return x

def resolve_job_refs_in_test(x):
    if isinstance(x, collections.Mapping):
        if "job" in x and "field" in x:
            job_result = dxpy.bindings.dxjob._test_harness_jobs[x["job"]]._test_harness_result
            return job_result[x["field"]]
        for key, value in x.iteritems():
            x[key] = resolve_job_refs_in_test(value)
    elif isinstance(x, list):
        for i in range(len(x)):
            x[i] = resolve_job_refs_in_test(x[i])
    return x

def parse_args_as_job_input(args, app_spec):
    parser = argparse.ArgumentParser()
    json_inputs = set()
    for ispec in app_spec.get("inputSpec", []):
        kwargs = {}
        if ispec.get("type") == "int":
            kwargs["type"] = int
        elif ispec.get("type") == "float":
            kwargs["type"] = float
        elif ispec.get("type") == "boolean":
            kwargs["type"] = bool
        elif ispec.get("type") != "string":
            json_inputs.add(ispec["name"])

        if ispec.get("optional") != None:
            kwargs["required"] = not ispec["optional"]

        parser.add_argument("--" + ispec["name"], **kwargs)

    inputs = {}
    for i, value in vars(parser.parse_args(args)).iteritems():
        if value is None:
            continue
        if i in json_inputs:
            inputs[i] = json.loads(value)
        else:
            inputs[i] = value

    return inputs

def entry_point(entry_point_name):
    '''
    Use this to decorate a DNAnexus execution environment entry point. Example:

    @dxpy.entry_point('main')
    def hello(i):
        pass

    '''
    def wrap(f):
        ENTRY_POINT_TABLE[entry_point_name] = f
        @wraps(f)
        def wrapped_f(*args, **kwargs):
            f(*args, **kwargs)
        return wrapped_f
    return wrap
<commit_msg>properly cast error to unicode, to avoid encoding issues<commit_after>'''
Utilities used in the DNAnexus execution environment and test harness.
'''

import os, json, collections, logging, argparse
from functools import wraps
import dxpy

ENTRY_POINT_TABLE = {}

RUN_COUNT = 0

def run(function_name=None, function_input=None):
    '''
    Triggers the execution environment entry point processor.

    Use this function in the program entry point code:

    import dxpy

    @dxpy.entry_point('main')
    def hello(i):
        pass

    dxpy.run()

    If the environment variable *DX_JOB_ID* is set, the processor retrieves the job with that ID from the API server.
    The job's *job.function* field is used to invoke the entry point function in the module from which run() has been
    called. The function name is looked up in the table of all functions decorated with *@dxpy.entry_point('name')*.
    This is the mode of operation used in the DNAnexus execution environment.
    WARNING: The parameters *function_name* and *function_input* are disregarded in this mode of operation.

    If the environment variable *DX_JOB_ID* is not set, the function name may be given in *function_name*; if not set,
    it is assumed to be *main*. The function input may be given in *function_input*; if not set, it is set by parsing
    JSON from the environment variable *DX_JOB_INPUT*; if that is not set, no input is given to the function.

    The absence of *DX_JOB_ID* signals to run() that execution is happening in the debug harness. In this mode of
    operation, all calls to *dxpy.api.jobNew* (and higher level handler methods which use it) are intercepted, and run()
    is invoked instead with appropriate inputs. The initial invocation of *dxpy.run()* (with no arguments) need not be
    changed; instead, use the environment variable *DX_JOB_INPUT* and/or command line arguments:

        script_name --spec=path/to/dxapp.spec --input1=value1 --input2=value2 ...

    With this, no program code requires changing between the two modes.
    '''
    global RUN_COUNT
    RUN_COUNT += 1

    if dxpy.JOB_ID is not None:
        logging.basicConfig()

        try:
            logging.getLogger().addHandler(dxpy.DXLogHandler())
        except dxpy.exceptions.DXError:
            print "TODO: FIXME: the EE client should die if logging is not available"

        dx_working_dir = os.getcwd()

        job = dxpy.describe(dxpy.JOB_ID)
    else:
        if function_name is None:
            function_name = 'main'
        if function_input is None:
            function_input = json.loads(os.environ.get('DX_JOB_INPUT', '{}'))

            # Try to parse args from the command line
            args, remaining_args = None, None
            try:
                parser = argparse.ArgumentParser()
                parser.add_argument("-s", "--spec", help="Path to app metadata definition file (dxapp.json)")
                args, remaining_args = parser.parse_known_args()
            except:
                pass

            if args is not None and args.spec is not None:
                function_input.update(parse_args_as_job_input(args=remaining_args, app_spec=json.load(open(args.spec))))

        job = {'function': function_name, 'input': function_input}
    job['input'] = resolve_job_refs_in_test(job['input'])
    print "Invoking", job.get('function'), "with", job.get('input')

    try:
        result = ENTRY_POINT_TABLE[job['function']](**job['input'])
    except dxpy.AppError as e:
        if dxpy.JOB_ID is not None:
            os.chdir(dx_working_dir)
            with open("job_error.json", "w") as fh:
                fh.write(json.dumps({"error": {"type": "AppError", "message": unicode(e)}}) + "\n")
        raise
    except Exception as e:
        if dxpy.JOB_ID is not None:
            os.chdir(dx_working_dir)
            with open("job_error.json", "w") as fh:
                fh.write(json.dumps({"error": {"type": "AppInternalError", "message": unicode(e)}}) + "\n")
        raise

    result = convert_handlers_to_dxlinks(result)

    if dxpy.JOB_ID is not None:
        # TODO: protect against client removing its original working directory
        os.chdir(dx_working_dir)
        with open("job_output.json", "w") as fh:
            fh.write(json.dumps(result) + "\n")
    else:
        result = resolve_job_refs_in_test(result)

    return result

# TODO: make this less naive with respect to cycles and any other things json.dumps() can handle
def convert_handlers_to_dxlinks(x):
    if isinstance(x, dxpy.DXObject):
        x = dxpy.dxlink(x)
    elif isinstance(x, collections.Mapping):
        for key, value in x.iteritems():
            x[key] = convert_handlers_to_dxlinks(value)
    elif isinstance(x, list):
        for i in range(len(x)):
            x[i] = convert_handlers_to_dxlinks(x[i])
    return x

def resolve_job_refs_in_test(x):
    if isinstance(x, collections.Mapping):
        if "job" in x and "field" in x:
            job_result = dxpy.bindings.dxjob._test_harness_jobs[x["job"]]._test_harness_result
            return job_result[x["field"]]
        for key, value in x.iteritems():
            x[key] = resolve_job_refs_in_test(value)
    elif isinstance(x, list):
        for i in range(len(x)):
            x[i] = resolve_job_refs_in_test(x[i])
    return x

def parse_args_as_job_input(args, app_spec):
    parser = argparse.ArgumentParser()
    json_inputs = set()
    for ispec in app_spec.get("inputSpec", []):
        kwargs = {}
        if ispec.get("type") == "int":
            kwargs["type"] = int
        elif ispec.get("type") == "float":
            kwargs["type"] = float
        elif ispec.get("type") == "boolean":
            kwargs["type"] = bool
        elif ispec.get("type") != "string":
            json_inputs.add(ispec["name"])

        if ispec.get("optional") != None:
            kwargs["required"] = not ispec["optional"]

        parser.add_argument("--" + ispec["name"], **kwargs)

    inputs = {}
    for i, value in vars(parser.parse_args(args)).iteritems():
        if value is None:
            continue
        if i in json_inputs:
            inputs[i] = json.loads(value)
        else:
            inputs[i] = value

    return inputs

def entry_point(entry_point_name):
    '''
    Use this to decorate a DNAnexus execution environment entry point. Example:

    @dxpy.entry_point('main')
    def hello(i):
        pass

    '''
    def wrap(f):
        ENTRY_POINT_TABLE[entry_point_name] = f
        @wraps(f)
        def wrapped_f(*args, **kwargs):
            f(*args, **kwargs)
        return wrapped_f
    return wrap
<|endoftext|>


<commit_before>"""Leetcode 278. 278. First Bad Version
Easy

URL: https://leetcode.com/problems/first-bad-version/

You are a product manager and currently leading a team to develop a
new product. Unfortunately, the latest version of your product fails
the quality check. Since each version is developed based on the previous
version, all the versions after a bad version are also bad.

Suppose you have n versions [1, 2, ..., n] and you want to find out the
first bad one, which causes all the following ones to be bad.

You are given an API bool isBadVersion(version) which will return whether
version is bad. Implement a function to find the first bad version.
You should minimize the number of calls to the API.

Example:

Given n = 5, and version = 4 is the first bad version.

call isBadVersion(3) -> false
call isBadVersion(5) -> true
call isBadVersion(4) -> true

Then 4 is the first bad version.
"""

# The isBadVersion API is already defined for you.
# @param version, an integer
# @return a bool
def isBadVersion(version):
	pass

class Solution(object):
    def firstBadVersion(self, n):
        """
        :type n: int
        :rtype: int
        """
        pass

def main():
	pass

if __init__ == '__main__':
	main()
<commit_msg>Complete lc278_first_bad_version.py<commit_after>"""Leetcode 278. First Bad Version
Easy

URL: https://leetcode.com/problems/first-bad-version/

You are a product manager and currently leading a team to develop a
new product. Unfortunately, the latest version of your product fails
the quality check. Since each version is developed based on the previous
version, all the versions after a bad version are also bad.

Suppose you have n versions [1, 2, ..., n] and you want to find out the
first bad one, which causes all the following ones to be bad.

You are given an API bool isBadVersion(version) which will return whether
version is bad. Implement a function to find the first bad version.
You should minimize the number of calls to the API.

Example:

Given n = 5, and version = 4 is the first bad version.

call isBadVersion(3) -> false
call isBadVersion(5) -> true
call isBadVersion(4) -> true

Then 4 is the first bad version.
"""

# The isBadVersion API is already defined for you.
# @param version, an integer
# @return a bool
def isBadVersion(version):
    return VERSION_FAILURES[version - 1]

class Solution(object):
    def firstBadVersion(self, n):
        """
        :type n: int
        :rtype: int

        Time complexity: O(logn).
        Space complexity: O(1).
        """
        start = 1
        final = n

        while final - start > 0:
            middle = start + (final - start) // 2
            if isBadVersion(middle):
                # Since middle is bad, use it as next final.
                final = middle
            else:
                # Since middle is good, use its next as next start.
                start = middle + 1

        return final

def main():
    global VERSION_FAILURES
    VERSION_FAILURES = [False] * 1000 + [True] * 100
    n = len(VERSION_FAILURES)

    print Solution().firstBadVersion(n)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests the scriptTree jobTree-script compiler.
"""

import unittest
import sys
import os
import random

from sonLib.bioio import TestStatus
from sonLib.bioio import parseSuiteTestOptions
from sonLib.bioio import system
from sonLib.bioio import getTempDirectory
from sonLib.bioio import getTempFile

from jobTree.src.common import parasolIsInstalled, gridEngineIsInstalled, workflowRootPath

from jobTree.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
from test import JobTreeTest

class TestCase(JobTreeTest):

    def setUp(self):
        super( TestCase, self ).setUp( )
        self.testNo = TestStatus.getTestSetup(1, 2, 10, 10)

    def testScriptTree_SortSimple(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        scriptTree_SortTest(self.testNo, "singleMachine")

    def testScriptTree_SortGridEngine(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        if gridEngineIsInstalled():
            scriptTree_SortTest(self.testNo, "gridengine")

    def testScriptTree_Parasol(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        if parasolIsInstalled():
            scriptTree_SortTest(self.testNo, "parasol")

    """
    def testScriptTree_SortAcid(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        scriptTree_SortTest(self.testNo, "acid_test")
    """

    #The following functions test the functions in the test!

    def testSort(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile1 = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile1)
            lines1 = loadFile(tempFile1)
            lines1.sort()
            sort(tempFile1)
            lines2 = loadFile(tempFile1)
            checkEqual(lines1, lines2)
            system("rm -rf %s" % tempDir)

    def testMerge(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile1 = getTempFile(rootDir=tempDir)
            tempFile2 = getTempFile(rootDir=tempDir)
            tempFile3 = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile1)
            makeFileToSort(tempFile2)
            sort(tempFile1)
            sort(tempFile2)
            with open(tempFile3, 'w') as fileHandle:
                with open( tempFile1 ) as tempFileHandle1:
                    with open( tempFile2 ) as tempFileHandle2:
                        merge(tempFileHandle1, tempFileHandle2, fileHandle)
            lines1 = loadFile(tempFile1) + loadFile(tempFile2)
            lines1.sort()
            lines2 = loadFile(tempFile3)
            checkEqual(lines1, lines2)
            system("rm -rf %s" % tempDir)

    def testCopySubRangeOfFile(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile = getTempFile(rootDir=tempDir)
            outputFile = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile)
            fileSize = os.path.getsize(tempFile)
            assert fileSize > 0
            fileStart = random.choice(xrange(0, fileSize))
            fileEnd = random.choice(xrange(fileStart, fileSize))
            fileHandle = open(outputFile, 'w')
            copySubRangeOfFile(tempFile, fileStart, fileEnd, fileHandle)
            fileHandle.close()
            l = open(outputFile, 'r').read()
            l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
            checkEqual(l, l2)
            system("rm -rf %s" % tempDir)

    def testGetMidPoint(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile)
            l = open(tempFile, 'r').read()
            fileSize = os.path.getsize(tempFile)
            midPoint = getMidPoint(tempFile, 0, fileSize)
            print "the mid point is %i of a file of %i bytes woth byte" % (midPoint, fileSize)
            assert midPoint < fileSize
            assert l[midPoint] == '\n'
            assert midPoint >= 0
            system("rm -rf %s" % tempDir)

def scriptTree_SortTest(testNo, batchSystem, lines=10000, maxLineLength=10, N=10000):
    """Tests scriptTree/jobTree by sorting a file in parallel.
    """
    for test in xrange(testNo):
        tempDir = getTempDirectory(os.getcwd())
        tempFile = getTempFile(rootDir=tempDir)
        jobTreeDir = os.path.join(tempDir, "testJobTree")
        makeFileToSort(tempFile, lines=lines, maxLineLength=maxLineLength)
        #First make our own sorted version
        fileHandle = open(tempFile, 'r')
        l = fileHandle.readlines()
        l.sort()
        fileHandle.close()
        #Sort the file
        rootPath = os.path.join(workflowRootPath(), "test/sort")
        while True:
            command = "%s/sort.py --jobTree %s --logLevel=DEBUG --fileToSort=%s --N %i --batchSystem %s --jobTime 1.0 --maxCpus 20 --retryCount 2" % (rootPath, jobTreeDir, tempFile, N, batchSystem) #, retryCount)
            system(command)
            try:
                system("jobTreeStatus --jobTree %s --failIfNotComplete" % jobTreeDir)
                break
            except:
                print "The jobtree failed and will be restarted"
                #raise RuntimeError()
                continue

        #Now check the file is properly sorted..
        #Now get the sorted file
        fileHandle = open(tempFile, 'r')
        l2 = fileHandle.readlines()
        fileHandle.close()
        checkEqual(l, l2)
        system("rm -rf %s" % tempDir)

def checkEqual(i, j):
    if i != j:
        print "lengths", len(i), len(j)
        print "true", i
        print "false", j
    assert i == j

def loadFile(file):
    fileHandle = open(file, 'r')
    lines = fileHandle.readlines()
    fileHandle.close()
    return lines

def getRandomLine(maxLineLength):
    return "".join([ random.choice([ 'a', 'c', 't', 'g', "A", "C", "G", "T", "N", "X", "Y", "Z" ]) for i in xrange(maxLineLength) ]) + "\n"

def makeFileToSort(fileName, lines=10, maxLineLength=10):
    fileHandle = open(fileName, 'w')
    for line in xrange(lines):
        fileHandle.write(getRandomLine(maxLineLength))
    fileHandle.close()

def main():
    parseSuiteTestOptions()
    sys.argv = sys.argv[:1]
    unittest.main()

if __name__ == '__main__':
    main()
<commit_msg>Sort test: Allow for overriding job store implementation; cosmetics<commit_after>#!/usr/bin/env python
"""Tests the scriptTree jobTree-script compiler.
"""
import logging

import unittest
import sys
import os
import random
from uuid import uuid4

from sonLib.bioio import TestStatus
from sonLib.bioio import parseSuiteTestOptions
from sonLib.bioio import system
from sonLib.bioio import getTempDirectory
from sonLib.bioio import getTempFile

from jobTree.src.common import parasolIsInstalled, gridEngineIsInstalled, workflowRootPath

from jobTree.test.sort.lib import merge, sort, copySubRangeOfFile, getMidPoint
from test import JobTreeTest

class TestCase(JobTreeTest):

    def setUp(self):
        super( TestCase, self ).setUp( )
        self.testNo = TestStatus.getTestSetup(1, 2, 10, 10)

    def testScriptTree_SortSimple(self):
        """Tests scriptTree/jobTree by sorting a file in parallel.
        """
        scriptTree_SortTest(self.testNo, "singleMachine")

    def testScriptTree_SortGridEngine(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        if gridEngineIsInstalled():
            scriptTree_SortTest(self.testNo, "gridengine")

    def testScriptTree_Parasol(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        if parasolIsInstalled():
            scriptTree_SortTest(self.testNo, "parasol")

    """
    def testScriptTree_SortAcid(self):
        #Tests scriptTree/jobTree by sorting a file in parallel.
        scriptTree_SortTest(self.testNo, "acid_test")
    """

    #The following functions test the functions in the test!

    def testSort(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile1 = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile1)
            lines1 = loadFile(tempFile1)
            lines1.sort()
            sort(tempFile1)
            lines2 = loadFile(tempFile1)
            checkEqual(lines1, lines2)
            system("rm -rf %s" % tempDir)

    def testMerge(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile1 = getTempFile(rootDir=tempDir)
            tempFile2 = getTempFile(rootDir=tempDir)
            tempFile3 = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile1)
            makeFileToSort(tempFile2)
            sort(tempFile1)
            sort(tempFile2)
            with open(tempFile3, 'w') as fileHandle:
                with open( tempFile1 ) as tempFileHandle1:
                    with open( tempFile2 ) as tempFileHandle2:
                        merge(tempFileHandle1, tempFileHandle2, fileHandle)
            lines1 = loadFile(tempFile1) + loadFile(tempFile2)
            lines1.sort()
            lines2 = loadFile(tempFile3)
            checkEqual(lines1, lines2)
            system("rm -rf %s" % tempDir)

    def testCopySubRangeOfFile(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile = getTempFile(rootDir=tempDir)
            outputFile = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile)
            fileSize = os.path.getsize(tempFile)
            assert fileSize > 0
            fileStart = random.choice(xrange(0, fileSize))
            fileEnd = random.choice(xrange(fileStart, fileSize))
            fileHandle = open(outputFile, 'w')
            copySubRangeOfFile(tempFile, fileStart, fileEnd, fileHandle)
            fileHandle.close()
            l = open(outputFile, 'r').read()
            l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
            checkEqual(l, l2)
            system("rm -rf %s" % tempDir)

    def testGetMidPoint(self):
        for test in xrange(self.testNo):
            tempDir = getTempDirectory(os.getcwd())
            tempFile = getTempFile(rootDir=tempDir)
            makeFileToSort(tempFile)
            l = open(tempFile, 'r').read()
            fileSize = os.path.getsize(tempFile)
            midPoint = getMidPoint(tempFile, 0, fileSize)
            print "the mid point is %i of a file of %i bytes woth byte" % (midPoint, fileSize)
            assert midPoint < fileSize
            assert l[midPoint] == '\n'
            assert midPoint >= 0
            system("rm -rf %s" % tempDir)

def scriptTree_SortTest(testNo, batchSystem, jobStore='file', lines=10000, maxLineLength=10, N=10000):
    """Tests scriptTree/jobTree by sorting a file in parallel.
    """
    for test in xrange(testNo):
        tempDir = getTempDirectory(os.getcwd())
        tempFile = getTempFile(rootDir=tempDir)
        if jobStore == 'file':
            jobTreeDir = os.path.join(tempDir, "testJobTree")
        else:
            jobTreeDir = jobStore
        makeFileToSort(tempFile, lines=lines, maxLineLength=maxLineLength)
        #First make our own sorted version
        fileHandle = open(tempFile, 'r')
        l = fileHandle.readlines()
        l.sort()
        fileHandle.close()
        #Sort the file
        rootPath = os.path.join(workflowRootPath(), "test/sort")
        while True:
            command = "{rootPath}/sort.py " \
                      "--jobTree '{jobTreeDir}' " \
                      "--logLevel=DEBUG " \
                      "--fileToSort='{tempFile}' " \
                      "--N {N:d} " \
                      "--batchSystem {batchSystem} " \
                      "--jobTime 1000.0 " \
                      "--maxCpus 20 " \
                      "--retryCount 2".format( **locals() )
            system(command)
            try:
                system("jobTreeStatus --jobTree %s --failIfNotComplete" % jobTreeDir)
                break
            except:
                print "The jobtree failed and will be restarted"
                #raise RuntimeError()
                continue

        #Now check the file is properly sorted..
        #Now get the sorted file
        fileHandle = open(tempFile, 'r')
        l2 = fileHandle.readlines()
        fileHandle.close()
        checkEqual(l, l2)
        system("rm -rf %s" % tempDir)

def checkEqual(i, j):
    if i != j:
        print "lengths", len(i), len(j)
        print "true", i
        print "false", j
    assert i == j

def loadFile(file):
    fileHandle = open(file, 'r')
    lines = fileHandle.readlines()
    fileHandle.close()
    return lines

def getRandomLine(maxLineLength):
    return "".join([ random.choice([ 'a', 'c', 't', 'g', "A", "C", "G", "T", "N", "X", "Y", "Z" ]) for i in xrange(maxLineLength) ]) + "\n"

def makeFileToSort(fileName, lines=10, maxLineLength=10):
    fileHandle = open(fileName, 'w')
    for line in xrange(lines):
        fileHandle.write(getRandomLine(maxLineLength))
    fileHandle.close()

def main():
    parseSuiteTestOptions()
    sys.argv = sys.argv[:1]
    unittest.main()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>'''
@author: jnaous
'''
from django.views.generic import simple, create_update
from expedient.common.messaging.models import DatedMessage
from django.http import HttpResponseRedirect, HttpResponseNotAllowed
from django.core.urlresolvers import reverse
from forms import OpenFlowAggregateForm
from expedient.clearinghouse.aggregate.models import AggregateAdminInfo
from expedient.common.xmlrpc_serverproxy.forms import PasswordXMLRPCServerProxyForm
from models import OpenFlowAggregate
from django.shortcuts import get_object_or_404
from django.db import transaction

import logging
logger = logging.getLogger("OpenFlow plugin views")

def aggregate_crud(request, agg_id=None):
    '''
    Create/update an OpenFlow Aggregate.
    '''

    if agg_id != None:
        aggregate = get_object_or_404(OpenFlowAggregate, pk=agg_id)
        client = aggregate.client
    else:
        aggregate = None
        client = None

    if request.method == "GET":
        agg_form = OpenFlowAggregateForm(instance=aggregate)
        client_form = PasswordXMLRPCServerProxyForm(instance=client)

    elif request.method == "POST":
        logger.debug("aggregate_crud got post")
        agg_form = OpenFlowAggregateForm(data=request.POST,
                                         instance=aggregate)
        client_form = PasswordXMLRPCServerProxyForm(data=request.POST,
                                                    instance=client)
        logger.debug("Validating")
        if client_form.is_valid() and agg_form.is_valid():
            logger.debug("Forms are valid")
            # Save the client first
            client = client_form.save()
            # Then save the aggregate and add the client
            aggregate = agg_form.save(commit=False)
            aggregate.client = client
            aggregate.save()
            agg_form.save_m2m()
            # Add current user as owner for the aggregate
            admin_info, created = AggregateAdminInfo.objects.get_or_create(
                admin=request.user,
            )
            aggregate.admins_info.add(admin_info)
            err = aggregate.setup_new_aggregate(request.META['HTTP_HOST'])
            if err:
                transaction.rollback()
                if agg_id:
                    msg = "Problem setting up the updated aggregate. %s" % err
                else:
                    msg = "Problem setting up the new aggregate. %s" % err
                DatedMessage.objects.post_message_to_users(
                    msg, pk=request.user.pk,
                )
                print err
            else:
                aggregate.save()
            return HttpResponseRedirect(reverse("aggregate_all"))
        logger.debug("Validation failed")
    else:
        return HttpResponseNotAllowed("GET", "POST")

    available = aggregate.check_status() if aggregate else False
    return simple.direct_to_template(
        request,
        template="openflow/plugin/aggregate_crud.html",
        extra_context={
            "agg_form": agg_form,
            "client_form": client_form,
            "create": not agg_id,
            "aggregate": aggregate,
            "available": available,
        },
    )

def aggregate_delete(request, agg_id):
    """
    Delete and aggregate.
    """
    return create_update.delete_object(
        request,
        OpenFlowAggregate,
        reverse("aggregate_all"),
        agg_id,
        template_name="openflow/plugin/aggregate_confirm_delete.html",
    )
<commit_msg>don't add multiple admininfos for an aggregate<commit_after>'''
@author: jnaous
'''
from django.views.generic import simple, create_update
from expedient.common.messaging.models import DatedMessage
from django.http import HttpResponseRedirect, HttpResponseNotAllowed
from django.core.urlresolvers import reverse
from forms import OpenFlowAggregateForm
from expedient.clearinghouse.aggregate.models import AggregateAdminInfo
from expedient.common.xmlrpc_serverproxy.forms import PasswordXMLRPCServerProxyForm
from models import OpenFlowAggregate
from django.shortcuts import get_object_or_404
from django.db import transaction

import logging
logger = logging.getLogger("OpenFlow plugin views")

def aggregate_crud(request, agg_id=None):
    '''
    Create/update an OpenFlow Aggregate.
    '''

    if agg_id != None:
        aggregate = get_object_or_404(OpenFlowAggregate, pk=agg_id)
        client = aggregate.client
    else:
        aggregate = None
        client = None

    if request.method == "GET":
        agg_form = OpenFlowAggregateForm(instance=aggregate)
        client_form = PasswordXMLRPCServerProxyForm(instance=client)

    elif request.method == "POST":
        logger.debug("aggregate_crud got post")
        agg_form = OpenFlowAggregateForm(data=request.POST,
                                         instance=aggregate)
        client_form = PasswordXMLRPCServerProxyForm(data=request.POST,
                                                    instance=client)
        logger.debug("Validating")
        if client_form.is_valid() and agg_form.is_valid():
            logger.debug("Forms are valid")
            logger.debug("Got logo %s" % request.POST.get("logo", ""))
            # Save the client first
            client = client_form.save()
            # Then save the aggregate and add the client
            aggregate = agg_form.save(commit=False)
            aggregate.client = client
            aggregate.save()
            agg_form.save_m2m()
            # Add current user as owner for the aggregate
            admin_info, created = AggregateAdminInfo.objects.get_or_create(
                admin=request.user,
                aggregate=aggregate,
            )
            aggregate.admins_info.add(admin_info)
            err = aggregate.setup_new_aggregate(request.META['HTTP_HOST'])
            if err:
                transaction.rollback()
                if agg_id:
                    msg = "Problem setting up the updated aggregate. %s" % err
                else:
                    msg = "Problem setting up the new aggregate. %s" % err
                DatedMessage.objects.post_message_to_users(
                    msg, pk=request.user.pk,
                )
                print err
            else:
                aggregate.save()
            return HttpResponseRedirect(reverse("aggregate_all"))
        logger.debug("Validation failed")
    else:
        return HttpResponseNotAllowed("GET", "POST")

    available = aggregate.check_status() if aggregate else False
    return simple.direct_to_template(
        request,
        template="openflow/plugin/aggregate_crud.html",
        extra_context={
            "agg_form": agg_form,
            "client_form": client_form,
            "create": not agg_id,
            "aggregate": aggregate,
            "available": available,
        },
    )

def aggregate_delete(request, agg_id):
    """
    Delete and aggregate.
    """
    return create_update.delete_object(
        request,
        OpenFlowAggregate,
        reverse("aggregate_all"),
        agg_id,
        template_name="openflow/plugin/aggregate_confirm_delete.html",
    )
<|endoftext|>