<commit_before># Copyright (c) 2012, Braiden Kindt.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#   1. Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
# ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import threading
import logging
import array
import collections
import errno
import Queue
import time

import antagent.antmsg as antmsg

_LOG = logging.getLogger("antagent.ant")

def msg_to_string(msg):
    return array.array("B", msg).tostring().encode("hex")

def generate_checksum(msg):
    return reduce(lambda x, y: x ^ y, msg)

def validate_checksum(msg):
    return generate_checksum(msg) == 0

def tokenize_message(msg):
    while msg:
        assert msg[0] & 0xFE == antmsg.SYNC
        length = msg[1]
        yield msg[:4 + length]
        msg = msg[4 + length:]

def is_same_channel(request, response):
    try: response_channel = response.msg_args.channel_number
    except AttributeError: return True
    try: request_channel = request.msg_args.channel_number
    except AttributeError: request_channel = -1
    return response_channel == request_channel

class AntCommand(object):

    def __init__(self, msg_type, *args, **kwds):
        self.msg_type = msg_type
        try:
            self.msg_args = msg_type.msg_tuple(*args, **kwds)
        except AttributeError:
            self.msg_args = args

    def __str__(self):
        return str(self.msg_args)

class AntCore(object):

    def __init__(self, hardware, messages=antmsg.ALL_MESSAGES):
        self.hardware = hardware
        self.input_msg_by_id = dict((m.msg_id, m) for m in messages if m.msg_dir == antmsg.DIR_IN)
        # per ant protocol doc, writing 15 zeros
        # should reset internal state of device.
        #self.hardware.write([0] * 15, 100)

    def pack(self, command):
        if command.msg_type.msg_id == antmsg.DIR_IN:
            _LOG.warning("Request to pack input message %s.", command)
        try:
            struct = command.msg_type.msg_struct
        except AttributeError:
            _LOG.warning("Attempt to pack unsupported command %s.", command)
            return []
        else:
            msg = [antmsg.SYNC, struct.size, command.msg_type.msg_id]
            msg.extend(array.array("B", struct.pack(*command.msg_args)))
            msg.append(generate_checksum(msg))
            return msg

    def unpack(self, msg):
        if not validate_checksum(msg):
            raise IOError("Invalid Checksum", msg)
        sync, length, msg_id = msg[:3]
        try:
            msg_type = self.input_msg_by_id[msg_id]
            struct = msg_type.msg_struct
        except (KeyError, AttributeError):
            _LOG.warning("Attempt to unpack unkown message (0x%02x) %s.", msg_id, msg)
            return AntCommand(msg_id)
        else:
            msg_args = struct.unpack(array.array("B", msg[3:-1]).tostring())
            return AntCommand(msg_type, *msg_args)

    def send(self, command, timeout=100):
        msg = self.pack(command)
        _LOG.debug("SEND: %s", msg_to_string(msg))
        # ant protocol status \x00\x00 padding is optiontal
        # but nRF24AP2 seems to not occaionally not reply to
        # to short commands when zero padding is excluded.
        msg.extend([0] * 2)
        try:
            self.hardware.write(msg, timeout)
            return True
        except IOError as (err, msg):
            if err == errno.ETIMEDOUT: return False
            else: raise

    def recv(self, timeout=500):
        while True:
            try:
                # tokenize message (possibly more than on per read)
                for msg in tokenize_message(self.hardware.read(timeout)):
                    _LOG.debug("RECV: %s", msg_to_string(msg))
                    yield self.unpack(msg)
            except IOError as (err, msg):
                # iteration terminates on timeout
                if err == errno.ETIMEDOUT: raise StopIteration()
                else: raise

class AntSession(object):

    SEND_TIMEOUT = 1

    def __init__(self, ant_core, open=True):
        self.ant_core = ant_core
        self.running = False
        self.running_cmd = None
        if open: self.open()

    def send(self, message_type, *args, **kwds):
        assert not self.running_cmd
        # create a new command. After setting self.running_cmd
        # access to this command from this tread is invalid
        # unless cmd.done event is set.
        cmd = AntCommand(message_type, *args, **kwds)
        cmd.expiration = time.time() + AntSession.SEND_TIMEOUT
        cmd.done = threading.Event()
        self.running_cmd = cmd
        # continue trying to commit command until session closed or command timeout
        while self.running and not cmd.done.is_set() and not self.ant_core.send(cmd):
            _LOG.warning("Device write timeout. Will keep trying.")
        # continue waiting for command completion until session closed
        while self.running and not cmd.done.is_set():
            cmd.done.wait(1)
        # cmd.done guarenetees a result is availible
        if cmd.done.is_set():
            try:
                return cmd.result
            except AttributeError:
                raise cmd.error
        else:
            raise IOError(errno.EBADF, "Session closed.")

    def open(self):
        if not self.running:
            self.running = True
            self.thread = threading.Thread(target=self.loop)
            self.thread.daemon = True
            self.thread.start()

    def close(self):
        try:
            self.running = False
            self.thread.join(1000)
        except AttributeError: pass

    def _set_result(self, result):
        if self.running_cmd:
            self.running_cmd.result = result
            self.running_cmd.done.set()
            self.running_cmd = None

    def _set_error(self, err):
        if self.running_cmd:
            self.running_cmd.error = err
            self.running_cmd.done.set()
            self.running_cmd = None

    def _handle_command(self, cmd):
        if self.running_cmd and is_same_channel(self.running_cmd, cmd):
            if self.running_cmd.msg_type == antmsg.REQUEST_MESSAGE \
                    and self.running_cmd.msg_args.msg_id == cmd.msg_type.msg_id:
                # running command is REQUEST_MESSAGE, return reply.
                # no validation is necessary.
                self._set_result(cmd)
            elif self.running_cmd.msg_type == antmsg.CLOSE_CHANNEL \
                    and cmd.msg_type == antmsg.CHANNEL_EVENT \
                    and cmd.msg_args.msg_id == 1 \
                    and cmd.msg_args.msg_code == antmsg.EVENT_CHANNEL_CLOSED:
                # channel closed, return event.
                # no validation necessary.
                self._set_result(cmd)
            elif cmd.msg_type == antmsg.CHANNEL_EVENT \
                    and cmd.msg_args.msg_id == self.running_cmd.msg_type.msg_id:
                # incoming command is channel event matching
                # the currently running command.
                if cmd.msg_args.msg_code != antmsg.RESPONSE_NO_ERROR:
                    # command failed
                    self._set_error(IOError(errno.EINVAL, "Failed to exceute command message_code=%d. %s" % (cmd.msg_args.msg_code, self.running_cmd)))
                else:
                    # command succeeded
                    self._set_result(cmd)
            elif cmd.msg_type == antmsg.STARTUP_MESSAGE \
                    and self.running_cmd.msg_type == antmsg.RESET_SYSTEM:
                # reset, return reply. FIXME, STARTUP MESSAGE
                # not implemented by older ANT hardware
                self._set_result(cmd)
        # check for timeout condition
        self._handle_timeout()

    def _handle_timeout(self):
        # if a command is currently running, check for timeout condition
        if self.running_cmd and time.time() > self.running_cmd.expiration:
            self._set_error(IOError(errno.ETIMEDOUT, "No reply to command. %s" %  self.running_cmd))

    def loop(self):
        try:
            while self.running:
                for cmd in self.ant_core.recv():
                    if not self.running: break
                    self._handle_command(cmd)
                else:
                    if not self.running: break
                    self._handle_timeout()
        except Exception:
            _LOG.error("Caught Exception reading from device.", exc_info=True)
        finally:
            self.running_cmd = None
            self.running = False

# vim: ts=4 sts=4 et
<commit_msg>timeout/retry cleanup<commit_after># Copyright (c) 2012, Braiden Kindt.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#   1. Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
# ''AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
# WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import threading
import logging
import array
import errno
import time

import antagent.antmsg as antmsg

_LOG = logging.getLogger("antagent.ant")

def msg_to_string(msg):
    return array.array("B", msg).tostring().encode("hex")

def generate_checksum(msg):
    return reduce(lambda x, y: x ^ y, msg)

def validate_checksum(msg):
    return generate_checksum(msg) == 0

def tokenize_message(msg):
    while msg:
        assert msg[0] & 0xFE == antmsg.SYNC
        length = msg[1]
        yield msg[:4 + length]
        msg = msg[4 + length:]

def is_same_channel(request, response):
    try: response_channel = response.args.channel_number
    except AttributeError: return True
    try: request_channel = request.args.channel_number
    except AttributeError: request_channel = -1
    return response_channel == request_channel

class AntCommand(object):

    def __init__(self, msg, *args, **kwds):
        self.msg = msg
        try:
            self.args = msg.msg_tuple(*args, **kwds)
        except AttributeError:
            self.args = args

    def __str__(self):
        return str(self.args)

class AntCore(object):

    def __init__(self, hardware, messages=antmsg.ALL_MESSAGES):
        self.hardware = hardware
        self.input_msg_by_id = dict((m.msg_id, m) for m in messages if m.msg_dir == antmsg.DIR_IN)
        # per ant protocol doc, writing 15 zeros
        # should reset internal state of device.
        #self.hardware.write([0] * 15, 100)

    def pack(self, command):
        if command.msg.msg_id == antmsg.DIR_IN:
            _LOG.warning("Request to pack input message. %s", command)
        try:
            struct = command.msg.msg_struct
        except AttributeError:
            _LOG.warning("Attempt to pack unsupported command. %s", command)
            return []
        else:
            msg = [antmsg.SYNC, struct.size, command.msg.msg_id]
            msg.extend(array.array("B", struct.pack(*command.args)))
            msg.append(generate_checksum(msg))
            return msg

    def unpack(self, msg):
        if not validate_checksum(msg):
            _LOG.error("Invalid checksum, mesage discarded. %s", msg_to_string(msg))
            return None
        sync, length, msg_id = msg[:3]
        try:
            msg_type = self.input_msg_by_id[msg_id]
            struct = msg_type.msg_struct
        except (KeyError, AttributeError):
            _LOG.warning("Attempt to unpack unkown message (0x%02x). %s", msg_id, msg_to_string(msg))
            return AntCommand(msg_id)
        else:
            msg_args = struct.unpack(array.array("B", msg[3:-1]).tostring())
            return AntCommand(msg_type, *msg_args)

    def send(self, command, timeout=100):
        msg = self.pack(command)
        _LOG.debug("SEND: %s", msg_to_string(msg))
        # ant protocol states \x00\x00 padding is optiontal
        # but nRF24AP2 seems to occaionally not reply to
        # commands when zero padding is excluded.
        msg.extend([0] * 2)
        try:
            self.hardware.write(msg, timeout)
            return True
        except IOError as (err, msg):
            if err == errno.ETIMEDOUT: return False
            else: raise

    def recv(self, timeout=500):
        while True:
            try:
                # tokenize message (possibly more than on per read)
                for msg in tokenize_message(self.hardware.read(timeout)):
                    _LOG.debug("RECV: %s", msg_to_string(msg))
                    cmd = self.unpack(msg)
                    if cmd: yield cmd
            except IOError as (err, msg):
                # iteration terminates on timeout
                if err == errno.ETIMEDOUT: raise StopIteration()
                else: raise

class AntSession(object):

    def __init__(self, ant_core, open=True):
        self.ant_core = ant_core
        self.running = False
        self.running_cmd = None
        if open: self.open()

    def open(self):
        if not self.running:
            self.running = True
            self.thread = threading.Thread(target=self.loop)
            self.thread.daemon = True
            self.thread.start()

    def close(self):
        try:
            self.running = False
            self.thread.join(1)
        except AttributeError: pass

    def _send(self, cmd, timeout=1, retry=0):
        for t in range(0, retry + 1):
            assert not self.running_cmd
            # set expiration and event on command. Once self.runnning_cmd
            # is set access to this command from this tread is invalid
            # until event object is set.
            cmd.expiration = time.time() + timeout
            cmd.done = threading.Event()
            self.running_cmd = cmd
            # continue trying to commit command until session closed or command timeout
            while self.running and not cmd.done.is_set() and not self.ant_core.send(cmd):
                _LOG.warning("Device write timeout. Will keep trying.")
            # continue waiting for command completion until session closed
            while self.running and not cmd.done.is_set():
                cmd.done.wait(1)
            # cmd.done guarenetees a result is availible
            if cmd.done.is_set():
                try:
                    return cmd.result
                except AttributeError:
                    if t >= retry:
                        raise cmd.error
                    else:
                        _LOG.warning("Failed to execute command. %d tries remaining. %s", retry - t, cmd, exc_info=True)
            else:
                raise IOError(errno.EBADF, "Session closed.")

    def _handle_reply(self, cmd):
        if self.running_cmd and is_same_channel(self.running_cmd, cmd):
            if self.running_cmd.msg == antmsg.REQUEST_MESSAGE \
                    and self.running_cmd.args.msg_id == cmd.msg.msg_id:
                # running command is REQUEST_MESSAGE, return reply.
                # no validation is necessary.
                self._set_result(cmd)
            elif self.running_cmd.msg == antmsg.CLOSE_CHANNEL \
                    and cmd.msg == antmsg.CHANNEL_EVENT \
                    and cmd.args.msg_id == 1 \
                    and cmd.args.msg_code == antmsg.EVENT_CHANNEL_CLOSED:
                # channel closed, return event.
                # no validation necessary.
                self._set_result(cmd)
            elif cmd.msg == antmsg.CHANNEL_EVENT \
                    and cmd.args.msg_id == self.running_cmd.msg.msg_id:
                # incoming command is channel event matching
                # the currently running command.
                if cmd.args.msg_code != antmsg.RESPONSE_NO_ERROR:
                    # command failed
                    self._set_error(IOError(errno.EINVAL, "Failed to execute command message_code=%d. %s" % (cmd.args.msg_code, self.running_cmd)))
                else:
                    # command succeeded
                    self._set_result(cmd)
            elif cmd.msg == antmsg.STARTUP_MESSAGE \
                    and self.running_cmd.msg == antmsg.RESET_SYSTEM:
                # reset, return reply. FIXME, STARTUP MESSAGE
                # not implemented by older ANT hardware
                self._set_result(cmd)
        # check for timeout condition
        self._handle_timeout()

    def _handle_timeout(self):
        # if a command is currently running, check for timeout condition
        if self.running_cmd and time.time() > self.running_cmd.expiration:
            self._set_error(IOError(errno.ETIMEDOUT, "No reply to command. %s" %  self.running_cmd))

    def _set_result(self, result):
        if self.running_cmd:
            self.running_cmd.result = result
            self.running_cmd.done.set()
            self.running_cmd = None

    def _set_error(self, err):
        if self.running_cmd:
            self.running_cmd.error = err
            self.running_cmd.done.set()
            self.running_cmd = None

    def loop(self):
        try:
            while self.running:
                for cmd in self.ant_core.recv():
                    if not self.running: break
                    self._handle_reply(cmd)
                else:
                    if not self.running: break
                    self._handle_timeout()
        except Exception:
            _LOG.error("Caught Exception reading from device.", exc_info=True)
        finally:
            self.running_cmd = None
            self.running = False

# vim: ts=4 sts=4 et
<|endoftext|>


<commit_before>#!/usr/bin/env python
################################################################################
# DATE: 2016/May/06, rev: 2016/June/03
#
# SCRIPT: flag.py
#
# VERSION: 1.1
#
# AUTHOR: Miguel A Ibarra (miguelib@ufl.edu) Edited by: Matt Thoburn (mthoburn@ufl.edu)
#
# The output is a pdf of distributions, and a spreadsheet of flags
#
################################################################################
# Built-in packages
import logging

import argparse
from argparse import RawDescriptionHelpFormatter
import tempfile
import shutil
import os
from math import log, floor

# Add-on packages
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import numpy as np
import pandas as pd

# Local packages
import logger as sl
from interface import wideToDesign
from flags import Flags
from manager_color import colorHandler
from manager_figure import figureHandler
import module_lines as lines
import module_distribution as dist
import module_hist as hist

def getOptions(myopts=None):
    """Function to pull in arguments"""
    description="""The coefficient of variation (CV) is defined as the ratio
    of the sample standard deviation to the mean. It is a method to measure the
    variations of compounds. The variation of a peak intensity increases as
    its CV increases. And adjusted for the sample mean, CV does not have unit;
    thus, it is a standardized measurement for variation.

    A density plot of CVs for all compounds across samples by group is performed.
    And a set of flags of compounds with large CVs will be output. """

    parser=argparse.ArgumentParser(description=description,
                                formatter_class=RawDescriptionHelpFormatter)

    group1=parser.add_argument_group(title="Standard input",
                                description="Standard input for SECIM tools.")
    group1.add_argument("-i","--input", dest="input", action="store",
                        required=True, help="Input dataset in wide format.")
    group1.add_argument("-d","--design", dest="design", action="store",
                        required=True, help="Design file.")
    group1.add_argument("-id","--ID", dest="uniqID", action="store", required=True,
                         help="Name of the column with unique identifiers.")
    group1.add_argument("-f","--figure", dest="figure", action="store",
                        required=True, default="figure",
                        help="Name of the output PDF for CV plots.")
    group1.add_argument("-o","--flag", dest="flag", action="store",
                        required=True, default="RTflag",
                        help="Name of the output TSV for CV flags.")

    group2=parser.add_argument_group(title="Optional input",
                            description="Optional input for SECIM tools.")
    group2.add_argument("-g","--group", dest="group", action="store",
                        required=False, default=False,
                        help="""Add option to separate sample IDs by treatment
                        name. [optional]""")
    group2.add_argument("-c","--CVcutoff", dest="CVcutoff", action="store",
                        required=False, default=False,
                        help="""The default CV cutoff will flag 10 percent of
                        the rowIDs with larger CVs. If you want to set a CV
                        cutoff, put the number here. [optional]""")

    args=parser.parse_args()

    return(args)

def setflagByGroup(args, wide, dat):
    """
    Runs Count Values by group

    :Arguments:
        :type args: string
        :param args: command line args

        :type wide: pandas DataFrame
        :param wide: wide dataset

        :type dat: pandas DataFrame
        :param dat: wide and design datasets
    """
    #Open PDF pages
    with PdfPages(args.figure) as pdfOut:

        #Open CV dataframe
        CV=pd.DataFrame(index=wide.index)

        # Split design file by treatment group
        for title, group in dat.design.groupby(args.group):

            # Filter the wide file into a new dataframe
            currentFrame=wide[group.index]

            # Change dat.sampleIDs to match the design file
            dat.sampleIDs=group.index

            # Get CV and CVcutoff
            CV['cv_'+title], CVcutoff=setflag(args, currentFrame, dat, groupName=title)

        # Get max
        CV['cv']=CV.apply(np.max, axis=1)

        if not args.CVcutoff:
            CVcutoff=np.nanpercentile(CV['cv'].values, q=90)
            CVcutoff=round(CVcutoff, -int(floor(log(abs(CVcutoff), 10))) + 2)
        else:
            CVcutoff=float(args.CVcutoff)

        # Select palette
        ch=colorHandler('tableau','Tableau_10')
        logger.info(u"Using tableau Tableau_10 palette")

        # Get colors
        des,ucolor,combname=ch.getColors(dat.design,[args.group])
        logger.info("Plotting Data")

        # Split design file by treatment group
        for title, group in dat.design.groupby(args.group):

            # Open figure handler
            fh=figureHandler(proj='2d',figsize=(14,8))

            # Get xmin
            xmin=-np.nanpercentile(CV['cv_'+title].values,99)*0.2

            # Get xmax
            xmax=np.nanpercentile(CV['cv_'+title].values,99)*1.5

            # Plot histogram
            hist.serHist(ax=fh.ax[0],dat=CV['cv_'+title],color='grey',normed=1,
                        range=(xmin,xmax),bins=15)

            # Plot density plot
            dist.plotDensityDF(data=CV['cv_'+title],ax=fh.ax[0],lb="CV density",
                        colors=ucolor[title])

            # Plot cutoff
            lines.drawCutoffVert(ax=fh.ax[0],x=CVcutoff,
                                lb="Cutoff at: {0}".format(CVcutoff))

            # Plot legend
            fh.makeLegendLabel(ax=fh.ax[0])

            # Give format to the axis
            fh.formatAxis(yTitle='Density',xlim=(xmin,xmax), ylim="ignore",
                figTitle = "Density Plot of Coefficients of Variation in {0} {1}".format(args.group,title))

            # Shrink figure to fit legend
            fh.shrink()

            # Add plot to PDF
            fh.addToPdf(pdfPages=pdfOut)

        # Open new figureHandler instance
        fh=figureHandler(proj='2d')

        #Get xmin
        xmin=-np.nanpercentile(CV['cv'].values,99)*0.2

        #Get xmax
        xmax=np.nanpercentile(CV['cv'].values,99)*1.5

        # Create flag file instance
        flag=Flags(index=CV['cv'].index)

        # Split design file by treatment group
        for title, group in dat.design.groupby(args.group):

            # Plot density plot
            dist.plotDensityDF(data=CV["cv_"+title],ax=fh.ax[0],colors=ucolor[title],
                                lb="CV density in group {0}".format(title))

            # Create new flag row for each group
            flag.addColumn(column="flag_feature_big_CV_{0}".format(title),
                         mask=((CV['cv_'+title].get_values() > CVcutoff) |
                          CV['cv_'+title].isnull()))

        # Plot cutoff
        lines.drawCutoffVert(ax=fh.ax[0],x=CVcutoff,lb="Cutoff at: {0}".format(CVcutoff))

        # Plot legend
        fh.makeLegendLabel(ax=fh.ax[0])

        # Give format to the axis
        fh.formatAxis(yTitle="Density", xlim=(xmin,xmax),
            figTitle="Density Plot of Coefficients of Variation by {0}".format(args.group))

        # Shrink figure
        fh.shrink()

        # Add figure to PDF
        fh.addToPdf(pdfPages=pdfOut)

    # Write flag file
    flag.df_flags.to_csv(args.flag, sep='\t')

def setflag(args, wide, dat, groupName=''):
    """
    Runs Count Values by group

    :Arguments:
        :type args: string
        :param args: command line args

        :type wide: pandas DataFrame
        :param wide: wide dataset

        :type dat: pandas DataFrame
        :param dat: wide and design datasets

        :type groupName: string
        :param groupName: name of optional group
    """
    # Round all values to 3 significant digits
    DATround=wide.applymap(lambda x: x)

    # Create empty dataset with metabolites names as index
    DATstat=pd.DataFrame(index=DATround.index)

    #Calculate STD
    DATstat["std"] =DATround.apply(np.std, axis=1)

    #Calculate for mean
    DATstat["mean"]=DATround.apply(np.mean, axis=1)

    DATstat["cv"]  =abs(DATstat["std"] / DATstat["mean"])

    if not args.CVcutoff:
        CVcutoff=np.nanpercentile(DATstat['cv'].values, q=90)

        CVcutoff=round(CVcutoff, -int(floor(log(abs(CVcutoff), 10))) + 2)
    else:
        CVcutoff=float(args.CVcutoff)

    # Plot CVs
    logger.info("Plotting Data")
    if groupName == "":
        # Initiate figure instance
        fh=figureHandler(proj="2d", figsize=(14,8))

        # Calculate xmin
        xmin=-np.nanpercentile(DATstat['cv'].values,99)*0.2

        # Calculate xmax
        xmax=np.nanpercentile(DATstat['cv'].values,99)*1.5

        # Plot histogram
        hist.quickHist(ax=fh.ax[0],dat=DATstat['cv'],color="grey")

        # Plot density
        dist.plotDensityDF(data=DATstat['cv'],ax=fh.ax[0],colors='g',lb="CV density")

        # Ploot cutoff
        lines.drawCutoffVert(ax=fh.ax[0],x=CVcutoff,lb="Cutoff at: {0}".format(CVcutoff))

        # Plot legend
        fh.makeLegendLabel(ax=fh.ax[0])

        # Give format to axis
        fh.formatAxis(figTitle="Density Plot of Coefficients of Variation",
                    yTitle="Density",xlim=(xmin,xmax),ylim="ignore")

        # Shirnk plot
        fh.shrink()

        # Export figure
        fh.export(args.figure)

        # Create flag instance
        flag=Flags(index=DATstat.index)

        # Create new flag column with flags
        flag.addColumn(column='flag_feature_big_CV',
                    mask=((DATstat['cv'].get_values() > CVcutoff) |
                    DATstat['cv'].isnull()))

        # Write output
        flag.df_flags.to_csv(args.flag, sep='\t')
    else:
        return DATstat['cv'], CVcutoff

def main(args):
    """ Function to input all the arguments"""
    # Import data
    dat=wideToDesign(args.input, args.design, args.uniqID, group=args.group)

    # Use group separation or not depending on user input
    if not args.group:
        logger.info("running CV without groups")
        setflag(args, dat.wide, dat)
    else:
        logger.info("running CV with groups")
        dat.removeSingle()
        setflagByGroup(args, dat.wide, dat)

    logger.info("DONE")

if __name__ == '__main__':
    # Command line options
    args=getOptions()

    # Logger
    logger=logging.getLogger()
    sl.setLogger(logger)

    # Main
    main(args)

<commit_msg>CV flags logger that to interface<commit_after>#!/usr/bin/env python
################################################################################
# DATE: 2016/May/06, rev: 2016/June/03
#
# SCRIPT: flag.py
#
# VERSION: 1.1
#
# AUTHOR: Miguel A Ibarra (miguelib@ufl.edu) Edited by: Matt Thoburn (mthoburn@ufl.edu)
#
# The output is a pdf of distributions, and a spreadsheet of flags
#
################################################################################
# Built-in packages
import os
import shutil
import logging
import argparse
import tempfile
from math import log, floor
from argparse import RawDescriptionHelpFormatter

# Add-on packages
import matplotlib
matplotlib.use('Agg')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

# Local packages
from flags import Flags
from interface import wideToDesign
from manager_color import colorHandler
from manager_figure import figureHandler
import logger as sl
import module_hist as hist
import module_lines as lines
import module_distribution as dist

def getOptions(myopts=None):
    """Function to pull in arguments"""
    description="""The coefficient of variation (CV) is defined as the ratio
    of the sample standard deviation to the mean. It is a method to measure the
    variations of compounds. The variation of a peak intensity increases as
    its CV increases. And adjusted for the sample mean, CV does not have unit;
    thus, it is a standardized measurement for variation.

    A density plot of CVs for all compounds across samples by group is performed.
    And a set of flags of compounds with large CVs will be output. """

    parser=argparse.ArgumentParser(description=description,
                                formatter_class=RawDescriptionHelpFormatter)

    standard = parser.add_argument_group(title="Standard input",
                                description="Standard input for SECIM tools.")
    standard.add_argument("-i","--input", dest="input", action="store",
                        required=True, help="Input dataset in wide format.")
    standard.add_argument("-d","--design", dest="design", action="store",
                        required=True, help="Design file.")
    standard.add_argument("-id","--ID", dest="uniqID", action="store", required=True,
                         help="Name of the column with unique identifiers.")
    standard.add_argument("-g","--group", dest="group", action='store',
                        required=False, default=False, help="Name of column in "\
                        "design file with Group/treatment information.")
    standard.add_argument("-l","--levels",dest="levels",action="store",
                        required=False, default=False, help="Different groups to"\
                        "sort by separeted by commas.")

    tool = parser.add_argument_group(title="Tool specific input",
                                description="Input specific for the tool.")
    tool.add_argument("-c","--CVcutoff", dest="CVcutoff", action="store",
                        required=False, default=False, type=float,
                        help="The default CV cutoff will flag 10 percent of "\
                        "the rowIDs with larger CVs. If you want to set a CV "\
                        "cutoff, put the number here. [optional]")

    output = parser.add_argument_group(title="Output",
                            description="Paths for output files.")
    output.add_argument("-f","--figure", dest="figure", action="store",
                        required=True, default="figure",
                        help="Name of the output PDF for CV plots.")
    output.add_argument("-o","--flag", dest="flag", action="store",
                        required=True, default="RTflag",
                        help="Name of the output TSV for CV flags.")

    plot = parser.add_argument_group(title='Plot options')
    plot.add_argument("-pal","--palette",dest="palette",action='store',required=False,
                        default="tableau", help="Name of the palette to use.")
    plot.add_argument("-col","--color",dest="color",action="store",required=False,
                        default="Tableau_20", help="Name of a valid color scheme"\
                        " on the selected palette")
    args=parser.parse_args()

    # Standardize paths
    args.input  = os.path.abspath(args.input)
    args.design = os.path.abspath(args.design)
    args.figure = os.path.abspath(args.figure)
    args.flag   = os.path.abspath(args.flag)

    # if args.levels then split otherwise args.level = emptylist
    if args.levels:
        args.levels = args.levels.split(",")

    return(args)

def calculateCV(data, design, cutoff, levels):
    """
    Runs Count Values by group

    :Arguments:
        :type data: pandas.DataFrame
        :param data: wide dataset

        :type design: pandas.DataFrame
        :param design: design dataset

        :type cutoff: float
        :param cutoff: Value of cutoff if non provided it will be canculated.

        :type levels: str
        :param levels: Name of the groups to groupby
    """

    #Open CV dataframe
    CV = pd.DataFrame(index=data.index)

    # Split design file by treatment group
    for title, group in design.groupby(levels):
        # Create empty dataset with metabolites names as index and calculate their
        # standar deviation and mean
        DATstat=pd.DataFrame(index=data[group.index].index)
        DATstat.loc[:,"std"]  = np.std(data[group.index], axis=1)
        DATstat.loc[:,"mean"] = np.mean(data[group.index],axis=1)

        # Calculate the Coefficient of Variation for that group (if groups)
        # or al data (if not groups provided).
        CV.loc[:,"cv_"+title] = abs(DATstat["std"] / DATstat["mean"])

    # Get max CV
    CV.loc[:,'cv'] = CV.apply(np.max, axis=1)

    # Calculate a CVcutoff if user provides one use that instead.
    if not cutoff:
        CVcutoff = np.nanpercentile(CV['cv'].values, q=90)
        CVcutoff = round(CVcutoff, -int(floor(log(abs(CVcutoff), 10))) + 2)
    else:
        CVcutoff = cutoff

    return (CV, CVcutoff)

def plotCVplots(data, cutoff, palette, pdf):
    #Iterate over groups
    for name,group in palette.design.groupby(palette.combName):
        # Open figure handler
        fh=figureHandler(proj='2d',figsize=(14,8))

        # Get xmin and xmax
        xmin = -np.nanpercentile(data['cv_'+name].values,99)*0.2
        xmax = np.nanpercentile(data['cv_'+name].values,99)*1.5

        # Plot histogram
        hist.serHist(ax=fh.ax[0],dat=data['cv_'+name],color='grey',normed=1,
                    range=(xmin,xmax),bins=15)

        # Plot density plot
        dist.plotDensityDF(data=data['cv_'+name],ax=fh.ax[0],lb="CV density",
                           colors=palette.ugColors[name])

        # Plot cutoff
        lines.drawCutoffVert(ax=fh.ax[0],x=cutoff, lb="Cutoff at: {0}".format(cutoff))

        # Plot legend
        fh.makeLegendLabel(ax=fh.ax[0])

        # Give format to the axis
        fh.formatAxis(yTitle='Density',xlim=(xmin,xmax), ylim="ignore",
            figTitle = "Density Plot of Coefficients of Variation in {0}".format(name))

        # Shrink figure to fit legend
        fh.shrink()

        # Add plot to PDF
        fh.addToPdf(pdfPages=pdf)

def plotDistributions(data, cutoff, palette,pdf):
    # Open new figureHandler instance
    fh=figureHandler(proj='2d', figsize=(14,8))

    #Get xmin and xmax
    xmin = -np.nanpercentile(data['cv'].values,99)*0.2
    xmax = np.nanpercentile(data['cv'].values,99)*1.5

    # Split design file by treatment group and plot density plot
    for name, group in palette.design.groupby(palette.combName):
        dist.plotDensityDF(data=data["cv_"+name],ax=fh.ax[0],colors=palette.ugColors[name],
                            lb="{0}".format(name))

    # Plot cutoff
    lines.drawCutoffVert(ax=fh.ax[0],x=cutoff,lb="Cutoff at: {0}".format(cutoff))

    # Plot legend
    fh.makeLegendLabel(ax=fh.ax[0])

    # Give format to the axis
    fh.formatAxis(yTitle="Density", xlim=(xmin,xmax),
        figTitle="Density Plot of Coefficients of Variation by {0}".format(palette.combName))

    # Shrink figure
    fh.shrink()

    # Add figure to PDF
    fh.addToPdf(pdfPages=pdf)

def main(args):
    """ Function to input all the arguments"""
    # Checking if levels
    if args.levels and args.group:
        levels = [args.group]+args.levels
    elif args.group and not args.levels:
        levels = [args.group]
    else:
        levels = []
    logger.info(u"Groups used to color by: {0}".format(",".join(levels)))

    # Import data
    dat = wideToDesign(args.input, args.design, args.uniqID, group=args.group,
                    anno=args.levels, logger=logger)

    # Remove groups with just one element
    dat.removeSingle()

    # Treat everything as float and round it to 3 digits
    dat.wide = dat.wide.applymap(lambda x: round(x,3))

    # Get colors
    palette.getColors(dat.design,levels)

    # Use group separation or not depending on user input
    CV,CVcutoff = calculateCV(data=dat.wide, design=palette.design,
                              cutoff=args.CVcutoff, levels=palette.combName)

    # Plot CVplots for each group and a distribution plot for all groups together
    logger.info("Plotting Data")
    with PdfPages(args.figure) as pdf:
        plotCVplots      (data=CV, cutoff=CVcutoff, palette=palette, pdf=pdf)
        plotDistributions(data=CV, cutoff=CVcutoff, palette=palette, pdf=pdf)

    # Create flag file instance and output flags by group
    logger.info("Creatting Flags")
    flag = Flags(index=CV['cv'].index)
    for name, group in palette.design.groupby(palette.combName):
        flag.addColumn(column="flag_feature_big_CV_{0}".format(name),
                       mask=((CV['cv_'+name].get_values() > CVcutoff) | CV['cv_'+name].isnull()))

    # Write flag file
    flag.df_flags.to_csv(args.flag, sep='\t')

   # Finishing script
    logger.info("Script Complete!")

if __name__ == '__main__':
    # Command line options
    args=getOptions()

    # Logger
    logger=logging.getLogger()
    sl.setLogger(logger)

    # Standar logging
    logger.info(u"Importing data with following parameters: "\
                "\n\tWide: {0}"\
                "\n\tDesign: {1}"\
                "\n\tUnique ID: {2}"\
                "\n\tGroup: {3}".format(args.input, args.design, args.uniqID,
                    args.group))

    # Stablishing color palette for adata and cutoffs
    palette = colorHandler(pal=args.palette, col=args.color)
    logger.info(u"Using {0} color scheme from {1} palette".format(args.color,
                args.palette))
    # Main
    main(args)<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyPetsc4py(PythonPackage):
    """This package provides Python bindings for the PETSc package.
    """
    homepage = "https://pypi.python.org/pypi/petsc4py"
    url      = "https://pypi.io/packages/source/p/petsc4py/petsc4py-3.7.0.tar.gz"

    version('3.7.0', '816a20040a6a477bd637f397c9fb5b6d')

    depends_on('py-setuptools', type='build')
    depends_on('py-mpi4py', type=('build', 'run'))
    depends_on('petsc+mpi')
<commit_msg>py-petsc4py: Update version to 3.8.0 and add dependencies (#5792)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class PyPetsc4py(PythonPackage):
    """This package provides Python bindings for the PETSc package.
    """

    homepage = "https://pypi.python.org/pypi/petsc4py"
    url      = "https://pypi.io/packages/source/p/petsc4py/petsc4py-3.8.1.tar.gz"

    version('3.8.1', '5157220c2b81765c581d2b17c03259f8')
    version('3.8.0', '02029be4bdec904854f0e0692005fb06')
    version('3.7.0', '816a20040a6a477bd637f397c9fb5b6d')

    depends_on('python@2.6:2.8,3.3:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-mpi4py', type=('build', 'run'))
    depends_on('petsc+mpi')
<|endoftext|>


<commit_before>#! /usr/bin/python

# smallest number evenly divisable by 1 through 20
# numbers that are divisable by another number under 20,
# that other number does not need to be checked, so
# only these need to be checked:

divisables = [7, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]

def is_evenly_divisable(n):
    for i in divisables:
        if n % i != 0:
            return False
    return True
<commit_msg>wrote find_divisable and it works<commit_after>#! /usr/bin/python
# solution to Project Euler Problem

from numpy import prod

# smallest number evenly divisable by 1 through 20
# numbers that are divisable by another number under 20,
# that other number does not need to be checked, so
# only these need to be checked:

divisables = [7, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]

# the number has to be divisable by the product of all the primes
# prime and nonprime divisables:
prime_d = [7, 11, 13, 17, 19]
non_d = [12, 14, 15, 16, 18, 20]

# start with product of all primes and then mutiply that by
# a counter until we find a number divisable by all

def is_evenly_divisable(n):
    for i in divisables:
        if n % i != 0:
            return False
    return True

def find_divisable():
    product = prod(prime_d)
    count = 1
    while True:
        if is_evenly_divisable(product * count):
            return product * count
        count += 1

if __name__ == '__main__':
    print(find_divisable())<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This script checks the current state of the source code for minor issues,
including incorrect file permissions, presence of tabs, non-Unix line endings,
trailing whitespace, and presence of UTF-8 BOM.
Note: requires python 3, must be run from Mbed TLS root.
"""

import os
import argparse
import logging
import codecs
import re
import subprocess
import sys

class FileIssueTracker:
    """Base class for file-wide issue tracking.

    To implement a checker that processes a file as a whole, inherit from
    this class and implement `check_file_for_issue` and define ``heading``.

    ``suffix_exemptions``: files whose name ends with a string in this set
     will not be checked.

    ``path_exemptions``: files whose path (relative to the root of the source
    tree) matches this regular expression will not be checked. This can be
    ``None`` to match no path. Paths are normalized and converted to ``/``
    separators before matching.

    ``heading``: human-readable description of the issue
    """

    suffix_exemptions = frozenset()
    path_exemptions = None
    # heading must be defined in derived classes.
    # pylint: disable=no-member

    def __init__(self):
        self.files_with_issues = {}

    @staticmethod
    def normalize_path(filepath):
        """Normalize ``filepath`` with / as the directory separator."""
        filepath = os.path.normpath(filepath)
        # On Windows, we may have backslashes to separate directories.
        # We need slashes to match exemption lists.
        seps = os.path.sep
        if os.path.altsep is not None:
            seps += os.path.altsep
        return '/'.join(filepath.split(seps))

    def should_check_file(self, filepath):
        """Whether the given file name should be checked.

        Files whose name ends with a string listed in ``self.suffix_exemptions``
        or whose path matches ``self.path_exemptions`` will not be checked.
        """
        for files_exemption in self.suffix_exemptions:
            if filepath.endswith(files_exemption):
                return False
        if self.path_exemptions and \
           re.match(self.path_exemptions, self.normalize_path(filepath)):
            return False
        return True

    def check_file_for_issue(self, filepath):
        """Check the specified file for the issue that this class is for.

        Subclasses must implement this method.
        """
        raise NotImplementedError

    def record_issue(self, filepath, line_number):
        """Record that an issue was found at the specified location."""
        if filepath not in self.files_with_issues.keys():
            self.files_with_issues[filepath] = []
        self.files_with_issues[filepath].append(line_number)

    def output_file_issues(self, logger):
        """Log all the locations where the issue was found."""
        if self.files_with_issues.values():
            logger.info(self.heading)
            for filename, lines in sorted(self.files_with_issues.items()):
                if lines:
                    logger.info("{}: {}".format(
                        filename, ", ".join(str(x) for x in lines)
                    ))
                else:
                    logger.info(filename)
            logger.info("")

BINARY_FILE_PATH_RE_LIST = [
    r'docs/.*\.pdf\Z',
    r'programs/fuzz/corpuses/[^.]+\Z',
    r'tests/data_files/[^.]+\Z',
    r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z',
    r'tests/data_files/.*\.req\.[^/]+\Z',
    r'tests/data_files/.*malformed[^/]+\Z',
    r'tests/data_files/format_pkcs12\.fmt\Z',
]
BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST))

class LineIssueTracker(FileIssueTracker):
    """Base class for line-by-line issue tracking.

    To implement a checker that processes files line by line, inherit from
    this class and implement `line_with_issue`.
    """

    # Exclude binary files.
    path_exemptions = BINARY_FILE_PATH_RE

    def issue_with_line(self, line, filepath):
        """Check the specified line for the issue that this class is for.

        Subclasses must implement this method.
        """
        raise NotImplementedError

    def check_file_line(self, filepath, line, line_number):
        if self.issue_with_line(line, filepath):
            self.record_issue(filepath, line_number)

    def check_file_for_issue(self, filepath):
        """Check the lines of the specified file.

        Subclasses must implement the ``issue_with_line`` method.
        """
        with open(filepath, "rb") as f:
            for i, line in enumerate(iter(f.readline, b"")):
                self.check_file_line(filepath, line, i + 1)

def is_windows_file(filepath):
    _root, ext = os.path.splitext(filepath)
    return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj')

class PermissionIssueTracker(FileIssueTracker):
    """Track files with bad permissions.

    Files that are not executable scripts must not be executable."""

    heading = "Incorrect permissions:"

    # .py files can be either full scripts or modules, so they may or may
    # not be executable.
    suffix_exemptions = frozenset({".py"})

    def check_file_for_issue(self, filepath):
        is_executable = os.access(filepath, os.X_OK)
        should_be_executable = filepath.endswith((".sh", ".pl"))
        if is_executable != should_be_executable:
            self.files_with_issues[filepath] = None

class EndOfFileNewlineIssueTracker(FileIssueTracker):
    """Track files that end with an incomplete line
    (no newline character at the end of the last line)."""

    heading = "Missing newline at end of file:"

    path_exemptions = BINARY_FILE_PATH_RE

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            try:
                f.seek(-1, 2)
            except OSError:
                # This script only works on regular files. If we can't seek
                # 1 before the end, it means that this position is before
                # the beginning of the file, i.e. that the file is empty.
                return
            if f.read(1) != b"\n":
                self.files_with_issues[filepath] = None

class Utf8BomIssueTracker(FileIssueTracker):
    """Track files that start with a UTF-8 BOM.
    Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""

    heading = "UTF-8 BOM present:"

    suffix_exemptions = frozenset([".vcxproj", ".sln"])
    path_exemptions = BINARY_FILE_PATH_RE

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if f.read().startswith(codecs.BOM_UTF8):
                self.files_with_issues[filepath] = None

class UnixLineEndingIssueTracker(LineIssueTracker):
    """Track files with non-Unix line endings (i.e. files with CR)."""

    heading = "Non-Unix line endings:"

    def should_check_file(self, filepath):
        if not super().should_check_file(filepath):
            return False
        return not is_windows_file(filepath)

    def issue_with_line(self, line, _filepath):
        return b"\r" in line

class WindowsLineEndingIssueTracker(LineIssueTracker):
    """Track files with non-Windows line endings (i.e. CR or LF not in CRLF)."""

    heading = "Non-Windows line endings:"

    def should_check_file(self, filepath):
        if not super().should_check_file(filepath):
            return False
        return is_windows_file(filepath)

    def issue_with_line(self, line, _filepath):
        return not line.endswith(b"\r\n") or b"\r" in line[:-2]

class TrailingWhitespaceIssueTracker(LineIssueTracker):
    """Track lines with trailing whitespace."""

    heading = "Trailing whitespace:"
    suffix_exemptions = frozenset([".dsp", ".md"])

    def issue_with_line(self, line, _filepath):
        return line.rstrip(b"\r\n") != line.rstrip()

class TabIssueTracker(LineIssueTracker):
    """Track lines with tabs."""

    heading = "Tabs present:"
    suffix_exemptions = frozenset([
        ".pem", # some openssl dumps have tabs
        ".sln",
        "/Makefile",
        "/Makefile.inc",
        "/generate_visualc_files.pl",
    ])

    def issue_with_line(self, line, _filepath):
        return b"\t" in line

class MergeArtifactIssueTracker(LineIssueTracker):
    """Track lines with merge artifacts.
    These are leftovers from a ``git merge`` that wasn't fully edited."""

    heading = "Merge artifact:"

    def issue_with_line(self, line, _filepath):
        # Detect leftover git conflict markers.
        if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
            return True
        if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
            return True
        if line.rstrip(b'\r\n') == b'=======' and \
           not _filepath.endswith('.md'):
            return True
        return False

class IntegrityChecker:
    """Sanity-check files under the current directory."""

    def __init__(self, log_file):
        """Instantiate the sanity checker.
        Check files under the current directory.
        Write a report of issues to log_file."""
        self.check_repo_path()
        self.logger = None
        self.setup_logger(log_file)
        self.issues_to_check = [
            PermissionIssueTracker(),
            EndOfFileNewlineIssueTracker(),
            Utf8BomIssueTracker(),
            UnixLineEndingIssueTracker(),
            WindowsLineEndingIssueTracker(),
            TrailingWhitespaceIssueTracker(),
            TabIssueTracker(),
            MergeArtifactIssueTracker(),
        ]

    @staticmethod
    def check_repo_path():
        if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
            raise Exception("Must be run from Mbed TLS root")

    def setup_logger(self, log_file, level=logging.INFO):
        self.logger = logging.getLogger()
        self.logger.setLevel(level)
        if log_file:
            handler = logging.FileHandler(log_file)
            self.logger.addHandler(handler)
        else:
            console = logging.StreamHandler()
            self.logger.addHandler(console)

    @staticmethod
    def collect_files():
        bytes_output = subprocess.check_output(['git', 'ls-files', '-z'])
        bytes_filepaths = bytes_output.split(b'\0')[:-1]
        ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths)
        # Prepend './' to files in the top-level directory so that
        # something like `'/Makefile' in fp` matches in the top-level
        # directory as well as in subdirectories.
        return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp)
                for fp in ascii_filepaths]

    def check_files(self):
        for issue_to_check in self.issues_to_check:
            for filepath in self.collect_files():
                if issue_to_check.should_check_file(filepath):
                    issue_to_check.check_file_for_issue(filepath)

    def output_issues(self):
        integrity_return_code = 0
        for issue_to_check in self.issues_to_check:
            if issue_to_check.files_with_issues:
                integrity_return_code = 1
            issue_to_check.output_file_issues(self.logger)
        return integrity_return_code

def run_main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "-l", "--log_file", type=str, help="path to optional output log",
    )
    check_args = parser.parse_args()
    integrity_check = IntegrityChecker(check_args.log_file)
    integrity_check.check_files()
    return_code = integrity_check.output_issues()
    sys.exit(return_code)

if __name__ == "__main__":
    run_main()
<commit_msg>Test shebang lines<commit_after>#!/usr/bin/env python3

# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This script checks the current state of the source code for minor issues,
including incorrect file permissions, presence of tabs, non-Unix line endings,
trailing whitespace, and presence of UTF-8 BOM.
Note: requires python 3, must be run from Mbed TLS root.
"""

import os
import argparse
import logging
import codecs
import re
import subprocess
import sys

class FileIssueTracker:
    """Base class for file-wide issue tracking.

    To implement a checker that processes a file as a whole, inherit from
    this class and implement `check_file_for_issue` and define ``heading``.

    ``suffix_exemptions``: files whose name ends with a string in this set
     will not be checked.

    ``path_exemptions``: files whose path (relative to the root of the source
    tree) matches this regular expression will not be checked. This can be
    ``None`` to match no path. Paths are normalized and converted to ``/``
    separators before matching.

    ``heading``: human-readable description of the issue
    """

    suffix_exemptions = frozenset()
    path_exemptions = None
    # heading must be defined in derived classes.
    # pylint: disable=no-member

    def __init__(self):
        self.files_with_issues = {}

    @staticmethod
    def normalize_path(filepath):
        """Normalize ``filepath`` with / as the directory separator."""
        filepath = os.path.normpath(filepath)
        # On Windows, we may have backslashes to separate directories.
        # We need slashes to match exemption lists.
        seps = os.path.sep
        if os.path.altsep is not None:
            seps += os.path.altsep
        return '/'.join(filepath.split(seps))

    def should_check_file(self, filepath):
        """Whether the given file name should be checked.

        Files whose name ends with a string listed in ``self.suffix_exemptions``
        or whose path matches ``self.path_exemptions`` will not be checked.
        """
        for files_exemption in self.suffix_exemptions:
            if filepath.endswith(files_exemption):
                return False
        if self.path_exemptions and \
           re.match(self.path_exemptions, self.normalize_path(filepath)):
            return False
        return True

    def check_file_for_issue(self, filepath):
        """Check the specified file for the issue that this class is for.

        Subclasses must implement this method.
        """
        raise NotImplementedError

    def record_issue(self, filepath, line_number):
        """Record that an issue was found at the specified location."""
        if filepath not in self.files_with_issues.keys():
            self.files_with_issues[filepath] = []
        self.files_with_issues[filepath].append(line_number)

    def output_file_issues(self, logger):
        """Log all the locations where the issue was found."""
        if self.files_with_issues.values():
            logger.info(self.heading)
            for filename, lines in sorted(self.files_with_issues.items()):
                if lines:
                    logger.info("{}: {}".format(
                        filename, ", ".join(str(x) for x in lines)
                    ))
                else:
                    logger.info(filename)
            logger.info("")

BINARY_FILE_PATH_RE_LIST = [
    r'docs/.*\.pdf\Z',
    r'programs/fuzz/corpuses/[^.]+\Z',
    r'tests/data_files/[^.]+\Z',
    r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z',
    r'tests/data_files/.*\.req\.[^/]+\Z',
    r'tests/data_files/.*malformed[^/]+\Z',
    r'tests/data_files/format_pkcs12\.fmt\Z',
]
BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST))

class LineIssueTracker(FileIssueTracker):
    """Base class for line-by-line issue tracking.

    To implement a checker that processes files line by line, inherit from
    this class and implement `line_with_issue`.
    """

    # Exclude binary files.
    path_exemptions = BINARY_FILE_PATH_RE

    def issue_with_line(self, line, filepath):
        """Check the specified line for the issue that this class is for.

        Subclasses must implement this method.
        """
        raise NotImplementedError

    def check_file_line(self, filepath, line, line_number):
        if self.issue_with_line(line, filepath):
            self.record_issue(filepath, line_number)

    def check_file_for_issue(self, filepath):
        """Check the lines of the specified file.

        Subclasses must implement the ``issue_with_line`` method.
        """
        with open(filepath, "rb") as f:
            for i, line in enumerate(iter(f.readline, b"")):
                self.check_file_line(filepath, line, i + 1)

def is_windows_file(filepath):
    _root, ext = os.path.splitext(filepath)
    return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj')

class PermissionIssueTracker(FileIssueTracker):
    """Track files with bad permissions.

    Files that are not executable scripts must not be executable."""

    heading = "Incorrect permissions:"

    # .py files can be either full scripts or modules, so they may or may
    # not be executable.
    suffix_exemptions = frozenset({".py"})

    def check_file_for_issue(self, filepath):
        is_executable = os.access(filepath, os.X_OK)
        should_be_executable = filepath.endswith((".sh", ".pl"))
        if is_executable != should_be_executable:
            self.files_with_issues[filepath] = None

class ShebangIssueTracker(FileIssueTracker):
    """Track files with a bad, missing or extraneous shebang line.

    Executable scripts must start with a valid shebang (#!) line.
    """

    heading = "Invalid shebang line:"

    # Allow either /bin/sh, /bin/bash, or /usr/bin/env.
    # Allow at most one argument (this is a Linux limitation).
    # For sh and bash, the argument if present must be options.
    # For env, the argument must be the base name of the interpeter.
    _shebang_re = re.compile(rb'^#! ?(?:/bin/(bash|sh)(?: -[^\n ]*)?'
                             rb'|/usr/bin/env ([^\n /]+))$')
    _extensions = {
        b'bash': 'sh',
        b'perl': 'pl',
        b'python3': 'py',
        b'sh': 'sh',
    }

    def is_valid_shebang(self, first_line, filepath):
        m = re.match(self._shebang_re, first_line)
        if not m:
            return False
        interpreter = m.group(1) or m.group(2)
        if interpreter not in self._extensions:
            return False
        if not filepath.endswith('.' + self._extensions[interpreter]):
            return False
        return True

    def check_file_for_issue(self, filepath):
        is_executable = os.access(filepath, os.X_OK)
        with open(filepath, "rb") as f:
            first_line = f.readline()
        if first_line.startswith(b'#!'):
            if not is_executable:
                # Shebang on a non-executable file
                self.files_with_issues[filepath] = None
            elif not self.is_valid_shebang(first_line, filepath):
                self.files_with_issues[filepath] = [1]
        elif is_executable:
            # Executable without a shebang
            self.files_with_issues[filepath] = None

class EndOfFileNewlineIssueTracker(FileIssueTracker):
    """Track files that end with an incomplete line
    (no newline character at the end of the last line)."""

    heading = "Missing newline at end of file:"

    path_exemptions = BINARY_FILE_PATH_RE

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            try:
                f.seek(-1, 2)
            except OSError:
                # This script only works on regular files. If we can't seek
                # 1 before the end, it means that this position is before
                # the beginning of the file, i.e. that the file is empty.
                return
            if f.read(1) != b"\n":
                self.files_with_issues[filepath] = None

class Utf8BomIssueTracker(FileIssueTracker):
    """Track files that start with a UTF-8 BOM.
    Files should be ASCII or UTF-8. Valid UTF-8 does not start with a BOM."""

    heading = "UTF-8 BOM present:"

    suffix_exemptions = frozenset([".vcxproj", ".sln"])
    path_exemptions = BINARY_FILE_PATH_RE

    def check_file_for_issue(self, filepath):
        with open(filepath, "rb") as f:
            if f.read().startswith(codecs.BOM_UTF8):
                self.files_with_issues[filepath] = None

class UnixLineEndingIssueTracker(LineIssueTracker):
    """Track files with non-Unix line endings (i.e. files with CR)."""

    heading = "Non-Unix line endings:"

    def should_check_file(self, filepath):
        if not super().should_check_file(filepath):
            return False
        return not is_windows_file(filepath)

    def issue_with_line(self, line, _filepath):
        return b"\r" in line

class WindowsLineEndingIssueTracker(LineIssueTracker):
    """Track files with non-Windows line endings (i.e. CR or LF not in CRLF)."""

    heading = "Non-Windows line endings:"

    def should_check_file(self, filepath):
        if not super().should_check_file(filepath):
            return False
        return is_windows_file(filepath)

    def issue_with_line(self, line, _filepath):
        return not line.endswith(b"\r\n") or b"\r" in line[:-2]

class TrailingWhitespaceIssueTracker(LineIssueTracker):
    """Track lines with trailing whitespace."""

    heading = "Trailing whitespace:"
    suffix_exemptions = frozenset([".dsp", ".md"])

    def issue_with_line(self, line, _filepath):
        return line.rstrip(b"\r\n") != line.rstrip()

class TabIssueTracker(LineIssueTracker):
    """Track lines with tabs."""

    heading = "Tabs present:"
    suffix_exemptions = frozenset([
        ".pem", # some openssl dumps have tabs
        ".sln",
        "/Makefile",
        "/Makefile.inc",
        "/generate_visualc_files.pl",
    ])

    def issue_with_line(self, line, _filepath):
        return b"\t" in line

class MergeArtifactIssueTracker(LineIssueTracker):
    """Track lines with merge artifacts.
    These are leftovers from a ``git merge`` that wasn't fully edited."""

    heading = "Merge artifact:"

    def issue_with_line(self, line, _filepath):
        # Detect leftover git conflict markers.
        if line.startswith(b'<<<<<<< ') or line.startswith(b'>>>>>>> '):
            return True
        if line.startswith(b'||||||| '): # from merge.conflictStyle=diff3
            return True
        if line.rstrip(b'\r\n') == b'=======' and \
           not _filepath.endswith('.md'):
            return True
        return False

class IntegrityChecker:
    """Sanity-check files under the current directory."""

    def __init__(self, log_file):
        """Instantiate the sanity checker.
        Check files under the current directory.
        Write a report of issues to log_file."""
        self.check_repo_path()
        self.logger = None
        self.setup_logger(log_file)
        self.issues_to_check = [
            PermissionIssueTracker(),
            ShebangIssueTracker(),
            EndOfFileNewlineIssueTracker(),
            Utf8BomIssueTracker(),
            UnixLineEndingIssueTracker(),
            WindowsLineEndingIssueTracker(),
            TrailingWhitespaceIssueTracker(),
            TabIssueTracker(),
            MergeArtifactIssueTracker(),
        ]

    @staticmethod
    def check_repo_path():
        if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
            raise Exception("Must be run from Mbed TLS root")

    def setup_logger(self, log_file, level=logging.INFO):
        self.logger = logging.getLogger()
        self.logger.setLevel(level)
        if log_file:
            handler = logging.FileHandler(log_file)
            self.logger.addHandler(handler)
        else:
            console = logging.StreamHandler()
            self.logger.addHandler(console)

    @staticmethod
    def collect_files():
        bytes_output = subprocess.check_output(['git', 'ls-files', '-z'])
        bytes_filepaths = bytes_output.split(b'\0')[:-1]
        ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths)
        # Prepend './' to files in the top-level directory so that
        # something like `'/Makefile' in fp` matches in the top-level
        # directory as well as in subdirectories.
        return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp)
                for fp in ascii_filepaths]

    def check_files(self):
        for issue_to_check in self.issues_to_check:
            for filepath in self.collect_files():
                if issue_to_check.should_check_file(filepath):
                    issue_to_check.check_file_for_issue(filepath)

    def output_issues(self):
        integrity_return_code = 0
        for issue_to_check in self.issues_to_check:
            if issue_to_check.files_with_issues:
                integrity_return_code = 1
            issue_to_check.output_file_issues(self.logger)
        return integrity_return_code

def run_main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "-l", "--log_file", type=str, help="path to optional output log",
    )
    check_args = parser.parse_args()
    integrity_check = IntegrityChecker(check_args.log_file)
    integrity_check.check_files()
    return_code = integrity_check.output_issues()
    sys.exit(return_code)

if __name__ == "__main__":
    run_main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Main module for maxify command line application.
"""

import argparse
import cmd
import shlex

import colorama
from maxify.units import ParsingError
from termcolor import colored

from maxify.config import (
    import_config,
    ImportStrategy,
    ProjectConflictError,
    ConfigError
)
from maxify.repo import Repository, Projects
from maxify.log import enable_loggers

help_texts = {
    "projects": """Prints a list of all available projects.

""",
    "import": """Imports projects defined in a specified configuration file.

Configuration files can either by a YAML file or a Python module.

Example:

> import projects.yaml

""",
    "switch": """Switches to the project with the specified name.

Examples:

> switch sample1
> switch scopetastic/maxify

"""
}

class MaxifyCmd(cmd.Cmd):
    """Command interpreter used for accepting commands from the user to
    manage a Maxify project.

    """

    def __init__(self, stdin=None, stdout=None, use_color=True):
        cmd.Cmd.__init__(self, stdin=stdin, stdout=stdout)
        self.intro = "Maxify programmer time tracker client"
        self.prompt = "> "
        self.current_project = None
        self.use_color = use_color
        self.projects = Projects()
        self._generate_help_funcs()

    def _generate_help_funcs(self):
        for command in help_texts:
            self._generate_help_func(command)

    def _generate_help_func(self, command):
        func_name = "help_" + command

        def help_func():
            self._title("Help - " + command)
            self._print(help_texts[command])

        setattr(self, func_name, help_func)

    def cmdloop(self, project_name=None):
        if project_name:
            self._set_current_project(project_name)
            if self.current_project:
                self.intro = self.intro + \
                    "\n\n" + \
                    "Switched to project '{0}'".format(project_name)
            else:
                self.intro = self.intro + \
                    "\n\nNo project found named '{0}'".format(project_name)

        cmd.Cmd.cmdloop(self)

    def _set_current_project(self, project_name):
        self.current_project = self.projects.get(project_name)

    def emptyline(self):
        """Handles an empty line (does nothing)."""
        pass

    # TODO: Keep this?
    def default(self, line):
        if self.current_project:
            task = self.current_project.task(line.strip())
        else:
            task = None

        if task:
            self._print_task(task)
            return

        cmd.Cmd.default(self, line)

    ########################################
    # Command - quit/exit
    ########################################

    def do_exit(self, line):
        """Exit the application."""
        return True

    def do_quit(self, line):
        """Exit the application."""
        return True

    ########################################
    # Command - switch
    ########################################

    def do_switch(self, line):
        """Switch to a project with the provided name."""
        self._set_current_project(line.strip())

        if not self.current_project:
            self._error("No project found named '{0}'".format(line))
        else:
            self._success("Switched to project '{0}'".format(
                self.current_project.name))

    def complete_switch(self, text, line, beginx, endidx):
        start_index = len("switch ")
        if beginx == start_index:
            return self.projects.matching_name(text)

        # If beginning index not immediately after command keyword, then
        # readline found a organization delimiter, so handle it correctly
        # in search for matches, then strip it out of returned results for
        # readline tokenized completion logic.
        organization = line[start_index:beginx]
        partial_name = organization + text
        matches = self.projects.matching_name(partial_name)
        return [m.replace(organization, "") for m in matches]

    ########################################
    # Command - projects
    ########################################

    def do_projects(self, line):
        """Lists all projects current defined in the user's data file.
        """
        projects = self.projects.all()

        if not len(projects):
            self._print("\nNo projects found\n")
            return

        orgs = {project.organization for project in projects}
        by_org = {org: [p for p in projects if p.organization == org]
                  for org in orgs}

        for org in sorted(orgs, key=lambda o: o if o is not None else ""):
            if org:
                self._title(org)
            else:
                self._title("default")

            for project in by_org[org]:
                self._print_project_summary(project)

        self._print("")

    def _print_project_summary(self, project):
        project_str = " * {name} - {desc}".format(
            name=project.qualified_name,
            desc=project.desc if project.desc else "No description provided")
        self._print(project_str)

    ########################################
    # Command - import
    ########################################

    def do_import(self, line):
        """Import projects from a configuration file.

        """
        # First, attempt an import and abort if a conflict happens
        file_path = line.strip()
        try:
            projects = import_config(file_path, ImportStrategy.abort)
            conflict = False
        except ProjectConflictError:
            conflict = True
        except ConfigError as e:
            self._error(str(e))
            return

        if conflict:
            self._warning("Conflicts found between current projects and "
                          "projects defined in '{}'.".format(file_path))

            self._print("\nYou can select one of the following options for "
                        "continuing with the import:\n"
                        " - (A)bort -  Stops the import and makes no changes.\n"
                        " - (M)erge - Merges current projects with those "
                        "being imported.\n"
                        " - (R)eplace - Replaces current projects with the "
                        "ones being imported. Any existing conflicting "
                        "projects will be deleted along with their data.\n\n")

            response = input("What would you like to do?: ")

            if response.upper() == "M":
                self._print("Merging projects...\n")
                projects = import_config(file_path, ImportStrategy.merge)
            elif response.upper() == "R":
                self._print("Replacing projects...\n")
                projects = import_config(file_path, ImportStrategy.overwrite)
            else:
                self._print("Import aborted.\n")
                projects = None

        if not projects:
            return

        self._print("\nThe following projects were imported:")
        for project in projects:
            self._print_project_summary(project)

        self._print("\n")

    ########################################
    # Command - metrics
    ########################################

    def do_metrics(self, line):
        """Print out metrics available for the current project."""
        if not self.current_project:
            self._error("Please select a project first using the 'switch' "
                        "command")
            return

        self._title(self.current_project.name + " Metrics:")
        for metric in sorted(self.current_project.metrics,
                             key=lambda metric: metric.name):
            self._print(" * {0} ({1})".format(metric.name,
                                              metric.units.display_name()))

            if metric.desc:
                self._print("   - Description: " + metric.desc)

            if metric.value_range:
                self._print("   - Possible Values: "
                            + ", ".join(map(str, metric.value_range)))

            if metric.default_value:
                self._print("   - Default Value: " + str(metric.default_value))

        self._print("")

    ########################################
    # Command - task
    ########################################

    def do_task(self, line):
        """Create a task or edit an existing task."""
        line = line.strip()
        if not line:
            self._error("You must specify a task to create or update.\n"
                        "Usage: task [TASK_NAME]")
            return

        tokens = shlex.split(line)
        task_name = tokens[0]
        args = tokens[1:]

        if len(args):
            success = self._update_task(task_name, args)
        else:
            success = self._update_task_interactive(task_name)

        if success:
            self._success("Task updated")

    def _update_task_interactive(self, task_name):
        return False

    def _update_task(self, task_name, args):
        metrics = []
        for i in range(0, len(args), 2):
            metric_name = args[i]
            value_str = args[i + 1]
            metric = self._get_metric(metric_name)
            if not metric:
                self._error("Invalid metric: " + metric_name)
                return

            try:
                value = metric.units.parse(value_str)
            except ParsingError as e:
                self._error(str(e))
                return

            metrics.append((metric, value))

        self.current_project.update_task(task_name, metrics=metrics)

        return True

    def _get_metric(self, metric_name):
        # First, try to use the string as is:
        metric = self.current_project.metric(metric_name)
        if metric:
            return metric

        metric_name = metric_name.replace("_", " ").title()
        return self.current_project.metric(metric_name)

    def _print_task(self, task):
        output = ["Created: " + str(task.created)]
        output.append("Last Updated: " + str(task.last_updated))
        output.append("\n")
        for data_point in task.data_points:
            output.append(" {0} -> {1}".format(data_point.metric,
                                               data_point.value))

        self._print("\n".join(output) + "\n")

    ########################################
    # Utility methods
    ########################################

    def _title(self, line):
        self._print("\n" + line)
        self._print("-" * min(len(line), 80) + "\n")

    def _success(self, msg):
        self._print(msg, 'green')
        print()

    def _warning(self, msg):
        self._print("Warning: " + msg, "yellow")
        print()

    def _error(self, msg):
        self._print("Error: " + msg, "red")
        print()

    def _print(self, msg=None, color=None):
        if msg and color and self.use_color:
            msg = colored(msg, color)

        print(msg, file=self.stdout)

def main():
    parser = argparse.ArgumentParser(description="Maxify programmer time "
                                                 "tracker client")
    parser.add_argument("-p",
                        "--project",
                        help="Name of project to start tracking time against "
                             "once the client starts. For a project belonging "
                             "to a particular organization, prefix the name "
                             "with the organization, like: scopetastic/maxify.")
    parser.add_argument("-f",
                        "--data-file",
                        default="maxify.db",
                        help="Path to Maxify data file. By default, this is "
                             "'maxify.db' in the current directory.")

    parser.add_argument("-x",
                        "--debug",
                        action="store_true",
                        help="Print debugging statements during execution.")

    args = parser.parse_args()

    if args.debug:
        enable_loggers()

    colorama.init()
    Repository.init(args.data_file)

    interpreter = MaxifyCmd()
    interpreter.cmdloop(args.project)

if __name__ == "__main__":
    main()
<commit_msg>[#5]: Added help for task command<commit_after>#!/usr/bin/env python

"""
Main module for maxify command line application.
"""

import argparse
import cmd
import shlex

import colorama
from maxify.units import ParsingError
from termcolor import colored

from maxify.config import (
    import_config,
    ImportStrategy,
    ProjectConflictError,
    ConfigError
)
from maxify.repo import Repository, Projects
from maxify.log import enable_loggers

help_texts = {
    "projects": """Prints a list of all available projects.

""",
    "import": """Imports projects defined in a specified configuration file.

Configuration files can either by a YAML file or a Python module.

Example:

    > import projects.yaml

""",
    "switch": """Switches to the project with the specified name.

Examples:

    > switch sample1
    > switch scopetastic/maxify

""",
    "metrics": """Displays available metrics for tasks contained in the current\
 project.

Example:

    > metrics

""",
    "task": """Create or update a task associated with the current project.

Tasks can be created or updated either in interactive or non-interactive mode.

Examples (non-interactive):

    > task maxify-1 compile_time 2hrs research_time 1hr
    > task maxify-1 "debug time" 20mins

Example (interactive):

    > task maxify-1
    Compile Time: 2 hrs
    Research Time: 1 hr
    Debug Time: 20 mins
    ...

"""
}

class MaxifyCmd(cmd.Cmd):
    """Command interpreter used for accepting commands from the user to
    manage a Maxify project.

    """

    def __init__(self, stdin=None, stdout=None, use_color=True):
        cmd.Cmd.__init__(self, stdin=stdin, stdout=stdout)
        self.intro = "Maxify programmer time tracker client"
        self.prompt = "> "
        self.current_project = None
        self.use_color = use_color
        self.projects = Projects()
        self._generate_help_funcs()

    def _generate_help_funcs(self):
        for command in help_texts:
            self._generate_help_func(command)

    def _generate_help_func(self, command):
        func_name = "help_" + command

        def help_func():
            self._title("Help - " + command)
            self._print(help_texts[command])

        setattr(self, func_name, help_func)

    def cmdloop(self, project_name=None):
        if project_name:
            self._set_current_project(project_name)
            if self.current_project:
                self.intro = self.intro + \
                    "\n\n" + \
                    "Switched to project '{0}'".format(project_name)
            else:
                self.intro = self.intro + \
                    "\n\nNo project found named '{0}'".format(project_name)

        cmd.Cmd.cmdloop(self)

    def _set_current_project(self, project_name):
        self.current_project = self.projects.get(project_name)

    def emptyline(self):
        """Handles an empty line (does nothing)."""
        pass

    # TODO: Keep this?
    def default(self, line):
        if self.current_project:
            task = self.current_project.task(line.strip())
        else:
            task = None

        if task:
            self._print_task(task)
            return

        cmd.Cmd.default(self, line)

    ########################################
    # Command - quit/exit
    ########################################

    def do_exit(self, line):
        """Exit the application."""
        return True

    def do_quit(self, line):
        """Exit the application."""
        return True

    ########################################
    # Command - switch
    ########################################

    def do_switch(self, line):
        """Switch to a project with the provided name."""
        self._set_current_project(line.strip())

        if not self.current_project:
            self._error("No project found named '{0}'".format(line))
        else:
            self._success("Switched to project '{0}'".format(
                self.current_project.name))

    def complete_switch(self, text, line, beginx, endidx):
        start_index = len("switch ")
        if beginx == start_index:
            return self.projects.matching_name(text)

        # If beginning index not immediately after command keyword, then
        # readline found a organization delimiter, so handle it correctly
        # in search for matches, then strip it out of returned results for
        # readline tokenized completion logic.
        organization = line[start_index:beginx]
        partial_name = organization + text
        matches = self.projects.matching_name(partial_name)
        return [m.replace(organization, "") for m in matches]

    ########################################
    # Command - projects
    ########################################

    def do_projects(self, line):
        """Lists all projects current defined in the user's data file.
        """
        projects = self.projects.all()

        if not len(projects):
            self._print("\nNo projects found\n")
            return

        orgs = {project.organization for project in projects}
        by_org = {org: [p for p in projects if p.organization == org]
                  for org in orgs}

        for org in sorted(orgs, key=lambda o: o if o is not None else ""):
            if org:
                self._title(org)
            else:
                self._title("default")

            for project in by_org[org]:
                self._print_project_summary(project)

        self._print("")

    def _print_project_summary(self, project):
        project_str = " * {name} - {desc}".format(
            name=project.qualified_name,
            desc=project.desc if project.desc else "No description provided")
        self._print(project_str)

    ########################################
    # Command - import
    ########################################

    def do_import(self, line):
        """Import projects from a configuration file.

        """
        # First, attempt an import and abort if a conflict happens
        file_path = line.strip()
        try:
            projects = import_config(file_path, ImportStrategy.abort)
            conflict = False
        except ProjectConflictError:
            conflict = True
        except ConfigError as e:
            self._error(str(e))
            return

        if conflict:
            self._warning("Conflicts found between current projects and "
                          "projects defined in '{}'.".format(file_path))

            self._print("\nYou can select one of the following options for "
                        "continuing with the import:\n"
                        " - (A)bort -  Stops the import and makes no changes.\n"
                        " - (M)erge - Merges current projects with those "
                        "being imported.\n"
                        " - (R)eplace - Replaces current projects with the "
                        "ones being imported. Any existing conflicting "
                        "projects will be deleted along with their data.\n\n")

            response = input("What would you like to do?: ")

            if response.upper() == "M":
                self._print("Merging projects...\n")
                projects = import_config(file_path, ImportStrategy.merge)
            elif response.upper() == "R":
                self._print("Replacing projects...\n")
                projects = import_config(file_path, ImportStrategy.overwrite)
            else:
                self._print("Import aborted.\n")
                projects = None

        if not projects:
            return

        self._print("\nThe following projects were imported:")
        for project in projects:
            self._print_project_summary(project)

        self._print("\n")

    ########################################
    # Command - metrics
    ########################################

    def do_metrics(self, line):
        """Print out metrics available for the current project."""
        if not self.current_project:
            self._error("Please select a project first using the 'switch' "
                        "command")
            return

        self._title(self.current_project.name + " Metrics:")
        for metric in sorted(self.current_project.metrics,
                             key=lambda metric: metric.name):
            self._print(" * {0} ({1})".format(metric.name,
                                              metric.units.display_name()))

            if metric.desc:
                self._print("   - Description: " + metric.desc)

            if metric.value_range:
                self._print("   - Possible Values: "
                            + ", ".join(map(str, metric.value_range)))

            if metric.default_value:
                self._print("   - Default Value: " + str(metric.default_value))

        self._print("")

    ########################################
    # Command - task
    ########################################

    def do_task(self, line):
        """Create a task or edit an existing task."""
        line = line.strip()
        if not line:
            self._error("You must specify a task to create or update.\n"
                        "Usage: task [TASK_NAME]")
            return

        tokens = shlex.split(line)
        task_name = tokens[0]
        args = tokens[1:]

        if len(args):
            success = self._update_task(task_name, args)
        else:
            success = self._update_task_interactive(task_name)

        if success:
            self._success("Task updated")

    def _update_task_interactive(self, task_name):
        return False

    def _update_task(self, task_name, args):
        metrics = []
        for i in range(0, len(args), 2):
            metric_name = args[i]
            value_str = args[i + 1]
            metric = self._get_metric(metric_name)
            if not metric:
                self._error("Invalid metric: " + metric_name)
                return

            try:
                value = metric.units.parse(value_str)
            except ParsingError as e:
                self._error(str(e))
                return

            metrics.append((metric, value))

        self.current_project.update_task(task_name, metrics=metrics)

        return True

    def _get_metric(self, metric_name):
        # First, try to use the string as is:
        metric = self.current_project.metric(metric_name)
        if metric:
            return metric

        metric_name = metric_name.replace("_", " ").title()
        return self.current_project.metric(metric_name)

    def _print_task(self, task):
        output = ["Created: " + str(task.created)]
        output.append("Last Updated: " + str(task.last_updated))
        output.append("\n")
        for data_point in task.data_points:
            output.append(" {0} -> {1}".format(data_point.metric,
                                               data_point.value))

        self._print("\n".join(output) + "\n")

    ########################################
    # Utility methods
    ########################################

    def _title(self, line):
        self._print("\n" + line)
        self._print("-" * min(len(line), 80) + "\n")

    def _success(self, msg):
        self._print(msg, 'green')
        print()

    def _warning(self, msg):
        self._print("Warning: " + msg, "yellow")
        print()

    def _error(self, msg):
        self._print("Error: " + msg, "red")
        print()

    def _print(self, msg=None, color=None):
        if msg and color and self.use_color:
            msg = colored(msg, color)

        print(msg, file=self.stdout)

def main():
    parser = argparse.ArgumentParser(description="Maxify programmer time "
                                                 "tracker client")
    parser.add_argument("-p",
                        "--project",
                        help="Name of project to start tracking time against "
                             "once the client starts. For a project belonging "
                             "to a particular organization, prefix the name "
                             "with the organization, like: scopetastic/maxify.")
    parser.add_argument("-f",
                        "--data-file",
                        default="maxify.db",
                        help="Path to Maxify data file. By default, this is "
                             "'maxify.db' in the current directory.")

    parser.add_argument("-x",
                        "--debug",
                        action="store_true",
                        help="Print debugging statements during execution.")

    args = parser.parse_args()

    if args.debug:
        enable_loggers()

    colorama.init()
    Repository.init(args.data_file)

    interpreter = MaxifyCmd()
    interpreter.cmdloop(args.project)

if __name__ == "__main__":
    main()
<|endoftext|>